nidx-binding 6.3.5.post224__tar.gz → 6.3.5.post234__tar.gz

nidx_binding-6.3.5.post234/.sqlx/query-8d33717587c6ee8f5fc339a80b1212a73d6c03e45856b1d55457fc8074709dd0.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT\n        COUNT(*) FILTER (WHERE started_at IS NULL AND enqueued_at < NOW() - INTERVAL '1 minute') AS \"queued!\",\n        COUNT(*) FILTER (WHERE started_at IS NULL AND enqueued_at > NOW() - INTERVAL '1 minute') AS \"recently_queued!\",\n        COUNT(*) FILTER (WHERE started_at IS NOT NULL) AS \"running!\"\n        FROM merge_jobs;\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "queued!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "recently_queued!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "running!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "8d33717587c6ee8f5fc339a80b1212a73d6c03e45856b1d55457fc8074709dd0"
+}

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.5.post224
+Version: 6.3.5.post234
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.5.post224"
+version = "6.3.5.post234"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.5.post224"
+version = "6.3.5.post234"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/src/indexer.rs

@@ -42,10 +42,9 @@ use crate::errors::NidxError;
 use crate::grpc_server::GrpcServer;
 use crate::metrics::IndexKindLabels;
 use crate::metrics::OperationStatusLabels;
-use crate::metrics::indexer::INDEXING_COUNTER;
-use crate::metrics::indexer::PER_INDEX_INDEXING_TIME;
-use crate::metrics::indexer::TOTAL_INDEXING_TIME;
+use crate::metrics::indexer::*;
 use crate::segment_store::pack_and_upload;
+use crate::utilization_tracker::UtilizationTracker;
 use crate::{Settings, metadata::*};
 
 #[cfg(feature = "telemetry")]
@@ -138,7 +137,12 @@ pub async fn run_nats(settings: Settings, shutdown: CancellationToken) -> anyhow
         None => tempfile::env::temp_dir(),
     };
 
+    let utilization = UtilizationTracker::new(|busy, duration| {
+        INDEXING_BUSY.get_or_create(&busy.into()).inc_by(duration.as_secs_f64());
+    });
+
     while !shutdown.is_cancelled() {
+        utilization.idle().await;
         let sub_msg = tokio::select! {
             sub_msg = subscription.next() => sub_msg,
             _ = shutdown.cancelled() => { return Ok(()) }
@@ -164,6 +168,7 @@ pub async fn run_nats(settings: Settings, shutdown: CancellationToken) -> anyhow
             continue;
         }
 
+        utilization.busy().await;
         let span = info_span!("indexer_message", ?seq);
         let (msg, acker) = msg.split();
 

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/src/lib.rs

@@ -29,6 +29,7 @@ pub mod searcher;
 mod segment_store;
 pub mod settings;
 pub mod tool;
+mod utilization_tracker;
 pub mod worker;
 
 #[cfg(feature = "telemetry")]

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/src/metrics.rs

@@ -127,6 +127,31 @@ impl OperationStatusLabels {
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash, EncodeLabelValue)]
+enum Utilization {
+    Free,
+    Busy,
+}
+
+#[derive(Clone, Debug, EncodeLabelSet, PartialEq, Eq, Hash)]
+pub struct UtilizationLabels {
+    status: Utilization,
+}
+
+impl From<bool> for UtilizationLabels {
+    fn from(busy: bool) -> Self {
+        if busy {
+            Self {
+                status: Utilization::Busy,
+            }
+        } else {
+            Self {
+                status: Utilization::Free,
+            }
+        }
+    }
+}
+
 pub mod common {
     metrics! {
         SPAN_DURATION: Family<Vec<(String, String)>, Histogram>{exponential_buckets(0.001, 2.0, 20)} as "span_duration_seconds" ("Duration of a tracing span"),
@@ -137,6 +162,7 @@ pub mod scheduler {
     #[derive(Clone, Debug, EncodeLabelValue, PartialEq, Eq, Hash)]
     pub enum JobState {
         Queued,
+        RecentlyQueued,
         Running,
     }
 
@@ -169,6 +195,7 @@ pub mod indexer {
         INDEXING_COUNTER: Family<OperationStatusLabels, Counter> as "indexer_message_count" ("Number of indexing operations per status"),
         TOTAL_INDEXING_TIME: Histogram[exponential_buckets(0.01, 2.0, 12)] as "total_indexing_time_seconds" ("Time it took to process an entire index message"),
         PER_INDEX_INDEXING_TIME: Family<IndexKindLabels, Histogram>{exponential_buckets(0.01, 2.0, 12)} as "indexing_time_seconds" ("Time it took to index a resource to an index"),
+        INDEXING_BUSY: Family<UtilizationLabels, Counter::<f64, AtomicU64>> as "indexer_utilization_seconds" ("Time spent in free/busy status"),
     }
 }
 
@@ -176,5 +203,6 @@ pub mod worker {
     metrics! {
         MERGE_COUNTER: Family<OperationStatusLabels, Counter> as "merge_job_count" ("Number of merge jobs per status"),
         PER_INDEX_MERGE_TIME: Family<IndexKindLabels, Histogram>{exponential_buckets(1.0, 2.0, 12)} as "merge_time_seconds" ("Time it took to run a merge job"),
+        WORKER_BUSY: Family<UtilizationLabels, Counter::<f64, AtomicU64>> as "merge_worker_utilization_seconds" ("Time spent in free/busy status"),
     }
 }

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/src/scheduler/metrics_task.rs

@@ -30,7 +30,8 @@ pub async fn update_merge_job_metric(metadb: &NidxMetadata) -> anyhow::Result<()
     let job_states = sqlx::query!(
         r#"
         SELECT
-        COUNT(*) FILTER (WHERE started_at IS NULL) AS "queued!",
+        COUNT(*) FILTER (WHERE started_at IS NULL AND enqueued_at < NOW() - INTERVAL '1 minute') AS "queued!",
+        COUNT(*) FILTER (WHERE started_at IS NULL AND enqueued_at > NOW() - INTERVAL '1 minute') AS "recently_queued!",
         COUNT(*) FILTER (WHERE started_at IS NOT NULL) AS "running!"
         FROM merge_jobs;
         "#
@@ -42,6 +43,11 @@ pub async fn update_merge_job_metric(metadb: &NidxMetadata) -> anyhow::Result<()
             state: JobState::Queued,
         })
         .set(job_states.queued);
+    metrics::scheduler::QUEUED_JOBS
+        .get_or_create(&JobFamily {
+            state: JobState::RecentlyQueued,
+        })
+        .set(job_states.queued);
     metrics::scheduler::QUEUED_JOBS
         .get_or_create(&JobFamily {
             state: JobState::Running,

nidx_binding-6.3.5.post234/src/utilization_tracker.rs

@@ -0,0 +1,62 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+use std::time::{Duration, Instant};
+
+use tokio::{sync::mpsc::*, time::sleep};
+
+pub struct UtilizationTracker {
+    channel: Sender<bool>,
+}
+
+impl UtilizationTracker {
+    pub fn new(publish: impl Fn(bool, Duration) + Send + 'static) -> Self {
+        let (tx, rx) = channel(10);
+        tokio::spawn(run(publish, rx));
+        Self { channel: tx }
+    }
+
+    pub async fn busy(&self) {
+        let _ = self.channel.send(true).await;
+    }
+
+    pub async fn idle(&self) {
+        let _ = self.channel.send(false).await;
+    }
+}
+
+async fn run(publish: impl Fn(bool, Duration), mut channel: Receiver<bool>) {
+    let mut busy = false;
+    let mut since = Instant::now();
+    loop {
+        tokio::select! {
+            rxed = channel.recv() => {
+                let Some(rxed) = rxed else { break };
+                publish(busy, since.elapsed());
+                busy = rxed;
+                since = Instant::now();
+            },
+            _ = sleep(Duration::from_secs(30)) => {
+                publish(busy, since.elapsed());
+                since = Instant::now();
+            }
+        }
+    }
+}

{nidx_binding-6.3.5.post224 → nidx_binding-6.3.5.post234}/src/worker.rs

@@ -39,11 +39,9 @@ use tracing::*;
 use crate::{
     NidxMetadata, Settings,
     metadata::{Deletion, Index, IndexKind, MergeJob, NewSegment, Segment},
-    metrics::{
-        IndexKindLabels, OperationStatusLabels,
-        worker::{MERGE_COUNTER, PER_INDEX_MERGE_TIME},
-    },
+    metrics::{IndexKindLabels, OperationStatusLabels, worker::*},
     segment_store::{download_segment, pack_and_upload},
+    utilization_tracker::UtilizationTracker,
 };
 
 pub async fn run(settings: Settings, shutdown: CancellationToken) -> anyhow::Result<()> {
@@ -55,9 +53,14 @@ pub async fn run(settings: Settings, shutdown: CancellationToken) -> anyhow::Res
         None => tempfile::env::temp_dir(),
     };
 
+    let utilization = UtilizationTracker::new(|busy, duration| {
+        WORKER_BUSY.get_or_create(&busy.into()).inc_by(duration.as_secs_f64());
+    });
+
     while !shutdown.is_cancelled() {
         let job = MergeJob::take(&meta.pool).await?;
         if let Some(job) = job {
+            utilization.busy().await;
             let span = info_span!("worker_job", ?job.id);
             info!(job.id, "Running job");
 
@@ -85,6 +88,7 @@ pub async fn run(settings: Settings, shutdown: CancellationToken) -> anyhow::Res
             // Stop keep alives
             keepalive.abort();
         } else {
+            utilization.idle().await;
             debug!("No jobs, waiting for more");
             tokio::select! {
                 _ = tokio::time::sleep(Duration::from_secs(5)) => {},

nidx_binding-6.3.5.post224/.sqlx/query-bfb8ae2e860e451b0868a7b1b50a451d998b2105f9fdfd307b26a1775d145e9f.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n        COUNT(*) FILTER (WHERE started_at IS NULL) AS \"queued!\",\n        COUNT(*) FILTER (WHERE started_at IS NOT NULL) AS \"running!\"\n        FROM merge_jobs;\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "queued!",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "running!",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null,
-      null
-    ]
-  },
-  "hash": "bfb8ae2e860e451b0868a7b1b50a451d998b2105f9fdfd307b26a1775d145e9f"
-}