ofd-validator 0.4.0__tar.gz → 0.5.1__tar.gz

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/Cargo.lock

@@ -805,7 +805,7 @@ dependencies = [
 
 [[package]]
 name = "ofd-validator-core"
-version = "0.4.0"
+version = "0.5.1"
 dependencies = [
  "image",
  "jsonschema",
@@ -818,7 +818,7 @@ dependencies = [
 
 [[package]]
 name = "ofd-validator-js"
-version = "0.4.0"
+version = "0.5.1"
 dependencies = [
  "napi",
  "napi-build",
@@ -831,7 +831,7 @@ dependencies = [
 
 [[package]]
 name = "ofd-validator-python"
-version = "0.4.0"
+version = "0.5.1"
 dependencies = [
  "ofd-validator-core",
  "pyo3",

ofd_validator-0.5.1/Cargo.toml

@@ -0,0 +1,3 @@
+[workspace]
+members = ["crates/ofd-validator-core", "crates/ofd-validator-python"]
+resolver = "2"

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ofd-validator
-Version: 0.4.0
+Version: 0.5.1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/crates/ofd-validator-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ofd-validator-core"
-version = "0.4.0"
+version = "0.5.1"
 edition = "2021"
 authors = ["Open Filament Collective"]
 license = "MIT"

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/crates/ofd-validator-core/src/orchestrator.rs

@@ -1,4 +1,6 @@
 use std::collections::HashSet;
+use std::path::Path;
+use serde::Deserialize;
 use serde_json::Value;
 use rayon::prelude::*;
 
@@ -26,6 +28,220 @@ pub struct DataSet {
     pub schema_cache: SchemaCache,
 }
 
+// --- Types for applying pending changes from the WebUI ---
+
+#[derive(Deserialize)]
+struct ChangesPayload {
+    changes: Vec<ChangeEntry>,
+}
+
+#[derive(Deserialize)]
+struct ChangeEntry {
+    entity: EntityRef,
+    operation: String,
+    data: Option<Value>,
+}
+
+#[derive(Deserialize)]
+struct EntityRef {
+    path: String,
+    #[serde(rename = "type")]
+    entity_type: String,
+}
+
+/// Internal fields added by the WebUI that should be stripped before validation.
+const STRIP_FIELDS: &[&str] = &[
+    "brandId", "brand_id", "materialType", "filamentDir", "filament_id", "slug",
+];
+
+/// Info needed to overlay a change onto the DataSet.
+struct EntityFsInfo {
+    json_path: String,
+    folder_path: String,
+    folder_name: String,
+    schema_name: String,
+    folder_json_key: String,
+}
+
+/// Map a WebUI entity path to filesystem paths and metadata.
+fn entity_path_to_fs_info(
+    entity_path: &str,
+    entity_type: &str,
+    data_dir: &Path,
+    stores_dir: &Path,
+) -> Option<EntityFsInfo> {
+    let parts: Vec<&str> = entity_path.split('/').collect();
+
+    let (json_path, folder_path, folder_name, folder_json_key) = match parts.as_slice() {
+        ["stores", slug] => (
+            stores_dir.join(slug).join("store.json"),
+            stores_dir.join(slug),
+            slug.to_string(),
+            "id".to_string(),
+        ),
+        ["brands", slug] => (
+            data_dir.join(slug).join("brand.json"),
+            data_dir.join(slug),
+            slug.to_string(),
+            "id".to_string(),
+        ),
+        ["brands", slug, "materials", mat_type] => (
+            data_dir.join(slug).join(mat_type).join("material.json"),
+            data_dir.join(slug).join(mat_type),
+            mat_type.to_string(),
+            "material".to_string(),
+        ),
+        ["brands", slug, "materials", mat_type, "filaments", name] => (
+            data_dir.join(slug).join(mat_type).join(name).join("filament.json"),
+            data_dir.join(slug).join(mat_type).join(name),
+            name.to_string(),
+            "id".to_string(),
+        ),
+        ["brands", slug, "materials", mat_type, "filaments", name, "variants", variant] => (
+            data_dir.join(slug).join(mat_type).join(name).join(variant).join("variant.json"),
+            data_dir.join(slug).join(mat_type).join(name).join(variant),
+            variant.to_string(),
+            "id".to_string(),
+        ),
+        _ => return None,
+    };
+
+    Some(EntityFsInfo {
+        json_path: json_path.to_string_lossy().to_string(),
+        folder_path: folder_path.to_string_lossy().to_string(),
+        folder_name,
+        schema_name: entity_type.to_string(),
+        folder_json_key,
+    })
+}
+
+/// Strip internal WebUI tracking fields and empty strings from entity data.
+fn clean_change_data(data: &Value) -> Value {
+    if let Value::Object(map) = data {
+        let mut clean = serde_json::Map::new();
+        for (key, value) in map {
+            if STRIP_FIELDS.contains(&key.as_str()) {
+                continue;
+            }
+            if value == &Value::String(String::new()) {
+                continue;
+            }
+            clean.insert(key.clone(), value.clone());
+        }
+        Value::Object(clean)
+    } else {
+        data.clone()
+    }
+}
+
+impl DataSet {
+    /// Apply pending changes from the WebUI onto the in-memory dataset.
+    ///
+    /// `changes_json` should be a JSON string with the format:
+    /// `{ "changes": [{ "entity": { "path": "...", "type": "..." }, "operation": "create"|"update"|"delete", "data": {...} }] }`
+    pub fn apply_changes(&mut self, changes_json: &str, data_dir: &Path, stores_dir: &Path) {
+        let payload: ChangesPayload = match serde_json::from_str(changes_json) {
+            Ok(p) => p,
+            Err(e) => {
+                eprintln!("Warning: failed to parse changes JSON: {}", e);
+                return;
+            }
+        };
+
+        for change in &payload.changes {
+            let info = match entity_path_to_fs_info(
+                &change.entity.path,
+                &change.entity.entity_type,
+                data_dir,
+                stores_dir,
+            ) {
+                Some(i) => i,
+                None => continue,
+            };
+
+            match change.operation.as_str() {
+                "create" | "update" => {
+                    let data = match &change.data {
+                        Some(d) => clean_change_data(d),
+                        None => continue,
+                    };
+
+                    // Update or add json_entries
+                    if let Some(existing) = self.json_entries.iter_mut().find(|(p, _, _)| *p == info.json_path) {
+                        existing.2 = data.clone();
+                    } else {
+                        self.json_entries.push((
+                            info.json_path.clone(),
+                            info.schema_name.clone(),
+                            data.clone(),
+                        ));
+                    }
+
+                    // Update or add folder_entries
+                    if let Some(existing) = self.folder_entries.iter_mut().find(|(p, _, _, _)| *p == info.folder_path) {
+                        existing.2 = data.clone();
+                    } else {
+                        self.folder_entries.push((
+                            info.folder_path.clone(),
+                            info.folder_name.clone(),
+                            data.clone(),
+                            info.folder_json_key.clone(),
+                        ));
+                    }
+
+                    // For stores: update valid_store_ids
+                    if info.schema_name == "store" {
+                        if let Some(id) = data.get("id").and_then(|v| v.as_str()) {
+                            self.valid_store_ids.insert(id.to_string());
+                        }
+                    }
+
+                    // For variants with sizes: update sizes_entries
+                    if info.schema_name == "variant" {
+                        if let Some(sizes) = data.get("sizes") {
+                            let sizes_path = Path::new(&info.folder_path)
+                                .join("sizes.json")
+                                .to_string_lossy()
+                                .to_string();
+                            if let Some(existing) = self.sizes_entries.iter_mut().find(|(p, _)| *p == sizes_path) {
+                                existing.1 = sizes.clone();
+                            } else {
+                                self.sizes_entries.push((sizes_path, sizes.clone()));
+                            }
+                        }
+                    }
+                }
+                "delete" => {
+                    // Remove from json_entries
+                    self.json_entries.retain(|(p, _, _)| *p != info.json_path);
+                    // Remove from folder_entries
+                    self.folder_entries.retain(|(p, _, _, _)| *p != info.folder_path);
+                    // Remove logo entries for this folder
+                    self.logo_entries.retain(|(p, _, _, _)| !p.starts_with(&info.folder_path));
+                    // Remove sizes entries under this folder
+                    self.sizes_entries.retain(|(p, _)| !p.starts_with(&info.folder_path));
+
+                    // For stores: remove from valid_store_ids
+                    if info.schema_name == "store" {
+                        // Extract the ID from the path (last component)
+                        if let Some(slug) = change.entity.path.split('/').last() {
+                            self.valid_store_ids.remove(slug);
+                        }
+                    }
+
+                    // Also remove any child entries (e.g., deleting a brand removes its materials/filaments/variants)
+                    let prefix = format!("{}/", info.folder_path);
+                    self.json_entries.retain(|(p, _, _)| !p.starts_with(&prefix));
+                    self.folder_entries.retain(|(p, _, _, _)| !p.starts_with(&prefix));
+                    self.logo_entries.retain(|(p, _, _, _)| !p.starts_with(&prefix));
+                    self.sizes_entries.retain(|(p, _)| !p.starts_with(&prefix));
+                }
+                _ => {}
+            }
+        }
+    }
+}
+
 #[cfg(feature = "filesystem")]
 impl DataSet {
     /// Build a DataSet by walking the filesystem.

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/crates/ofd-validator-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ofd-validator-python"
-version = "0.4.0"
+version = "0.5.1"
 edition = "2021"
 authors = ["Open Filament Collective"]
 license = "MIT"

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/crates/ofd-validator-python/src/lib.rs

@@ -6,7 +6,8 @@ mod util;
 mod validators;
 
 use orchestrator::{
-    validate_all, validate_folder_names, validate_json_files, validate_logo_files,
+    validate_all, validate_all_with_changes, validate_folder_names, validate_json_files,
+    validate_logo_files,
 };
 use types::{ValidationError, ValidationLevel, ValidationResult};
 use validators::{
@@ -22,6 +23,7 @@ fn ofd_validator(m: &Bound<'_, PyModule>) -> PyResult<()> {
 
     // Orchestrated batch validators (internally parallel with rayon)
     m.add_function(wrap_pyfunction!(validate_all, m)?)?;
+    m.add_function(wrap_pyfunction!(validate_all_with_changes, m)?)?;
     m.add_function(wrap_pyfunction!(validate_json_files, m)?)?;
     m.add_function(wrap_pyfunction!(validate_logo_files, m)?)?;
     m.add_function(wrap_pyfunction!(validate_folder_names, m)?)?;

ofd_validator-0.5.1/crates/ofd-validator-python/src/orchestrator.rs

@@ -0,0 +1,206 @@
+use std::path::PathBuf;
+
+use pyo3::prelude::*;
+use rayon::ThreadPoolBuilder;
+
+use ofd_validator_core as core;
+
+use crate::types::ValidationResult;
+use crate::util::log_step;
+
+/// Run the closure on a custom Rayon thread pool when `max_workers` is set,
+/// otherwise use the global pool (default Rayon behaviour).
+fn with_thread_pool<F, R>(max_workers: Option<usize>, f: F) -> R
+where
+    F: FnOnce() -> R + Send,
+    R: Send,
+{
+    match max_workers {
+        Some(n) if n > 0 => {
+            let pool = ThreadPoolBuilder::new()
+                .num_threads(n)
+                .build()
+                .expect("failed to build Rayon thread pool");
+            pool.install(f)
+        }
+        _ => f(),
+    }
+}
+
+#[pyfunction]
+#[pyo3(signature = (data_dir, stores_dir, schemas_dir=None, max_workers=None))]
+pub fn validate_all(
+    py: Python<'_>,
+    data_dir: &str,
+    stores_dir: &str,
+    schemas_dir: Option<&str>,
+    max_workers: Option<usize>,
+) -> ValidationResult {
+    let data_dir = PathBuf::from(data_dir);
+    let stores_dir = PathBuf::from(stores_dir);
+    let schemas_dir = PathBuf::from(schemas_dir.unwrap_or("schemas"));
+
+    py.allow_threads(|| {
+        with_thread_pool(max_workers, || {
+            log_step("Loading dataset", None);
+            let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
+
+            log_step("Checking required files", None);
+            log_step("Validating JSON schemas", Some(dataset.json_entries.len()));
+            log_step("Validating logos", Some(dataset.logo_entries.len()));
+            log_step("Validating folder names", Some(dataset.folder_entries.len()));
+            log_step("Validating store IDs", None);
+            log_step("Validating GTIN/EAN codes", None);
+
+            core::validate_dataset(&dataset).into()
+        })
+    })
+}
+
+#[pyfunction]
+#[pyo3(signature = (data_dir, stores_dir, changes_json, schemas_dir=None, max_workers=None))]
+pub fn validate_all_with_changes(
+    py: Python<'_>,
+    data_dir: &str,
+    stores_dir: &str,
+    changes_json: &str,
+    schemas_dir: Option<&str>,
+    max_workers: Option<usize>,
+) -> ValidationResult {
+    let data_dir = PathBuf::from(data_dir);
+    let stores_dir = PathBuf::from(stores_dir);
+    let schemas_dir = PathBuf::from(schemas_dir.unwrap_or("schemas"));
+    let changes_json = changes_json.to_string();
+
+    py.allow_threads(|| {
+        with_thread_pool(max_workers, || {
+            log_step("Loading dataset", None);
+            let mut dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
+
+            log_step("Applying pending changes", None);
+            dataset.apply_changes(&changes_json, &data_dir, &stores_dir);
+
+            log_step("Checking required files", None);
+            log_step("Validating JSON schemas", Some(dataset.json_entries.len()));
+            log_step("Validating logos", Some(dataset.logo_entries.len()));
+            log_step("Validating folder names", Some(dataset.folder_entries.len()));
+            log_step("Validating store IDs", None);
+            log_step("Validating GTIN/EAN codes", None);
+
+            core::validate_dataset(&dataset).into()
+        })
+    })
+}
+
+#[pyfunction]
+#[pyo3(signature = (data_dir, stores_dir, schemas_dir=None, max_workers=None))]
+pub fn validate_json_files(
+    py: Python<'_>,
+    data_dir: &str,
+    stores_dir: &str,
+    schemas_dir: Option<&str>,
+    max_workers: Option<usize>,
+) -> ValidationResult {
+    let data_dir = PathBuf::from(data_dir);
+    let stores_dir = PathBuf::from(stores_dir);
+    let schemas_dir = PathBuf::from(schemas_dir.unwrap_or("schemas"));
+
+    py.allow_threads(|| {
+        with_thread_pool(max_workers, || {
+            let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
+            log_step("Validating JSON schemas", Some(dataset.json_entries.len()));
+
+            use rayon::prelude::*;
+            let results: Vec<core::ValidationResult> = dataset.json_entries
+                .par_iter()
+                .map(|(path, schema_name, data)| {
+                    core::validators::validate_json(data, schema_name, &dataset.schema_cache, Some(path))
+                })
+                .collect();
+
+            let mut result = core::ValidationResult::default();
+            for r in results {
+                result.merge_from(&r);
+            }
+            result.into()
+        })
+    })
+}
+
+#[pyfunction]
+#[pyo3(signature = (data_dir, stores_dir, max_workers=None))]
+pub fn validate_logo_files(
+    py: Python<'_>,
+    data_dir: &str,
+    stores_dir: &str,
+    max_workers: Option<usize>,
+) -> ValidationResult {
+    let data_dir = PathBuf::from(data_dir);
+    let stores_dir = PathBuf::from(stores_dir);
+    let schemas_dir = PathBuf::from("schemas");
+
+    py.allow_threads(|| {
+        with_thread_pool(max_workers, || {
+            let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
+            log_step("Validating logos", Some(dataset.logo_entries.len()));
+
+            use rayon::prelude::*;
+            let results: Vec<core::ValidationResult> = dataset.logo_entries
+                .par_iter()
+                .map(|(path, filename, bytes, logo_name)| {
+                    if bytes.is_empty() {
+                        let mut r = core::ValidationResult::default();
+                        r.add(core::ValidationError::error(
+                            "Logo",
+                            "Logo file not found",
+                            Some(path.clone()),
+                        ));
+                        r
+                    } else {
+                        core::validators::validate_logo(bytes, filename, logo_name.as_deref(), Some(path))
+                    }
+                })
+                .collect();
+
+            let mut result = core::ValidationResult::default();
+            for r in results {
+                result.merge_from(&r);
+            }
+            result.into()
+        })
+    })
+}
+
+#[pyfunction]
+#[pyo3(signature = (data_dir, stores_dir, max_workers=None))]
+pub fn validate_folder_names(
+    py: Python<'_>,
+    data_dir: &str,
+    stores_dir: &str,
+    max_workers: Option<usize>,
+) -> ValidationResult {
+    let data_dir = PathBuf::from(data_dir);
+    let stores_dir = PathBuf::from(stores_dir);
+    let schemas_dir = PathBuf::from("schemas");
+
+    py.allow_threads(|| {
+        with_thread_pool(max_workers, || {
+            let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
+            log_step("Validating folder names", Some(dataset.folder_entries.len()));
+
+            use rayon::prelude::*;
+            let results: Vec<core::ValidationResult> = dataset.folder_entries
+                .par_iter()
+                .map(|(path, folder_name, json_data, json_key)| {
+                    core::validators::validate_folder_name(folder_name, json_data, json_key, Some(path))
+                })
+                .collect();
+
+            let mut result = core::ValidationResult::default();
+            for r in results {
+                result.merge_from(&r);
+            }
+            result.into()
+        })
+    })
+}

{ofd_validator-0.4.0 → ofd_validator-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "ofd-validator"
-version = "0.4.0"
+version = "0.5.1"
 description = "High-performance validation library for the Open Filament Database"
 readme = "README.md"
 requires-python = ">=3.10"

ofd_validator-0.4.0/Cargo.toml

@@ -1,3 +0,0 @@
-[workspace]
-members = ["crates/ofd-validator-python"]
-resolver = "2"

ofd_validator-0.4.0/crates/ofd-validator-python/src/orchestrator.rs

@@ -1,139 +0,0 @@
-use std::path::PathBuf;
-
-use pyo3::prelude::*;
-
-use ofd_validator_core as core;
-
-use crate::types::ValidationResult;
-use crate::util::log_step;
-
-#[pyfunction]
-#[pyo3(signature = (data_dir, stores_dir, schemas_dir=None))]
-pub fn validate_all(
-    py: Python<'_>,
-    data_dir: &str,
-    stores_dir: &str,
-    schemas_dir: Option<&str>,
-) -> ValidationResult {
-    let data_dir = PathBuf::from(data_dir);
-    let stores_dir = PathBuf::from(stores_dir);
-    let schemas_dir = PathBuf::from(schemas_dir.unwrap_or("schemas"));
-
-    py.allow_threads(|| {
-        log_step("Loading dataset", None);
-        let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
-
-        log_step("Checking required files", None);
-        log_step("Validating JSON schemas", Some(dataset.json_entries.len()));
-        log_step("Validating logos", Some(dataset.logo_entries.len()));
-        log_step("Validating folder names", Some(dataset.folder_entries.len()));
-        log_step("Validating store IDs", None);
-        log_step("Validating GTIN/EAN codes", None);
-
-        core::validate_dataset(&dataset).into()
-    })
-}
-
-#[pyfunction]
-#[pyo3(signature = (data_dir, stores_dir, schemas_dir=None))]
-pub fn validate_json_files(
-    py: Python<'_>,
-    data_dir: &str,
-    stores_dir: &str,
-    schemas_dir: Option<&str>,
-) -> ValidationResult {
-    let data_dir = PathBuf::from(data_dir);
-    let stores_dir = PathBuf::from(stores_dir);
-    let schemas_dir = PathBuf::from(schemas_dir.unwrap_or("schemas"));
-
-    py.allow_threads(|| {
-        let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
-        log_step("Validating JSON schemas", Some(dataset.json_entries.len()));
-
-        use rayon::prelude::*;
-        let results: Vec<core::ValidationResult> = dataset.json_entries
-            .par_iter()
-            .map(|(path, schema_name, data)| {
-                core::validators::validate_json(data, schema_name, &dataset.schema_cache, Some(path))
-            })
-            .collect();
-
-        let mut result = core::ValidationResult::default();
-        for r in results {
-            result.merge_from(&r);
-        }
-        result.into()
-    })
-}
-
-#[pyfunction]
-#[pyo3(signature = (data_dir, stores_dir))]
-pub fn validate_logo_files(
-    py: Python<'_>,
-    data_dir: &str,
-    stores_dir: &str,
-) -> ValidationResult {
-    let data_dir = PathBuf::from(data_dir);
-    let stores_dir = PathBuf::from(stores_dir);
-    let schemas_dir = PathBuf::from("schemas");
-
-    py.allow_threads(|| {
-        let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
-        log_step("Validating logos", Some(dataset.logo_entries.len()));
-
-        use rayon::prelude::*;
-        let results: Vec<core::ValidationResult> = dataset.logo_entries
-            .par_iter()
-            .map(|(path, filename, bytes, logo_name)| {
-                if bytes.is_empty() {
-                    let mut r = core::ValidationResult::default();
-                    r.add(core::ValidationError::error(
-                        "Logo",
-                        "Logo file not found",
-                        Some(path.clone()),
-                    ));
-                    r
-                } else {
-                    core::validators::validate_logo(bytes, filename, logo_name.as_deref(), Some(path))
-                }
-            })
-            .collect();
-
-        let mut result = core::ValidationResult::default();
-        for r in results {
-            result.merge_from(&r);
-        }
-        result.into()
-    })
-}
-
-#[pyfunction]
-#[pyo3(signature = (data_dir, stores_dir))]
-pub fn validate_folder_names(
-    py: Python<'_>,
-    data_dir: &str,
-    stores_dir: &str,
-) -> ValidationResult {
-    let data_dir = PathBuf::from(data_dir);
-    let stores_dir = PathBuf::from(stores_dir);
-    let schemas_dir = PathBuf::from("schemas");
-
-    py.allow_threads(|| {
-        let dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
-        log_step("Validating folder names", Some(dataset.folder_entries.len()));
-
-        use rayon::prelude::*;
-        let results: Vec<core::ValidationResult> = dataset.folder_entries
-            .par_iter()
-            .map(|(path, folder_name, json_data, json_key)| {
-                core::validators::validate_folder_name(folder_name, json_data, json_key, Some(path))
-            })
-            .collect();
-
-        let mut result = core::ValidationResult::default();
-        for r in results {
-            result.merge_from(&r);
-        }
-        result.into()
-    })
-}