ofd-validator 0.5.0__tar.gz → 0.5.2__tar.gz

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/Cargo.lock

@@ -805,7 +805,7 @@ dependencies = [
 
 [[package]]
 name = "ofd-validator-core"
-version = "0.5.0"
+version = "0.5.2"
 dependencies = [
  "image",
  "jsonschema",
@@ -813,12 +813,13 @@ dependencies = [
  "regex",
  "serde",
  "serde_json",
+ "url",
  "walkdir",
 ]
 
 [[package]]
 name = "ofd-validator-js"
-version = "0.5.0"
+version = "0.5.2"
 dependencies = [
  "napi",
  "napi-build",
@@ -831,7 +832,7 @@ dependencies = [
 
 [[package]]
 name = "ofd-validator-python"
-version = "0.5.0"
+version = "0.5.2"
 dependencies = [
  "ofd-validator-core",
  "pyo3",

ofd_validator-0.5.2/Cargo.toml

@@ -0,0 +1,3 @@
+[workspace]
+members = ["crates/ofd-validator-core", "crates/ofd-validator-python"]
+resolver = "2"

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ofd-validator
-Version: 0.5.0
+Version: 0.5.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ofd-validator-core"
-version = "0.5.0"
+version = "0.5.2"
 edition = "2021"
 authors = ["Open Filament Collective"]
 license = "MIT"
@@ -11,6 +11,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 jsonschema = "0.28"
 regex = "1"
+url = "2"
 image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
 rayon = "1"
 walkdir = { version = "2", optional = true }

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-core/src/orchestrator.rs

@@ -1,4 +1,6 @@
 use std::collections::HashSet;
+use std::path::Path;
+use serde::Deserialize;
 use serde_json::Value;
 use rayon::prelude::*;
 
@@ -26,6 +28,220 @@ pub struct DataSet {
     pub schema_cache: SchemaCache,
 }
 
+// --- Types for applying pending changes from the WebUI ---
+
+#[derive(Deserialize)]
+struct ChangesPayload {
+    changes: Vec<ChangeEntry>,
+}
+
+#[derive(Deserialize)]
+struct ChangeEntry {
+    entity: EntityRef,
+    operation: String,
+    data: Option<Value>,
+}
+
+#[derive(Deserialize)]
+struct EntityRef {
+    path: String,
+    #[serde(rename = "type")]
+    entity_type: String,
+}
+
+/// Internal fields added by the WebUI that should be stripped before validation.
+const STRIP_FIELDS: &[&str] = &[
+    "brandId", "brand_id", "materialType", "filamentDir", "filament_id", "slug",
+];
+
+/// Info needed to overlay a change onto the DataSet.
+struct EntityFsInfo {
+    json_path: String,
+    folder_path: String,
+    folder_name: String,
+    schema_name: String,
+    folder_json_key: String,
+}
+
+/// Map a WebUI entity path to filesystem paths and metadata.
+fn entity_path_to_fs_info(
+    entity_path: &str,
+    entity_type: &str,
+    data_dir: &Path,
+    stores_dir: &Path,
+) -> Option<EntityFsInfo> {
+    let parts: Vec<&str> = entity_path.split('/').collect();
+
+    let (json_path, folder_path, folder_name, folder_json_key) = match parts.as_slice() {
+        ["stores", slug] => (
+            stores_dir.join(slug).join("store.json"),
+            stores_dir.join(slug),
+            slug.to_string(),
+            "id".to_string(),
+        ),
+        ["brands", slug] => (
+            data_dir.join(slug).join("brand.json"),
+            data_dir.join(slug),
+            slug.to_string(),
+            "id".to_string(),
+        ),
+        ["brands", slug, "materials", mat_type] => (
+            data_dir.join(slug).join(mat_type).join("material.json"),
+            data_dir.join(slug).join(mat_type),
+            mat_type.to_string(),
+            "material".to_string(),
+        ),
+        ["brands", slug, "materials", mat_type, "filaments", name] => (
+            data_dir.join(slug).join(mat_type).join(name).join("filament.json"),
+            data_dir.join(slug).join(mat_type).join(name),
+            name.to_string(),
+            "id".to_string(),
+        ),
+        ["brands", slug, "materials", mat_type, "filaments", name, "variants", variant] => (
+            data_dir.join(slug).join(mat_type).join(name).join(variant).join("variant.json"),
+            data_dir.join(slug).join(mat_type).join(name).join(variant),
+            variant.to_string(),
+            "id".to_string(),
+        ),
+        _ => return None,
+    };
+
+    Some(EntityFsInfo {
+        json_path: json_path.to_string_lossy().to_string(),
+        folder_path: folder_path.to_string_lossy().to_string(),
+        folder_name,
+        schema_name: entity_type.to_string(),
+        folder_json_key,
+    })
+}
+
+/// Strip internal WebUI tracking fields and empty strings from entity data.
+fn clean_change_data(data: &Value) -> Value {
+    if let Value::Object(map) = data {
+        let mut clean = serde_json::Map::new();
+        for (key, value) in map {
+            if STRIP_FIELDS.contains(&key.as_str()) {
+                continue;
+            }
+            if value == &Value::String(String::new()) {
+                continue;
+            }
+            clean.insert(key.clone(), value.clone());
+        }
+        Value::Object(clean)
+    } else {
+        data.clone()
+    }
+}
+
+impl DataSet {
+    /// Apply pending changes from the WebUI onto the in-memory dataset.
+    ///
+    /// `changes_json` should be a JSON string with the format:
+    /// `{ "changes": [{ "entity": { "path": "...", "type": "..." }, "operation": "create"|"update"|"delete", "data": {...} }] }`
+    pub fn apply_changes(&mut self, changes_json: &str, data_dir: &Path, stores_dir: &Path) {
+        let payload: ChangesPayload = match serde_json::from_str(changes_json) {
+            Ok(p) => p,
+            Err(e) => {
+                eprintln!("Warning: failed to parse changes JSON: {}", e);
+                return;
+            }
+        };
+
+        for change in &payload.changes {
+            let info = match entity_path_to_fs_info(
+                &change.entity.path,
+                &change.entity.entity_type,
+                data_dir,
+                stores_dir,
+            ) {
+                Some(i) => i,
+                None => continue,
+            };
+
+            match change.operation.as_str() {
+                "create" | "update" => {
+                    let data = match &change.data {
+                        Some(d) => clean_change_data(d),
+                        None => continue,
+                    };
+
+                    // Update or add json_entries
+                    if let Some(existing) = self.json_entries.iter_mut().find(|(p, _, _)| *p == info.json_path) {
+                        existing.2 = data.clone();
+                    } else {
+                        self.json_entries.push((
+                            info.json_path.clone(),
+                            info.schema_name.clone(),
+                            data.clone(),
+                        ));
+                    }
+
+                    // Update or add folder_entries
+                    if let Some(existing) = self.folder_entries.iter_mut().find(|(p, _, _, _)| *p == info.folder_path) {
+                        existing.2 = data.clone();
+                    } else {
+                        self.folder_entries.push((
+                            info.folder_path.clone(),
+                            info.folder_name.clone(),
+                            data.clone(),
+                            info.folder_json_key.clone(),
+                        ));
+                    }
+
+                    // For stores: update valid_store_ids
+                    if info.schema_name == "store" {
+                        if let Some(id) = data.get("id").and_then(|v| v.as_str()) {
+                            self.valid_store_ids.insert(id.to_string());
+                        }
+                    }
+
+                    // For variants with sizes: update sizes_entries
+                    if info.schema_name == "variant" {
+                        if let Some(sizes) = data.get("sizes") {
+                            let sizes_path = Path::new(&info.folder_path)
+                                .join("sizes.json")
+                                .to_string_lossy()
+                                .to_string();
+                            if let Some(existing) = self.sizes_entries.iter_mut().find(|(p, _)| *p == sizes_path) {
+                                existing.1 = sizes.clone();
+                            } else {
+                                self.sizes_entries.push((sizes_path, sizes.clone()));
+                            }
+                        }
+                    }
+                }
+                "delete" => {
+                    // Remove from json_entries
+                    self.json_entries.retain(|(p, _, _)| *p != info.json_path);
+                    // Remove from folder_entries
+                    self.folder_entries.retain(|(p, _, _, _)| *p != info.folder_path);
+                    // Remove logo entries for this folder
+                    self.logo_entries.retain(|(p, _, _, _)| !p.starts_with(&info.folder_path));
+                    // Remove sizes entries under this folder
+                    self.sizes_entries.retain(|(p, _)| !p.starts_with(&info.folder_path));
+
+                    // For stores: remove from valid_store_ids
+                    if info.schema_name == "store" {
+                        // Extract the ID from the path (last component)
+                        if let Some(slug) = change.entity.path.split('/').last() {
+                            self.valid_store_ids.remove(slug);
+                        }
+                    }
+
+                    // Also remove any child entries (e.g., deleting a brand removes its materials/filaments/variants)
+                    let prefix = format!("{}/", info.folder_path);
+                    self.json_entries.retain(|(p, _, _)| !p.starts_with(&prefix));
+                    self.folder_entries.retain(|(p, _, _, _)| !p.starts_with(&prefix));
+                    self.logo_entries.retain(|(p, _, _, _)| !p.starts_with(&prefix));
+                    self.sizes_entries.retain(|(p, _)| !p.starts_with(&prefix));
+                }
+                _ => {}
+            }
+        }
+    }
+}
+
 #[cfg(feature = "filesystem")]
 impl DataSet {
     /// Build a DataSet by walking the filesystem.
@@ -356,5 +572,40 @@ pub fn validate_dataset(dataset: &DataSet) -> ValidationResult {
     // 6. GTIN/EAN validation
     result.merge_from(&validators::validate_gtin_ean(&sizes_refs));
 
+    // 7. URL tracking parameters in purchase links (warnings)
+    result.merge_from(&validators::validate_url_tracking(&sizes_refs));
+
+    // 8. Variant colour-name / folder-id material-type redundancy (warnings)
+    let material_types: HashSet<String> = dataset
+        .schema_cache
+        .get("material_types")
+        .and_then(|s| s.get("enum"))
+        .and_then(|e| e.as_array())
+        .map(|arr| {
+            arr.iter()
+                .filter_map(|v| v.as_str().map(|s| s.to_ascii_uppercase()))
+                .collect()
+        })
+        .unwrap_or_else(|| {
+            validators::variant_material::MATERIAL_TYPES_FALLBACK
+                .iter()
+                .map(|s| s.to_string())
+                .collect()
+        });
+    let variant_refs: Vec<(&str, &Value)> = dataset
+        .json_entries
+        .iter()
+        .filter(|(_, schema_name, _)| schema_name == "variant")
+        .map(|(p, _, v)| (p.as_str(), v))
+        .collect();
+    result.merge_from(&validators::validate_variant_material(&variant_refs, &material_types));
+
+    // 9. Purchase-link host consistency vs the store's storefront (warnings)
+    let store_hosts = validators::store_host::build_store_hosts(&dataset.json_entries);
+    result.merge_from(&validators::validate_store_host(&store_hosts, &sizes_refs));
+
+    // 10. Generic purchase link reused across many variants of a filament (warnings)
+    result.merge_from(&validators::validate_duplicate_links(&sizes_refs));
+
     result
 }

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-core/src/util.rs

@@ -16,6 +16,13 @@ pub fn cleanse_folder_name(name: &str) -> String {
     name.replace('/', " ").trim().to_string()
 }
 
+/// Extract the host (e.g. `shop.polymaker.com`) from an absolute URL, if parseable.
+pub fn url_host(url: &str) -> Option<String> {
+    url::Url::parse(url)
+        .ok()
+        .and_then(|u| u.host_str().map(|h| h.to_string()))
+}
+
 #[cfg(feature = "filesystem")]
 pub fn load_json(path: &std::path::Path) -> Option<Value> {
     let content = std::fs::read_to_string(path).ok()?;

ofd_validator-0.5.2/crates/ofd-validator-core/src/validators/duplicate_link.rs

@@ -0,0 +1,138 @@
+use serde_json::Value;
+use std::collections::{BTreeSet, HashMap};
+
+use crate::types::{ValidationError, ValidationResult};
+
+/// Minimum number of distinct variants sharing one URL before we warn.
+const MIN_VARIANTS: usize = 3;
+
+/// From a sizes.json path label `.../<brand>/<MAT>/<filament>/<variant>/sizes.json`,
+/// return `(filament_key, variant_folder)` = (`.../<brand>/<MAT>/<filament>`, `<variant>`).
+fn filament_key(path: &str) -> Option<(String, String)> {
+    let norm = path.replace('\\', "/");
+    let parts: Vec<&str> = norm.split('/').collect();
+    if parts.len() < 3 {
+        return None;
+    }
+    let variant = parts[parts.len() - 2].to_string();
+    let key = parts[..parts.len() - 2].join("/");
+    Some((key, variant))
+}
+
+/// Warn when the same purchase URL is reused across many variants of one filament (a generic
+/// link copied to every colour). Non-blocking; one warning per (filament, url).
+pub fn validate_duplicate_links(sizes_entries: &[(&str, &Value)]) -> ValidationResult {
+    let mut result = ValidationResult::default();
+
+    // filament_key -> url -> set of variant folder names
+    let mut map: HashMap<String, HashMap<String, BTreeSet<String>>> = HashMap::new();
+
+    for (path_str, sizes_data) in sizes_entries {
+        let (fkey, variant) = match filament_key(path_str) {
+            Some(v) => v,
+            None => continue,
+        };
+        let sizes_arr = match sizes_data.as_array() {
+            Some(a) => a,
+            None => continue,
+        };
+        for size in sizes_arr {
+            let purchase_links = match size.get("purchase_links").and_then(|v| v.as_array()) {
+                Some(l) => l,
+                None => continue,
+            };
+            for link in purchase_links {
+                if let Some(url) = link.get("url").and_then(|v| v.as_str()) {
+                    map.entry(fkey.clone())
+                        .or_default()
+                        .entry(url.to_string())
+                        .or_default()
+                        .insert(variant.clone());
+                }
+            }
+        }
+    }
+
+    // Deterministic output ordering.
+    let mut filaments: Vec<&String> = map.keys().collect();
+    filaments.sort();
+    for fkey in filaments {
+        let urls = &map[fkey];
+        let mut url_keys: Vec<&String> = urls.keys().collect();
+        url_keys.sort();
+        for url in url_keys {
+            let variants = &urls[url];
+            if variants.len() >= MIN_VARIANTS {
+                result.add(ValidationError::warning(
+                    "DuplicateLink",
+                    format!(
+                        "The same purchase url is reused across {} variants of this filament: {}",
+                        variants.len(),
+                        url
+                    ),
+                    Some(format!("{}/*/sizes.json", fkey)),
+                ));
+            }
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    fn sized(url: &str) -> Value {
+        json!([{
+            "filament_weight": 1000, "diameter": 1.75,
+            "purchase_links": [{ "store_id": "s", "url": url }]
+        }])
+    }
+
+    #[test]
+    fn flags_generic_link_reused_across_variants() {
+        let base = "data/b/PLA/snapspeed_pla";
+        let a = sized("https://a.com/generic");
+        let b = sized("https://a.com/generic");
+        let c = sized("https://a.com/generic");
+        let entries = vec![
+            (format!("{base}/red/sizes.json"), &a),
+            (format!("{base}/blue/sizes.json"), &b),
+            (format!("{base}/green/sizes.json"), &c),
+        ];
+        let refs: Vec<(&str, &Value)> = entries.iter().map(|(p, v)| (p.as_str(), *v)).collect();
+        let res = validate_duplicate_links(&refs);
+        assert_eq!(res.warning_count(), 1);
+        assert!(res.is_valid());
+    }
+
+    #[test]
+    fn distinct_links_pass() {
+        let base = "data/b/PLA/f";
+        let a = sized("https://a.com/1");
+        let b = sized("https://a.com/2");
+        let c = sized("https://a.com/3");
+        let entries = vec![
+            (format!("{base}/red/sizes.json"), &a),
+            (format!("{base}/blue/sizes.json"), &b),
+            (format!("{base}/green/sizes.json"), &c),
+        ];
+        let refs: Vec<(&str, &Value)> = entries.iter().map(|(p, v)| (p.as_str(), *v)).collect();
+        assert_eq!(validate_duplicate_links(&refs).warning_count(), 0);
+    }
+
+    #[test]
+    fn same_url_across_two_variants_below_threshold() {
+        let base = "data/b/PLA/f";
+        let a = sized("https://a.com/x");
+        let b = sized("https://a.com/x");
+        let entries = vec![
+            (format!("{base}/red/sizes.json"), &a),
+            (format!("{base}/blue/sizes.json"), &b),
+        ];
+        let refs: Vec<(&str, &Value)> = entries.iter().map(|(p, v)| (p.as_str(), *v)).collect();
+        assert_eq!(validate_duplicate_links(&refs).warning_count(), 0);
+    }
+}

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-core/src/validators/mod.rs

@@ -1,13 +1,21 @@
+mod duplicate_link;
 mod folder_name;
 mod gtin;
 mod json_validator;
 mod logo_validator;
 pub mod missing_files;
+pub mod store_host;
 mod store_id;
+mod url_tracker;
+pub mod variant_material;
 
+pub use duplicate_link::validate_duplicate_links;
 pub use folder_name::validate_folder_name;
 pub use gtin::validate_gtin_ean;
 pub use json_validator::validate_json;
 pub use logo_validator::validate_logo;
 pub use missing_files::validate_required_files;
+pub use store_host::validate_store_host;
 pub use store_id::validate_store_ids;
+pub use url_tracker::validate_url_tracking;
+pub use variant_material::validate_variant_material;

ofd_validator-0.5.2/crates/ofd-validator-core/src/validators/store_host.rs

@@ -0,0 +1,138 @@
+use serde_json::Value;
+use std::collections::HashMap;
+
+use crate::types::{ValidationError, ValidationResult};
+use crate::util::url_host;
+
+/// Build a map of `store_id -> canonical host` from store.json entries.
+/// `json_entries` are (path_label, schema_name, Value).
+pub fn build_store_hosts(json_entries: &[(String, String, Value)]) -> HashMap<String, String> {
+    let mut map = HashMap::new();
+    for (_, schema_name, data) in json_entries {
+        if schema_name != "store" {
+            continue;
+        }
+        let id = data.get("id").and_then(|v| v.as_str());
+        let host = data
+            .get("storefront_url")
+            .and_then(|v| v.as_str())
+            .and_then(url_host);
+        if let (Some(id), Some(host)) = (id, host) {
+            map.insert(id.to_string(), host);
+        }
+    }
+    map
+}
+
+/// The registrable-ish base domain: the last two dot-separated labels.
+/// `shop.polymaker.com` and `us.polymaker.com` both -> `polymaker.com`.
+fn base_domain(host: &str) -> String {
+    let labels: Vec<&str> = host.split('.').collect();
+    let n = labels.len();
+    if n >= 2 {
+        labels[n - 2..].join(".")
+    } else {
+        host.to_string()
+    }
+}
+
+/// Warn when a purchase-link URL's host differs from the store's storefront host but shares
+/// the same base domain (an inconsistent subdomain, e.g. `shop.` vs `us.`). Cross-domain
+/// links are left alone (they may be intentional resellers). Non-blocking warnings.
+pub fn validate_store_host(
+    store_hosts: &HashMap<String, String>,
+    sizes_entries: &[(&str, &Value)],
+) -> ValidationResult {
+    let mut result = ValidationResult::default();
+
+    for (path_str, sizes_data) in sizes_entries {
+        let sizes_arr = match sizes_data.as_array() {
+            Some(a) => a,
+            None => continue,
+        };
+        for (size_idx, size) in sizes_arr.iter().enumerate() {
+            let purchase_links = match size.get("purchase_links").and_then(|v| v.as_array()) {
+                Some(l) => l,
+                None => continue,
+            };
+            for (link_idx, link) in purchase_links.iter().enumerate() {
+                let store_id = match link.get("store_id").and_then(|v| v.as_str()) {
+                    Some(s) => s,
+                    None => continue,
+                };
+                let canonical = match store_hosts.get(store_id) {
+                    Some(h) => h,
+                    None => continue,
+                };
+                let link_host = match link.get("url").and_then(|v| v.as_str()).and_then(url_host) {
+                    Some(h) => h,
+                    None => continue,
+                };
+                if &link_host != canonical && base_domain(&link_host) == base_domain(canonical) {
+                    result.add(ValidationError::warning(
+                        "StoreHost",
+                        format!(
+                            "url host '{}' at $[{}].purchase_links[{}] differs from store '{}' storefront host '{}'",
+                            link_host, size_idx, link_idx, store_id, canonical
+                        ),
+                        Some(path_str.to_string()),
+                    ));
+                }
+            }
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    fn hosts() -> HashMap<String, String> {
+        let mut m = HashMap::new();
+        m.insert("polymaker".to_string(), "us.polymaker.com".to_string());
+        m
+    }
+
+    #[test]
+    fn flags_subdomain_drift() {
+        let sizes = json!([{
+            "filament_weight": 1000, "diameter": 1.75,
+            "purchase_links": [{ "store_id": "polymaker", "url": "https://shop.polymaker.com/products/x" }]
+        }]);
+        let entries = vec![("data/b/PLA/f/red/sizes.json", &sizes)];
+        let res = validate_store_host(&hosts(), &entries);
+        assert_eq!(res.warning_count(), 1);
+        assert!(res.is_valid());
+    }
+
+    #[test]
+    fn same_host_passes() {
+        let sizes = json!([{
+            "filament_weight": 1000, "diameter": 1.75,
+            "purchase_links": [{ "store_id": "polymaker", "url": "https://us.polymaker.com/products/x" }]
+        }]);
+        let entries = vec![("p", &sizes)];
+        assert_eq!(validate_store_host(&hosts(), &entries).warning_count(), 0);
+    }
+
+    #[test]
+    fn cross_domain_is_left_alone() {
+        let sizes = json!([{
+            "filament_weight": 1000, "diameter": 1.75,
+            "purchase_links": [{ "store_id": "polymaker", "url": "https://www.amazon.com/dp/x" }]
+        }]);
+        let entries = vec![("p", &sizes)];
+        assert_eq!(validate_store_host(&hosts(), &entries).warning_count(), 0);
+    }
+
+    #[test]
+    fn builds_hosts_from_store_entries() {
+        let store = json!({ "id": "polymaker", "storefront_url": "https://us.polymaker.com/" });
+        let entries = vec![("stores/polymaker/store.json".to_string(), "store".to_string(), store)];
+        let map = build_store_hosts(&entries);
+        assert_eq!(map.get("polymaker").map(|s| s.as_str()), Some("us.polymaker.com"));
+    }
+}

ofd_validator-0.5.2/crates/ofd-validator-core/src/validators/url_tracker.rs

@@ -0,0 +1,229 @@
+use serde_json::Value;
+use std::collections::HashSet;
+use std::sync::LazyLock;
+
+use crate::types::{ValidationError, ValidationResult};
+
+/// Known tracking query-parameter keys (compared case-insensitively). Prefix families
+/// (`utm_`, `mc_`, `pk_`) are handled separately in `is_tracking_key`.
+///
+/// Note: Shopify product selectors `variant` and `id` are intentionally NOT included —
+/// they can point at a specific colour/product, so we treat them as meaningful.
+static TRACKING_PARAMS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
+    [
+        "tag", "linkcode", "psc", "th", "ref", "ref_", "fbclid", "gclid", "gclsrc", "dclid",
+        "msclkid", "yclid", "twclid", "ttclid", "igshid", "si", "mc_cid", "mc_eid", "_pos", "_psq",
+        "_ss", "_v", "_sid", "spm", "scm", "aff", "affid", "srsltid", "gad_source", "epik",
+        "pk_campaign", "pk_kwd",
+        // Amazon / marketplace search-result context (identity lives in the /dp/<ASIN> path)
+        "dib", "dib_tag", "keywords", "qid", "sr", "sprefix", "crid", "ascsubtag", "smid",
+        "content-id", "qsid", "rnid", "refinements", "_encoding", "pldnsite",
+    ]
+    .into_iter()
+    .collect()
+});
+
+fn is_tracking_key(key: &str) -> bool {
+    let k = key.trim().to_ascii_lowercase();
+    k.starts_with("utm_")
+        || k.starts_with("mc_")
+        || k.starts_with("pk_")
+        || k.starts_with("pd_rd_")
+        || k.starts_with("pf_rd_")
+        || TRACKING_PARAMS.contains(k.as_str())
+}
+
+/// Hosts where the product identity is entirely in the path, so every query param is
+/// search/tracking context and can be dropped wholesale (e.g. Amazon `/dp/<ASIN>`).
+fn has_disposable_query(base: &str) -> bool {
+    let host = match crate::util::url_host(base) {
+        Some(h) => h.to_ascii_lowercase(),
+        None => return false,
+    };
+    if host == "a.co" || host == "amzn.to" || host == "amzn.eu" {
+        return true;
+    }
+    ["amazon", "ebay"]
+        .iter()
+        .any(|b| host.starts_with(&format!("{b}.")) || host.contains(&format!(".{b}.")))
+}
+
+/// Whether a URL fragment looks like tracking data (e.g. `#utm_source=x`) rather than a
+/// content anchor (e.g. `#productinfo`, which we keep).
+fn is_tracking_fragment(fragment: &str) -> bool {
+    if !fragment.contains('=') {
+        return false;
+    }
+    let first_key = fragment.split(['&', '=']).next().unwrap_or("");
+    is_tracking_key(first_key)
+}
+
+/// Remove known tracking parameters (plus an empty `?` and tracking-only fragments) from a
+/// URL, preserving the original ordering/encoding of everything kept. Idempotent.
+pub fn strip_tracking(url: &str) -> String {
+    let (before_frag, fragment) = match url.split_once('#') {
+        Some((b, f)) => (b, Some(f)),
+        None => (url, None),
+    };
+    let (base, query) = match before_frag.split_once('?') {
+        Some((b, q)) => (b, Some(q)),
+        None => (before_frag, None),
+    };
+
+    let mut out = String::from(base);
+
+    if let Some(q) = query {
+        if !has_disposable_query(base) {
+            let kept: Vec<&str> = q
+                .split('&')
+                .filter(|pair| {
+                    if pair.is_empty() {
+                        return false;
+                    }
+                    let key = pair.split('=').next().unwrap_or("");
+                    !is_tracking_key(key)
+                })
+                .collect();
+            if !kept.is_empty() {
+                out.push('?');
+                out.push_str(&kept.join("&"));
+            }
+        }
+    }
+
+    if let Some(frag) = fragment {
+        if !frag.is_empty() && !is_tracking_fragment(frag) {
+            out.push('#');
+            out.push_str(frag);
+        }
+    }
+
+    out
+}
+
+/// True if stripping tracking data would change the URL. Mirrors the webui `hasTracking`
+/// helper; retained as a public API for consumers even though the rule uses `strip_tracking`
+/// directly.
+#[allow(dead_code)]
+pub fn has_tracking(url: &str) -> bool {
+    strip_tracking(url) != url
+}
+
+/// Warn about tracking parameters in `purchase_links[].url` inside sizes.json entries.
+/// Each entry is (path_label, parsed sizes.json Value). Emits non-blocking warnings that
+/// suggest the cleaned URL in the message.
+pub fn validate_url_tracking(sizes_entries: &[(&str, &Value)]) -> ValidationResult {
+    let mut result = ValidationResult::default();
+
+    for (path_str, sizes_data) in sizes_entries {
+        let sizes_arr = match sizes_data.as_array() {
+            Some(a) => a,
+            None => continue,
+        };
+
+        for (size_idx, size) in sizes_arr.iter().enumerate() {
+            let purchase_links = match size.get("purchase_links").and_then(|v| v.as_array()) {
+                Some(links) => links,
+                None => continue,
+            };
+
+            for (link_idx, link) in purchase_links.iter().enumerate() {
+                if let Some(url) = link.get("url").and_then(|v| v.as_str()) {
+                    let cleaned = strip_tracking(url);
+                    if cleaned != url {
+                        result.add(ValidationError::warning(
+                            "URLTracking",
+                            format!(
+                                "Tracking parameters in url at $[{}].purchase_links[{}]; suggested: {}",
+                                size_idx, link_idx, cleaned
+                            ),
+                            Some(path_str.to_string()),
+                        ));
+                    }
+                }
+            }
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn strips_amazon_affiliate() {
+        assert_eq!(
+            strip_tracking("https://www.amazon.eg/dp/B0GVSM88NP/?tag=3dfil-20&linkCode=ogi&psc=1&th=1"),
+            "https://www.amazon.eg/dp/B0GVSM88NP/"
+        );
+    }
+
+    #[test]
+    fn strips_amazon_search_query_wholesale() {
+        let dirty = "https://www.amazon.com/AmeLeis-Printer-Filament-Bundle-1-75mm/dp/B0GYNWLN2F?dib=eyJ2IjoiMSJ9.abc&dib_tag=se&keywords=AmeLeis&qid=1784042998&sr=8-1";
+        assert_eq!(
+            strip_tracking(dirty),
+            "https://www.amazon.com/AmeLeis-Printer-Filament-Bundle-1-75mm/dp/B0GYNWLN2F"
+        );
+        assert!(has_tracking(dirty));
+    }
+
+    #[test]
+    fn keeps_shopify_variant_selector() {
+        let u = "https://shop.polymaker.com/products/panchroma-galaxy?variant=44933539987513";
+        assert_eq!(strip_tracking(u), u);
+    }
+
+    #[test]
+    fn keeps_variant_and_content_fragment() {
+        let u = "https://shop.polymaker.com/products/panchroma-pla?variant=45411122413625#productinfo";
+        assert_eq!(strip_tracking(u), u);
+    }
+
+    #[test]
+    fn removes_trailing_empty_query() {
+        assert_eq!(
+            strip_tracking("https://us.polymaker.com/products/panchroma-glow?"),
+            "https://us.polymaker.com/products/panchroma-glow"
+        );
+    }
+
+    #[test]
+    fn removes_trackers_but_keeps_meaningful() {
+        assert_eq!(
+            strip_tracking("https://x.com/p?utm_source=a&variant=5&gclid=z"),
+            "https://x.com/p?variant=5"
+        );
+    }
+
+    #[test]
+    fn strips_tracking_fragment_keeps_anchor() {
+        assert_eq!(strip_tracking("https://x.com/p#utm_source=a"), "https://x.com/p");
+        assert_eq!(strip_tracking("https://x.com/p#section-2"), "https://x.com/p#section-2");
+    }
+
+    #[test]
+    fn is_idempotent() {
+        let dirty = "https://www.amazon.eg/dp/B0GVSM88NP/?tag=3dfil-20&th=1";
+        let once = strip_tracking(dirty);
+        assert_eq!(strip_tracking(&once), once);
+        assert!(has_tracking(dirty));
+        assert!(!has_tracking(&once));
+    }
+
+    #[test]
+    fn rule_emits_warning_not_error() {
+        let sizes = json!([{
+            "filament_weight": 1000, "diameter": 1.75,
+            "purchase_links": [{ "store_id": "amazon", "url": "https://a.com/x?tag=aff-20" }]
+        }]);
+        let entries = vec![("data/b/PLA/f/red/sizes.json", &sizes)];
+        let res = validate_url_tracking(&entries);
+        assert_eq!(res.warning_count(), 1);
+        assert_eq!(res.error_count(), 0);
+        assert!(res.is_valid()); // warnings never invalidate
+    }
+}

ofd_validator-0.5.2/crates/ofd-validator-core/src/validators/variant_material.rs

@@ -0,0 +1,120 @@
+use serde_json::Value;
+use std::collections::HashSet;
+
+use crate::types::{ValidationError, ValidationResult};
+
+/// Canonical material-type tokens (mirrors `schemas/material_types_schema.json`).
+/// Used as a fallback; `validate_dataset` prefers the live schema enum.
+pub const MATERIAL_TYPES_FALLBACK: &[&str] = &[
+    "PLA", "PETG", "TPU", "ABS", "ASA", "PC", "PCTG", "PP", "PA6", "PA11", "PA12", "PA66", "CPE",
+    "TPE", "HIPS", "PHA", "PET", "PEI", "PBT", "PVB", "PVA", "PEKK", "PEEK", "BVOH", "TPC", "PPS",
+    "PPSU", "PVC", "PEBA", "PVDF", "PPA", "PCL", "PES", "PMMA", "POM", "PPE", "PS", "PSU", "TPI",
+    "SBS", "OBC", "EVA",
+];
+
+/// Split a display name / id into upper-cased alphanumeric tokens.
+fn tokens(s: &str) -> Vec<String> {
+    s.split(|c: char| !c.is_ascii_alphanumeric())
+        .filter(|t| !t.is_empty())
+        .map(|t| t.to_ascii_uppercase())
+        .collect()
+}
+
+/// Return the first material token embedded in `name`, if any (whole-token match).
+fn find_material_token(name: &str, material_types: &HashSet<String>) -> Option<String> {
+    tokens(name).into_iter().find(|t| material_types.contains(t))
+}
+
+/// Extract the variant folder name from a `.../<variant>/variant.json` path label.
+fn variant_folder(path: &str) -> Option<String> {
+    let norm = path.replace('\\', "/");
+    let mut parts = norm.rsplit('/');
+    let _file = parts.next()?; // variant.json
+    parts.next().map(|s| s.to_string())
+}
+
+/// Warn when a variant's colour name (or its folder id) embeds a material-type token,
+/// which is redundant with the parent MATERIAL directory. Non-blocking warnings;
+/// at most one per variant. `variant_entries` are (path_label, parsed variant.json Value).
+pub fn validate_variant_material(
+    variant_entries: &[(&str, &Value)],
+    material_types: &HashSet<String>,
+) -> ValidationResult {
+    let mut result = ValidationResult::default();
+
+    for (path_str, data) in variant_entries {
+        if let Some(name) = data.get("name").and_then(|v| v.as_str()) {
+            if let Some(tok) = find_material_token(name, material_types) {
+                result.add(ValidationError::warning(
+                    "VariantMaterial",
+                    format!(
+                        "Colour name '{}' contains material type '{}', which is redundant with the material folder",
+                        name, tok
+                    ),
+                    Some(path_str.to_string()),
+                ));
+                continue;
+            }
+        }
+
+        if let Some(folder) = variant_folder(path_str) {
+            if let Some(tok) = find_material_token(&folder, material_types) {
+                result.add(ValidationError::warning(
+                    "VariantMaterial",
+                    format!(
+                        "Variant folder id '{}' contains material type '{}', which is redundant with the material folder",
+                        folder, tok
+                    ),
+                    Some(path_str.to_string()),
+                ));
+            }
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    fn material_set() -> HashSet<String> {
+        MATERIAL_TYPES_FALLBACK.iter().map(|s| s.to_string()).collect()
+    }
+
+    #[test]
+    fn flags_material_in_colour_name() {
+        let v = json!({ "name": "Galaxy PETG", "color_hex": "#112233" });
+        let entries = vec![("data/b/PETG/f/galaxy_petg/variant.json", &v)];
+        let res = validate_variant_material(&entries, &material_set());
+        assert_eq!(res.warning_count(), 1);
+        assert_eq!(res.error_count(), 0);
+        assert!(res.is_valid());
+    }
+
+    #[test]
+    fn flags_material_in_folder_id_when_name_clean() {
+        let v = json!({ "name": "Sakura Pink", "color_hex": "#ffaabb" });
+        let entries = vec![("data/sunlu/PETG/petg/sakura_pink_petg/variant.json", &v)];
+        let res = validate_variant_material(&entries, &material_set());
+        assert_eq!(res.warning_count(), 1);
+    }
+
+    #[test]
+    fn clean_name_and_folder_pass() {
+        let v = json!({ "name": "Sakura Pink", "color_hex": "#ffaabb" });
+        let entries = vec![("data/sunlu/PETG/petg/sakura_pink/variant.json", &v)];
+        let res = validate_variant_material(&entries, &material_set());
+        assert_eq!(res.warning_count(), 0);
+    }
+
+    #[test]
+    fn whole_token_match_avoids_substrings() {
+        // "PETAL" must not match "PET"; "Space" must not match "PA"/"AS".
+        let v = json!({ "name": "Petal Space Grey", "color_hex": "#777777" });
+        let entries = vec![("data/b/PLA/f/petal/variant.json", &v)];
+        let res = validate_variant_material(&entries, &material_set());
+        assert_eq!(res.warning_count(), 0);
+    }
+}

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ofd-validator-python"
-version = "0.5.0"
+version = "0.5.2"
 edition = "2021"
 authors = ["Open Filament Collective"]
 license = "MIT"

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-python/src/lib.rs

@@ -6,7 +6,8 @@ mod util;
 mod validators;
 
 use orchestrator::{
-    validate_all, validate_folder_names, validate_json_files, validate_logo_files,
+    validate_all, validate_all_with_changes, validate_folder_names, validate_json_files,
+    validate_logo_files,
 };
 use types::{ValidationError, ValidationLevel, ValidationResult};
 use validators::{
@@ -22,6 +23,7 @@ fn ofd_validator(m: &Bound<'_, PyModule>) -> PyResult<()> {
 
     // Orchestrated batch validators (internally parallel with rayon)
     m.add_function(wrap_pyfunction!(validate_all, m)?)?;
+    m.add_function(wrap_pyfunction!(validate_all_with_changes, m)?)?;
     m.add_function(wrap_pyfunction!(validate_json_files, m)?)?;
     m.add_function(wrap_pyfunction!(validate_logo_files, m)?)?;
     m.add_function(wrap_pyfunction!(validate_folder_names, m)?)?;

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/crates/ofd-validator-python/src/orchestrator.rs

@@ -57,6 +57,41 @@ pub fn validate_all(
     })
 }
 
+#[pyfunction]
+#[pyo3(signature = (data_dir, stores_dir, changes_json, schemas_dir=None, max_workers=None))]
+pub fn validate_all_with_changes(
+    py: Python<'_>,
+    data_dir: &str,
+    stores_dir: &str,
+    changes_json: &str,
+    schemas_dir: Option<&str>,
+    max_workers: Option<usize>,
+) -> ValidationResult {
+    let data_dir = PathBuf::from(data_dir);
+    let stores_dir = PathBuf::from(stores_dir);
+    let schemas_dir = PathBuf::from(schemas_dir.unwrap_or("schemas"));
+    let changes_json = changes_json.to_string();
+
+    py.allow_threads(|| {
+        with_thread_pool(max_workers, || {
+            log_step("Loading dataset", None);
+            let mut dataset = core::DataSet::from_directories(&data_dir, &stores_dir, &schemas_dir);
+
+            log_step("Applying pending changes", None);
+            dataset.apply_changes(&changes_json, &data_dir, &stores_dir);
+
+            log_step("Checking required files", None);
+            log_step("Validating JSON schemas", Some(dataset.json_entries.len()));
+            log_step("Validating logos", Some(dataset.logo_entries.len()));
+            log_step("Validating folder names", Some(dataset.folder_entries.len()));
+            log_step("Validating store IDs", None);
+            log_step("Validating GTIN/EAN codes", None);
+
+            core::validate_dataset(&dataset).into()
+        })
+    })
+}
+
 #[pyfunction]
 #[pyo3(signature = (data_dir, stores_dir, schemas_dir=None, max_workers=None))]
 pub fn validate_json_files(

{ofd_validator-0.5.0 → ofd_validator-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "ofd-validator"
-version = "0.5.0"
+version = "0.5.2"
 description = "High-performance validation library for the Open Filament Database"
 readme = "README.md"
 requires-python = ">=3.10"

ofd_validator-0.5.0/Cargo.toml

@@ -1,3 +0,0 @@
-[workspace]
-members = ["crates/ofd-validator-python"]
-resolver = "2"