vesper-wizard 2.0.8 → 2.0.9

package/build/metadata/scraper.js

@@ -150,18 +150,61 @@ export class HuggingFaceScraper {
                             initialDelay: 2000, // Start with 2 seconds for HF API
                             maxDelay: 30000 // Max 30 seconds
                         });
-                        const splits = fullInfo.splits?.map((s) => ({
+                        const cardData = fullInfo.cardData || {};
+                        // Extract splits from cardData.dataset_info (where HF actually stores them)
+                        // cardData.dataset_info can be an object (single config) or array (multi-config)
+                        let rawSplits = [];
+                        const datasetInfoField = cardData.dataset_info;
+                        if (datasetInfoField) {
+                            const configs = Array.isArray(datasetInfoField) ? datasetInfoField : [datasetInfoField];
+                            for (const config of configs) {
+                                if (config?.splits && Array.isArray(config.splits)) {
+                                    rawSplits = rawSplits.concat(config.splits);
+                                }
+                            }
+                        }
+                        // Fallback: try top-level splits from the SDK (rarely populated)
+                        if (rawSplits.length === 0 && fullInfo.splits) {
+                            rawSplits = fullInfo.splits;
+                        }
+                        const splits = rawSplits.map((s) => ({
                             name: s.name,
-                            num_examples: s.numExamples || 0,
-                            size_bytes: s.sizeBytes
-                        })) || [];
-                        const totalExamples = splits.reduce((sum, s) => sum + (s.num_examples || 0), 0);
+                            num_examples: s.num_examples || s.numExamples || 0,
+                            size_bytes: s.num_bytes || s.sizeBytes || 0
+                        }));
+                        let totalExamples = splits.reduce((sum, s) => sum + (s.num_examples || 0), 0);
                         const totalSizeBytes = splits.reduce((sum, s) => sum + (s.size_bytes || 0), 0);
+                        // Fallback: estimate from size_categories when splits give 0
+                        if (totalExamples === 0) {
+                            const sizeCategories = cardData.size_categories;
+                            if (Array.isArray(sizeCategories) && sizeCategories.length > 0) {
+                                const cat = sizeCategories[0];
+                                const rangeMatch = cat.match(/([\d.]+[KMB]?)\s*<\s*n\s*<\s*([\d.]+[KMB]?)/i);
+                                if (rangeMatch) {
+                                    const parseHumanNum = (s) => {
+                                        const m = s.match(/^([\d.]+)([KMB])?$/i);
+                                        if (!m)
+                                            return 0;
+                                        const base = parseFloat(m[1]);
+                                        const suffix = (m[2] || '').toUpperCase();
+                                        if (suffix === 'K')
+                                            return base * 1000;
+                                        if (suffix === 'M')
+                                            return base * 1_000_000;
+                                        if (suffix === 'B')
+                                            return base * 1_000_000_000;
+                                        return base;
+                                    };
+                                    const lo = parseHumanNum(rangeMatch[1]);
+                                    const hi = parseHumanNum(rangeMatch[2]);
+                                    totalExamples = Math.round((lo + hi) / 2);
+                                }
+                            }
+                        }
                         const totalSizeMB = totalSizeBytes ? Math.round(totalSizeBytes / (1024 * 1024) * 100) / 100 : undefined;
                         const hasValidationSplit = splits.some((s) => s.name === "validation" || s.name === "val");
                         const licenseTag = tags.find(t => t.startsWith("license:"));
                         const licenseId = licenseTag ? licenseTag.replace("license:", "") : fullInfo.license;
-                        const cardData = fullInfo.cardData || {};
                         const licenseUrl = cardData.license?.[0]?.link || cardData.license_link;
                         const license = categorizeLicense(licenseId, licenseUrl);
                         if (license.category === "restricted") {

package/build/python/export_engine.py

@@ -50,6 +50,51 @@ def _load(file_path: str, options: dict) -> pl.DataFrame:
         df = pl.read_ipc(file_path)
     elif ext == ".jsonl":
         df = pl.read_ndjson(file_path)
+    elif ext == ".json":
+        # Auto-detect: array-of-objects vs NDJSON vs nested structures
+        try:
+            import json as _json
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                raw_text = fh.read(512)  # peek
+            stripped = raw_text.lstrip()
+            if stripped.startswith("["):
+                # Array of objects — standard JSON
+                with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                    data = _json.load(fh)
+                if isinstance(data, list) and len(data) > 0:
+                    df = pl.DataFrame(data)
+                else:
+                    raise ValueError("JSON file is empty or not an array of objects")
+            elif stripped.startswith("{"):
+                # Could be NDJSON or a single object wrapping rows
+                try:
+                    df = pl.read_ndjson(file_path)
+                except Exception:
+                    with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                        data = _json.load(fh)
+                    # Try common wrapper patterns: {"data": [...]}, {"rows": [...]}, etc.
+                    rows = None
+                    if isinstance(data, dict):
+                        for key in ("data", "rows", "records", "items", "results", "entries"):
+                            if key in data and isinstance(data[key], list):
+                                rows = data[key]
+                                break
+                        if rows is None:
+                            # Last resort: try to use the dict values
+                            rows = [data]
+                    if rows and len(rows) > 0:
+                        df = pl.DataFrame(rows)
+                    else:
+                        raise ValueError("Could not parse JSON structure into tabular data")
+            else:
+                raise ValueError("JSON file does not start with [ or {")
+        except pl.exceptions.ComputeError as ce:
+            raise ValueError(f"Failed to parse JSON: {ce}")
+    elif ext == ".xlsx":
+        try:
+            df = pl.read_excel(file_path)
+        except Exception as e:
+            raise ValueError(f"Failed to read Excel file: {e}")
     else:
         raise ValueError(f"Unsupported input format: {ext}")
 

package/build/python/normalize_engine.py

@@ -0,0 +1,83 @@
+"""
+Normalize any supported dataset file to parquet format.
+Usage: normalize_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        # Try NDJSON
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        # Try wrapper object
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            # Last resort - take first list value
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    if ext == ".txt":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def normalize(input_path: str, output_path: str):
+    df = _load(input_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    df.write_parquet(output_path)
+    return df.height
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        rows = normalize(input_path, output_path)
+        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/build/search/engine.js

@@ -96,6 +96,34 @@ export class SearchEngine {
             if (lexicalScore === 0 && positiveKeywords.length > 1) {
                 penalty += 0.2;
             }
+            // Penalty: Modality Mismatch
+            // Infer the expected modality from the query and penalize mismatches.
+            // e.g., "anime quotes" is text, so image-classification datasets get penalized.
+            const textIndicators = ["quotes", "text", "nlp", "sentiment", "review", "comment", "caption", "dialogue", "chat", "translation", "summarization", "classification"];
+            const imageIndicators = ["image", "photo", "picture", "vision", "detection", "segmentation", "face", "background"];
+            const queryLower = query.toLowerCase();
+            const queryLooksText = textIndicators.some(t => queryLower.includes(t));
+            const queryLooksImage = imageIndicators.some(t => queryLower.includes(t));
+            if (queryLooksText && !queryLooksImage) {
+                const resultTask = (metadata.task || "").toLowerCase();
+                const isImageResult = resultTask.includes("image") || resultTask.includes("object-detection") ||
+                    text.includes("image classification") || text.includes("image-classification") ||
+                    text.includes("object detection") || text.includes("image segmentation");
+                if (isImageResult) {
+                    log(`Modality penalty: text query but image dataset ${match.id}`);
+                    penalty += 0.35;
+                }
+            }
+            if (queryLooksImage && !queryLooksText) {
+                const resultTask = (metadata.task || "").toLowerCase();
+                const isTextResult = resultTask.includes("text-classification") || resultTask.includes("text-generation") ||
+                    resultTask.includes("translation") || resultTask.includes("summarization") ||
+                    resultTask.includes("question-answering");
+                if (isTextResult) {
+                    log(`Modality penalty: image query but text dataset ${match.id}`);
+                    penalty += 0.35;
+                }
+            }
             // D. Accessibility Bonuses (Prioritize low-friction sources)
             let bonus = 0;
             const sourceBonuses = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vesper-wizard",
-  "version": "2.0.8",
+  "version": "2.0.9",
   "description": "AI-powered dataset discovery, quality analysis, and preparation MCP server with multimodal support (text, image, audio, video)",
   "type": "module",
   "main": "build/index.js",

package/src/python/export_engine.py

@@ -50,6 +50,51 @@ def _load(file_path: str, options: dict) -> pl.DataFrame:
         df = pl.read_ipc(file_path)
     elif ext == ".jsonl":
         df = pl.read_ndjson(file_path)
+    elif ext == ".json":
+        # Auto-detect: array-of-objects vs NDJSON vs nested structures
+        try:
+            import json as _json
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                raw_text = fh.read(512)  # peek
+            stripped = raw_text.lstrip()
+            if stripped.startswith("["):
+                # Array of objects — standard JSON
+                with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                    data = _json.load(fh)
+                if isinstance(data, list) and len(data) > 0:
+                    df = pl.DataFrame(data)
+                else:
+                    raise ValueError("JSON file is empty or not an array of objects")
+            elif stripped.startswith("{"):
+                # Could be NDJSON or a single object wrapping rows
+                try:
+                    df = pl.read_ndjson(file_path)
+                except Exception:
+                    with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                        data = _json.load(fh)
+                    # Try common wrapper patterns: {"data": [...]}, {"rows": [...]}, etc.
+                    rows = None
+                    if isinstance(data, dict):
+                        for key in ("data", "rows", "records", "items", "results", "entries"):
+                            if key in data and isinstance(data[key], list):
+                                rows = data[key]
+                                break
+                        if rows is None:
+                            # Last resort: try to use the dict values
+                            rows = [data]
+                    if rows and len(rows) > 0:
+                        df = pl.DataFrame(rows)
+                    else:
+                        raise ValueError("Could not parse JSON structure into tabular data")
+            else:
+                raise ValueError("JSON file does not start with [ or {")
+        except pl.exceptions.ComputeError as ce:
+            raise ValueError(f"Failed to parse JSON: {ce}")
+    elif ext == ".xlsx":
+        try:
+            df = pl.read_excel(file_path)
+        except Exception as e:
+            raise ValueError(f"Failed to read Excel file: {e}")
     else:
         raise ValueError(f"Unsupported input format: {ext}")
 

package/src/python/normalize_engine.py

@@ -0,0 +1,83 @@
+"""
+Normalize any supported dataset file to parquet format.
+Usage: normalize_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        # Try NDJSON
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        # Try wrapper object
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            # Last resort - take first list value
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    if ext == ".txt":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def normalize(input_path: str, output_path: str):
+    df = _load(input_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    df.write_parquet(output_path)
+    return df.height
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        rows = normalize(input_path, output_path)
+        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()