wicked-garden 12.27.0

package/scripts/data/data_profiler.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""
+Data profiler: Analyze datasets and generate profiling reports.
+
+Profiles CSV/Excel files to understand:
+- Row/column counts
+- Data types
+- Null rates
+- Cardinality
+- Statistical summaries
+- Sample values
+"""
+
+import argparse
+import csv
+import json
+import re
+import sys
+import statistics
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Shared datetime regex — also used in schema_validator.py
+DATETIME_RE = re.compile(
+    r'^\d{4}[-/]\d{1,2}[-/]\d{1,2}'   # date part: YYYY-MM-DD
+    r'([T ]\d{1,2}:\d{2}(:\d{2})?)?'   # optional time part
+    r'([Zz]|[+-]\d{2}:?\d{2})?$'       # optional timezone
+)
+
+
+def detect_type(values: List[Any]) -> str:
+    """Infer data type from sample values."""
+    # Remove None/empty values for type detection
+    non_null = [v for v in values if v not in (None, '', 'NULL', 'null')]
+
+    if not non_null:
+        return "string"
+
+    # Try integer
+    try:
+        [int(v) for v in non_null[:100]]
+        return "integer"
+    except (ValueError, TypeError):
+        pass  # intentional: type probe — try next type
+
+    # Try decimal
+    try:
+        [float(v) for v in non_null[:100]]
+        return "decimal"
+    except (ValueError, TypeError):
+        pass  # intentional: type probe — try next type
+
+    # Try boolean
+    bool_values = {'true', 'false', '1', '0', 'yes', 'no', 't', 'f'}
+    if all(str(v).lower() in bool_values for v in non_null[:100]):
+        return "boolean"
+
+    # Try datetime — check a sample against common date formats
+    sample = non_null[:20]
+    if all(DATETIME_RE.match(str(v)) for v in sample):
+        return "datetime"
+
+    return "string"
+
+
+def calculate_stats(values: List[Any], data_type: str) -> Dict[str, Any]:
+    """Calculate statistical summary for numeric columns."""
+    if data_type not in ("integer", "decimal"):
+        return {}
+
+    # Convert to floats
+    numeric_values = []
+    for v in values:
+        if v not in (None, '', 'NULL', 'null'):
+            try:
+                numeric_values.append(float(v))
+            except (ValueError, TypeError):
+                pass  # fail open: non-numeric value skipped
+
+    if not numeric_values:
+        return {}
+
+    stats = {
+        "min": min(numeric_values),
+        "max": max(numeric_values),
+        "mean": statistics.mean(numeric_values),
+        "median": statistics.median(numeric_values)
+    }
+
+    if len(numeric_values) > 1:
+        stats["stddev"] = statistics.stdev(numeric_values)
+
+    return stats
+
+
+def profile_csv(file_path: Path, sample_size: int = 1000) -> Dict[str, Any]:
+    """Profile a CSV file."""
+    print(f"Profiling {file_path}...", file=sys.stderr)
+
+    # First pass: collect samples
+    column_samples = defaultdict(list)
+    row_count = 0
+    column_names = []
+
+    with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
+        reader = csv.DictReader(f)
+        column_names = reader.fieldnames or []
+
+        for row in reader:
+            row_count += 1
+
+            # Collect samples for profiling
+            if row_count <= sample_size:
+                for col in column_names:
+                    value = row.get(col, '')
+                    column_samples[col].append(value)
+
+    # Second pass: analyze each column
+    schema = []
+    for col in column_names:
+        samples = column_samples[col]
+
+        # Count nulls
+        null_count = sum(1 for v in samples if v in (None, '', 'NULL', 'null'))
+        null_rate = null_count / len(samples) if samples else 0
+
+        # Infer type
+        data_type = detect_type(samples)
+
+        # Calculate cardinality
+        unique_values = set(samples)
+        cardinality = len(unique_values)
+
+        # Get sample values (non-null)
+        non_null_samples = [v for v in samples if v not in (None, '', 'NULL', 'null')]
+        sample_values = list(set(non_null_samples[:10]))
+
+        # Calculate stats for numeric columns
+        stats = calculate_stats(samples, data_type)
+
+        column_info = {
+            "name": col,
+            "type": data_type,
+            "null_rate": round(null_rate, 4),
+            "cardinality": cardinality,
+            "sample": sample_values[:5]
+        }
+
+        if stats:
+            column_info["stats"] = {
+                k: round(v, 2) if isinstance(v, float) else v
+                for k, v in stats.items()
+            }
+
+        schema.append(column_info)
+
+    # Calculate quality score
+    quality_score = calculate_quality_score(schema, row_count)
+
+    # Identify issues
+    issues = []
+    for col in schema:
+        if col["null_rate"] > 0.05:
+            issues.append(f"Column '{col['name']}' has {col['null_rate']*100:.1f}% null values")
+
+        if col["cardinality"] == 1:
+            issues.append(f"Column '{col['name']}' has only one distinct value")
+
+    return {
+        "file": str(file_path),
+        "rows": row_count,
+        "columns": len(column_names),
+        "schema": schema,
+        "quality_score": quality_score,
+        "issues": issues
+    }
+
+
+def calculate_quality_score(schema: List[Dict], row_count: int) -> int:
+    """Calculate overall quality score (0-100)."""
+    if not schema or row_count == 0:
+        return 0
+
+    # Factors:
+    # - Low null rates (60% weight)
+    # - Type diversity (20% weight)
+    # - Reasonable cardinality (20% weight)
+
+    # Null rate score
+    avg_null_rate = sum(col["null_rate"] for col in schema) / len(schema)
+    null_score = max(0, 100 - (avg_null_rate * 100))
+
+    # Type diversity score
+    types = set(col["type"] for col in schema)
+    type_score = min(100, len(types) * 25)
+
+    # Cardinality score (avoid all unique or all same)
+    cardinality_scores = []
+    for col in schema:
+        ratio = col["cardinality"] / row_count if row_count > 0 else 0
+        # Good: 0.1 to 0.9 (neither all same nor all unique)
+        if 0.1 <= ratio <= 0.9:
+            cardinality_scores.append(100)
+        elif ratio == 1.0 and col["type"] in ("integer", "string"):
+            # IDs are OK
+            cardinality_scores.append(100)
+        else:
+            cardinality_scores.append(50)
+
+    card_score = statistics.mean(cardinality_scores) if cardinality_scores else 50
+
+    # Weighted average
+    quality_score = (null_score * 0.6) + (type_score * 0.2) + (card_score * 0.2)
+
+    return int(quality_score)
+
+
+def profile_excel(file_path: Path) -> Dict[str, Any]:
+    """Profile an Excel file."""
+    try:
+        import openpyxl
+    except ImportError:
+        return {
+            "error": "openpyxl not installed. Install with: pip install openpyxl"
+        }
+
+    # For now, return placeholder
+    # Full implementation would be similar to CSV
+    return {
+        "error": "Excel profiling not yet implemented. Convert to CSV first."
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Profile datasets")
+    parser.add_argument("--input", required=True, help="Input file path")
+    parser.add_argument("--output", help="Output JSON file (default: stdout)")
+    parser.add_argument("--sample-size", type=int, default=1000,
+                        help="Number of rows to sample (default: 1000)")
+
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+
+    if not input_path.exists():
+        print(f"Error: File not found: {input_path}", file=sys.stderr)
+        sys.exit(1)
+
+    # Detect file type and profile
+    if input_path.suffix.lower() == '.csv':
+        profile = profile_csv(input_path, args.sample_size)
+    elif input_path.suffix.lower() in ('.xlsx', '.xls'):
+        profile = profile_excel(input_path)
+    else:
+        print(f"Error: Unsupported file type: {input_path.suffix}", file=sys.stderr)
+        sys.exit(1)
+
+    # Output
+    output_json = json.dumps(profile, indent=2)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(output_json)
+        print(f"Profile written to {args.output}", file=sys.stderr)
+    else:
+        print(output_json)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/data/ontology_recommender.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Ontology recommender — sample dataset and suggest matching ontologies."""
+import json
+import re
+import sys
+from pathlib import Path
+
+# Known ontology catalogs with column name patterns
+ONTOLOGIES = {
+    "schema.org/Person": {
+        "patterns": {
+            "name": ["name", "full_name", "fullname", "display_name"],
+            "email": ["email", "e_mail", "email_address", "mail"],
+            "telephone": ["phone", "telephone", "mobile", "cell", "phone_number"],
+            "address": ["address", "street", "city", "state", "zip", "postal", "country"],
+            "birthDate": ["birth_date", "dob", "date_of_birth", "birthday"],
+            "gender": ["gender", "sex"],
+            "jobTitle": ["job_title", "title", "position", "role"],
+        },
+    },
+    "schema.org/Organization": {
+        "patterns": {
+            "name": ["company", "organization", "org_name", "company_name"],
+            "url": ["website", "url", "homepage", "web"],
+            "industry": ["industry", "sector", "vertical"],
+            "foundingDate": ["founded", "founding_date", "established"],
+            "numberOfEmployees": ["employees", "headcount", "team_size"],
+        },
+    },
+    "schema.org/Product": {
+        "patterns": {
+            "name": ["product", "product_name", "item", "item_name"],
+            "price": ["price", "cost", "amount", "unit_price"],
+            "sku": ["sku", "product_id", "item_id", "upc", "barcode"],
+            "category": ["category", "type", "product_type", "classification"],
+            "brand": ["brand", "manufacturer", "vendor"],
+            "description": ["description", "desc", "details", "summary"],
+        },
+    },
+    "schema.org/Event": {
+        "patterns": {
+            "name": ["event", "event_name", "title"],
+            "startDate": ["start_date", "start", "begin", "event_date"],
+            "endDate": ["end_date", "end", "finish"],
+            "location": ["location", "venue", "place"],
+            "organizer": ["organizer", "host", "creator"],
+        },
+    },
+    "Dublin Core": {
+        "patterns": {
+            "title": ["title", "name", "heading", "subject_line"],
+            "creator": ["creator", "author", "writer", "created_by"],
+            "subject": ["subject", "topic", "category", "tags"],
+            "description": ["description", "abstract", "summary", "body"],
+            "date": ["date", "created_at", "published", "modified"],
+            "format": ["format", "mime_type", "content_type", "file_type"],
+            "identifier": ["id", "identifier", "uri", "doi", "isbn"],
+            "language": ["language", "lang", "locale"],
+            "source": ["source", "origin", "reference"],
+        },
+    },
+    "DCAT": {
+        "patterns": {
+            "distribution": ["url", "download_url", "access_url", "file_path"],
+            "byteSize": ["size", "file_size", "byte_size", "bytes"],
+            "mediaType": ["media_type", "mime_type", "content_type"],
+            "issued": ["issued", "published", "release_date"],
+            "modified": ["modified", "updated", "last_updated"],
+            "keyword": ["keyword", "tag", "tags", "label"],
+        },
+    },
+    "FOAF": {
+        "patterns": {
+            "name": ["name", "display_name", "username"],
+            "mbox": ["email", "mail"],
+            "knows": ["friend", "connection", "contact", "follows"],
+            "interest": ["interest", "hobby", "preference"],
+            "account": ["account", "profile", "handle", "screen_name"],
+            "img": ["avatar", "photo", "image", "picture", "profile_pic"],
+        },
+    },
+    "GoodRelations": {
+        "patterns": {
+            "hasCurrencyValue": ["price", "amount", "cost", "value"],
+            "hasCurrency": ["currency", "currency_code"],
+            "eligibleRegion": ["region", "country", "market"],
+            "validFrom": ["valid_from", "start_date", "effective"],
+            "validThrough": ["valid_to", "end_date", "expiry", "expires"],
+            "condition": ["condition", "status", "state"],
+        },
+    },
+    "SKOS": {
+        "patterns": {
+            "prefLabel": ["label", "name", "title", "display"],
+            "broader": ["parent", "parent_id", "broader", "category"],
+            "narrower": ["child", "children", "subcategory", "narrower"],
+            "notation": ["code", "notation", "identifier", "abbreviation"],
+            "definition": ["definition", "description", "meaning"],
+        },
+    },
+}
+
+
+def normalize_column(col: str) -> str:
+    """Normalize column name for matching."""
+    return re.sub(r'[^a-z0-9]', '_', col.lower()).strip('_')
+
+
+def match_ontologies(columns: list[str]) -> list[dict]:
+    """Match columns against known ontology patterns."""
+    normalized = [normalize_column(c) for c in columns]
+    results = []
+
+    for ontology_name, config in ONTOLOGIES.items():
+        matched_cols = []
+        mappings = []
+
+        for prop_name, patterns in config["patterns"].items():
+            for i, norm_col in enumerate(normalized):
+                if norm_col in patterns or any(p in norm_col for p in patterns):
+                    matched_cols.append(columns[i])
+                    mappings.append({
+                        "column": columns[i],
+                        "property": f"{ontology_name.split('/')[0].lower()}:{prop_name}",
+                        "confidence": "high" if norm_col in patterns else "medium",
+                    })
+                    break
+
+        if matched_cols and config["patterns"]:
+            score = len(matched_cols) / len(config["patterns"])
+            results.append({
+                "ontology": ontology_name,
+                "score": round(score * 100),
+                "matched_columns": matched_cols,
+                "total_properties": len(config["patterns"]),
+                "mappings": mappings,
+            })
+
+    results.sort(key=lambda x: x["score"], reverse=True)
+    return results
+
+
+def profile_dataset(file_path: str) -> dict:
+    """Profile a dataset using DuckDB or fallback to csv module."""
+    path = Path(file_path)
+    suffix = path.suffix.lower()
+
+    try:
+        import duckdb
+        con = duckdb.connect()
+
+        if suffix == ".csv":
+            query = f"SELECT * FROM read_csv_auto('{file_path}') LIMIT 100"
+        elif suffix in (".xlsx", ".xls"):
+            query = f"SELECT * FROM read_excel('{file_path}') LIMIT 100"
+        elif suffix == ".parquet":
+            query = f"SELECT * FROM read_parquet('{file_path}') LIMIT 100"
+        elif suffix == ".json":
+            query = f"SELECT * FROM read_json_auto('{file_path}') LIMIT 100"
+        else:
+            return {"error": f"Unsupported format: {suffix}"}
+
+        result = con.execute(query)
+        columns = [desc[0] for desc in result.description]
+        rows = result.fetchall()
+        types = [desc[1] for desc in result.description]
+
+        return {
+            "columns": columns,
+            "types": [str(t) for t in types],
+            "row_count": len(rows),
+            "sample_values": {
+                col: [str(rows[i][j]) for i in range(min(5, len(rows))) if rows[i][j] is not None]
+                for j, col in enumerate(columns)
+            },
+        }
+    except ImportError:
+        # Fallback to csv module
+        import csv
+        if suffix != ".csv":
+            return {"error": "DuckDB not available, only CSV supported in fallback mode"}
+
+        with open(file_path) as f:
+            reader = csv.DictReader(f)
+            columns = reader.fieldnames or []
+            rows = []
+            for i, row in enumerate(reader):
+                if i >= 100:
+                    break
+                rows.append(row)
+
+        return {
+            "columns": list(columns),
+            "types": ["string"] * len(columns),
+            "row_count": len(rows),
+            "sample_values": {
+                col: [rows[i][col] for i in range(min(5, len(rows))) if rows[i].get(col)]
+                for col in columns
+            },
+        }
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(json.dumps({"error": "Usage: ontology_recommender.py <file-path>"}))
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+
+    if not Path(file_path).exists():
+        print(json.dumps({"error": f"File not found: {file_path}"}))
+        sys.exit(1)
+
+    profile = profile_dataset(file_path)
+    if "error" in profile:
+        print(json.dumps(profile))
+        sys.exit(1)
+
+    recommendations = match_ontologies(profile["columns"])
+
+    output = {
+        "file": file_path,
+        "profile": profile,
+        "recommendations": recommendations,
+    }
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()