devsper 2.1.6__py3-none-any.whl

devsper/tools/data/dataframe_filter.py

@@ -0,0 +1,51 @@
+"""Filter a CSV by column value (simple equality). Requires pandas."""
+
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class DataframeFilterTool(Tool):
+    """Filter CSV rows where column equals value. Requires pandas."""
+
+    name = "dataframe_filter"
+    description = "Filter CSV by column value. Returns matching rows as string. Requires pandas."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to CSV"},
+            "column": {"type": "string", "description": "Column name"},
+            "value": {"type": "string", "description": "Value to match (string)"},
+        },
+        "required": ["path", "column", "value"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        column = kwargs.get("column")
+        value = kwargs.get("value")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if not column or not isinstance(column, str):
+            return "Error: column must be a non-empty string"
+        if value is None:
+            return "Error: value is required"
+        try:
+            import pandas as pd
+        except ImportError:
+            return "Error: pandas required. Install with: pip install pandas"
+        p = Path(path)
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            df = pd.read_csv(p)
+            if column not in df.columns:
+                return f"Error: column '{column}' not in {list(df.columns)}"
+            filtered = df[df[column].astype(str) == str(value)]
+            return filtered.to_string() if not filtered.empty else "No matching rows."
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(DataframeFilterTool())

devsper/tools/data/dataframe_groupby.py

@@ -0,0 +1,47 @@
+"""Group CSV by column and show count per group. Requires pandas."""
+
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class DataframeGroupbyTool(Tool):
+    """Group CSV by a column and return count per group. Requires pandas."""
+
+    name = "dataframe_groupby"
+    description = "Group CSV by column and show counts. Requires pandas."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to CSV"},
+            "column": {"type": "string", "description": "Column to group by"},
+        },
+        "required": ["path", "column"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        column = kwargs.get("column")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if not column or not isinstance(column, str):
+            return "Error: column must be a non-empty string"
+        try:
+            import pandas as pd
+        except ImportError:
+            return "Error: pandas required. Install with: pip install pandas"
+        p = Path(path)
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            df = pd.read_csv(p)
+            if column not in df.columns:
+                return f"Error: column '{column}' not in {list(df.columns)}"
+            grouped = df.groupby(column).size()
+            return grouped.to_string()
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(DataframeGroupbyTool())

devsper/tools/data/dataframe_stats.py

@@ -0,0 +1,38 @@
+"""Compute basic statistics for a CSV as a dataframe (requires pandas)."""
+
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class DataframeStatsTool(Tool):
+    """Load CSV as dataframe and return describe() stats. Requires pandas."""
+
+    name = "dataframe_stats"
+    description = "Compute summary statistics for a CSV file. Requires pandas."
+    input_schema = {
+        "type": "object",
+        "properties": {"path": {"type": "string", "description": "Path to CSV file"}},
+        "required": ["path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        try:
+            import pandas as pd
+        except ImportError:
+            return "Error: pandas required. Install with: pip install pandas"
+        p = Path(path)
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            df = pd.read_csv(p)
+            return str(df.describe())
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(DataframeStatsTool())

devsper/tools/data/dataset_sampling.py

@@ -0,0 +1,55 @@
+"""Sample N random rows from a CSV."""
+
+import csv
+import random
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class DatasetSamplingTool(Tool):
+    """Return a random sample of N rows from a CSV file."""
+
+    name = "dataset_sampling"
+    description = "Sample N random rows from a CSV. Returns sampled rows as text."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to CSV"},
+            "n": {"type": "integer", "description": "Number of rows to sample"},
+        },
+        "required": ["path", "n"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        n = kwargs.get("n")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if not isinstance(n, int) or n < 1:
+            return "Error: n must be a positive integer"
+        p = Path(path)
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            with p.open(encoding="utf-8", errors="replace", newline="") as f:
+                reader = csv.reader(f)
+                rows = list(reader)
+            if not rows:
+                return "Empty CSV."
+            header = rows[0]
+            data = rows[1:]
+            if n >= len(data):
+                sample = data
+            else:
+                sample = random.sample(data, n)
+            lines = [",".join(header)]
+            for row in sample:
+                lines.append(",".join(str(c) for c in row))
+            return "\n".join(lines)
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(DatasetSamplingTool())

devsper/tools/data/dataset_schema.py

@@ -0,0 +1,45 @@
+"""Describe CSV schema: column names and sample value types."""
+
+import csv
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class DatasetSchemaTool(Tool):
+    """Return CSV schema: column names and a one-line description (from first row)."""
+
+    name = "dataset_schema"
+    description = "Describe CSV schema: column names and example types."
+    input_schema = {
+        "type": "object",
+        "properties": {"path": {"type": "string", "description": "Path to CSV file"}},
+        "required": ["path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        p = Path(path)
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            with p.open(encoding="utf-8", errors="replace", newline="") as f:
+                reader = csv.reader(f)
+                rows = list(reader)
+            if not rows:
+                return "Empty CSV."
+            header = rows[0]
+            sample = rows[1] if len(rows) > 1 else []
+            lines = [f"Columns: {len(header)}", ""]
+            for i, name in enumerate(header):
+                ex = sample[i] if i < len(sample) else ""
+                lines.append(f"  {name}: example = {repr(ex)[:50]}")
+            return "\n".join(lines)
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(DatasetSchemaTool())

devsper/tools/data/json_pretty_print.py

@@ -0,0 +1,37 @@
+"""Pretty-print JSON with indentation."""
+
+import json
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class JsonPrettyPrintTool(Tool):
+    """Format JSON string with indentation for readability."""
+
+    name = "json_pretty_print"
+    description = "Pretty-print a JSON string with indentation."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "json_str": {"type": "string", "description": "JSON string to format"},
+            "indent": {"type": "integer", "description": "Indent spaces (default 2)"},
+        },
+        "required": ["json_str"],
+    }
+
+    def run(self, **kwargs) -> str:
+        json_str = kwargs.get("json_str")
+        indent = kwargs.get("indent", 2)
+        if json_str is None:
+            return "Error: json_str is required"
+        if not isinstance(indent, int) or indent < 0:
+            indent = 2
+        try:
+            data = json.loads(json_str)
+            return json.dumps(data, indent=indent)
+        except json.JSONDecodeError as e:
+            return f"Invalid JSON: {e}"
+
+
+register(JsonPrettyPrintTool())

devsper/tools/data/json_query.py

@@ -0,0 +1,46 @@
+"""Query JSON with a simple key path (e.g. "a.b.0")."""
+
+import json
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class JsonQueryTool(Tool):
+    """Get a value from JSON by key path. Path uses dot notation and index for arrays (e.g. data.items.0)."""
+
+    name = "json_query"
+    description = "Query JSON with a dot-path (e.g. key.nested.0). Returns the value as string."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "json_str": {"type": "string", "description": "JSON string"},
+            "path": {"type": "string", "description": "Dot-separated path, use numbers for list index"},
+        },
+        "required": ["json_str", "path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        json_str = kwargs.get("json_str")
+        path = kwargs.get("path")
+        if json_str is None:
+            return "Error: json_str is required"
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        try:
+            data = json.loads(json_str)
+        except json.JSONDecodeError as e:
+            return f"Invalid JSON: {e}"
+        parts = path.strip().split(".")
+        try:
+            for p in parts:
+                if p.isdigit():
+                    data = data[int(p)]
+                else:
+                    data = data[p]
+            return json.dumps(data) if not isinstance(data, str) else data
+        except (KeyError, IndexError, TypeError) as e:
+            return f"Path error: {e}"
+
+
+register(JsonQueryTool())

devsper/tools/data/missing_value_report.py

@@ -0,0 +1,47 @@
+"""Report missing/empty values per column in a CSV."""
+
+import csv
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class MissingValueReportTool(Tool):
+    """Count missing or empty values per column in a CSV."""
+
+    name = "missing_value_report"
+    description = "Report count of missing/empty values per column in a CSV."
+    input_schema = {
+        "type": "object",
+        "properties": {"path": {"type": "string", "description": "Path to CSV file"}},
+        "required": ["path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        p = Path(path)
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            with p.open(encoding="utf-8", errors="replace", newline="") as f:
+                reader = csv.reader(f)
+                rows = list(reader)
+            if not rows:
+                return "Empty CSV."
+            header = rows[0]
+            data = rows[1:]
+            n_rows = len(data)
+            lines = [f"Total rows: {n_rows}", ""]
+            for col_idx, col_name in enumerate(header):
+                missing = sum(1 for row in data if col_idx >= len(row) or (row[col_idx] or "").strip() == "")
+                pct = (100 * missing / n_rows) if n_rows else 0
+                lines.append(f"{col_name}: {missing} missing ({pct:.1f}%)")
+            return "\n".join(lines)
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(MissingValueReportTool())

devsper/tools/data_science/__init__.py

@@ -0,0 +1,13 @@
+"""Data science tools: profiling, distributions, outliers, correlation, drift, bias."""
+
+from devsper.tools.data_science.dataset_profile import DatasetProfileTool
+from devsper.tools.data_science.feature_importance_estimator import FeatureImportanceEstimatorTool
+from devsper.tools.data_science.dataset_distribution_report import DatasetDistributionReportTool
+from devsper.tools.data_science.dataset_outlier_detector import DatasetOutlierDetectorTool
+from devsper.tools.data_science.correlation_heatmap import CorrelationHeatmapTool
+from devsper.tools.data_science.time_series_analyzer import TimeSeriesAnalyzerTool
+from devsper.tools.data_science.model_input_validator import ModelInputValidatorTool
+from devsper.tools.data_science.dataset_drift_detector import DatasetDriftDetectorTool
+from devsper.tools.data_science.feature_engineering_suggestions import FeatureEngineeringSuggestionsTool
+from devsper.tools.data_science.dataset_bias_detector import DatasetBiasDetectorTool
+from devsper.tools.data_science.distributed_dataset_processor import DistributedDatasetProcessorTool

devsper/tools/data_science/correlation_heatmap.py

@@ -0,0 +1,72 @@
+"""Compute pairwise correlation matrix for numeric columns and return as text table."""
+
+import csv
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+try:
+    import numpy as np
+except ImportError:
+    np = None
+
+
+class CorrelationHeatmapTool(Tool):
+    """
+    Compute pairwise correlations for numeric columns and return a text-based heatmap (matrix).
+    """
+
+    name = "correlation_heatmap"
+    description = "Compute pairwise correlation matrix for numeric columns; return as text table."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to CSV"},
+            "max_columns": {"type": "integer", "description": "Max columns (default 10)"},
+        },
+        "required": ["path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        max_columns = kwargs.get("max_columns", 10)
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if not isinstance(max_columns, int) or max_columns < 1:
+            max_columns = 10
+        if np is None:
+            return "Error: numpy is required for this tool"
+        p = Path(path).resolve()
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            with p.open(encoding="utf-8", errors="replace", newline="") as f:
+                reader = csv.DictReader(f)
+                rows = list(reader)
+            if not rows:
+                return "Empty CSV."
+            numeric_cols = []
+            for col in list(rows[0].keys())[:max_columns]:
+                try:
+                    arr = np.array([float(row.get(col, np.nan)) for row in rows], dtype=float)
+                    if np.isnan(arr).all():
+                        continue
+                    numeric_cols.append((col, arr))
+                except (ValueError, TypeError):
+                    continue
+            if len(numeric_cols) < 2:
+                return "Need at least 2 numeric columns."
+            names = [n for n, _ in numeric_cols]
+            mat = np.column_stack([a for _, a in numeric_cols])
+            corr = np.corrcoef(mat.T)
+            lines = ["Correlation matrix", "=" * 40, "Columns: " + ", ".join(names), ""]
+            for i, name in enumerate(names):
+                parts = [f"{corr[i, j]:.2f}" for j in range(len(names))]
+                lines.append(f"  {name}: " + " ".join(parts))
+            return "\n".join(lines)
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(CorrelationHeatmapTool())

devsper/tools/data_science/dataset_bias_detector.py

@@ -0,0 +1,49 @@
+"""Detect potential bias: check balance of a categorical variable (e.g. class or group)."""
+
+from collections import Counter
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class DatasetBiasDetectorTool(Tool):
+    """
+    Detect potential class/group imbalance: report distribution of a categorical variable.
+    """
+
+    name = "dataset_bias_detector"
+    description = "Detect potential bias: report balance of a categorical (e.g. class) distribution."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "values": {
+                "type": "array",
+                "items": {},
+                "description": "List of categorical values (e.g. labels or group IDs)",
+            },
+            "imbalance_ratio_threshold": {"type": "number", "description": "Flag if max/min count ratio > this (default 10)"},
+        },
+        "required": ["values"],
+    }
+
+    def run(self, **kwargs) -> str:
+        values = kwargs.get("values")
+        threshold = kwargs.get("imbalance_ratio_threshold", 10.0)
+        if not values or not isinstance(values, list):
+            return "Error: values must be a non-empty list"
+        counts = Counter(str(v) for v in values)
+        if len(counts) < 2:
+            return "Need at least 2 distinct categories to assess balance."
+        total = sum(counts.values())
+        max_c, min_c = max(counts.values()), min(counts.values())
+        ratio = max_c / min_c if min_c > 0 else float("inf")
+        lines = ["Bias / balance report", "=" * 40, f"Total: {total}, Categories: {len(counts)}", f"Max/min count ratio: {ratio:.1f}"]
+        if ratio > threshold:
+            lines.append("Imbalance detected (ratio > threshold).")
+        for label, c in counts.most_common(15):
+            pct = 100 * c / total
+            lines.append(f"  {label}: {c} ({pct:.1f}%)")
+        return "\n".join(lines)
+
+
+register(DatasetBiasDetectorTool())

devsper/tools/data_science/dataset_distribution_report.py

@@ -0,0 +1,64 @@
+"""Report distribution statistics for numeric columns: mean, std, quartiles."""
+
+import csv
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+try:
+    import numpy as np
+except ImportError:
+    np = None
+
+
+class DatasetDistributionReportTool(Tool):
+    """
+    Report distribution stats for numeric columns: mean, std, min, max, quartiles.
+    """
+
+    name = "dataset_distribution_report"
+    description = "Report distribution statistics for numeric columns in a dataset."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to CSV"},
+            "max_columns": {"type": "integer", "description": "Max columns to report (default 20)"},
+        },
+        "required": ["path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        max_columns = kwargs.get("max_columns", 20)
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if not isinstance(max_columns, int) or max_columns < 1:
+            max_columns = 20
+        if np is None:
+            return "Error: numpy is required for this tool"
+        p = Path(path).resolve()
+        if not p.exists() or not p.is_file():
+            return f"Error: file not found: {path}"
+        try:
+            with p.open(encoding="utf-8", errors="replace", newline="") as f:
+                reader = csv.DictReader(f)
+                rows = list(reader)
+            if not rows:
+                return "Empty CSV."
+            lines = ["Distribution report", "=" * 40]
+            for col in list(rows[0].keys())[:max_columns]:
+                try:
+                    arr = np.array([float(row.get(col, np.nan)) for row in rows], dtype=float)
+                    arr = arr[~np.isnan(arr)]
+                    if len(arr) < 2:
+                        continue
+                    lines.append(f"{col}: mean={np.mean(arr):.4f}, std={np.std(arr):.4f}, min={np.min(arr):.4f}, max={np.max(arr):.4f}, q25={np.percentile(arr, 25):.4f}, q75={np.percentile(arr, 75):.4f}")
+                except (ValueError, TypeError):
+                    continue
+            return "\n".join(lines) if len(lines) > 1 else "No numeric columns found."
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(DatasetDistributionReportTool())

devsper/tools/data_science/dataset_drift_detector.py

@@ -0,0 +1,64 @@
+"""Detect simple distribution drift between two samples: compare mean and std."""
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+try:
+    import numpy as np
+except ImportError:
+    np = None
+
+
+class DatasetDriftDetectorTool(Tool):
+    """
+    Detect basic drift between two numeric samples: difference in mean and std.
+    """
+
+    name = "dataset_drift_detector"
+    description = "Detect distribution drift between two numeric samples (mean, std comparison)."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "baseline": {
+                "type": "array",
+                "items": {"type": "number"},
+                "description": "Baseline sample",
+            },
+            "current": {
+                "type": "array",
+                "items": {"type": "number"},
+                "description": "Current sample",
+            },
+            "threshold_std": {"type": "number", "description": "Flag if mean diff > this many baseline stds (default 2)"},
+        },
+        "required": ["baseline", "current"],
+    }
+
+    def run(self, **kwargs) -> str:
+        baseline = kwargs.get("baseline")
+        current = kwargs.get("current")
+        threshold_std = kwargs.get("threshold_std", 2.0)
+        if not baseline or not isinstance(baseline, list):
+            return "Error: baseline must be a non-empty list of numbers"
+        if not current or not isinstance(current, list):
+            return "Error: current must be a non-empty list of numbers"
+        if np is None:
+            return "Error: numpy is required for this tool"
+        try:
+            b = np.array(baseline, dtype=float)
+            c = np.array(current, dtype=float)
+            b, c = b[~np.isnan(b)], c[~np.isnan(c)]
+        except (ValueError, TypeError):
+            return "Error: values must be numeric"
+        if len(b) < 2 or len(c) < 2:
+            return "Error: need at least 2 values per sample"
+        mean_b, std_b = np.mean(b), np.std(b)
+        mean_c, std_c = np.mean(c), np.std(c)
+        mean_diff = abs(mean_c - mean_b)
+        std_b_safe = std_b if std_b > 0 else 1e-10
+        z = mean_diff / std_b_safe
+        drift = "Drift detected" if z > threshold_std else "No significant drift"
+        return f"{drift}. Baseline: mean={mean_b:.4f}, std={std_b:.4f}. Current: mean={mean_c:.4f}, std={std_c:.4f}. Mean diff (in baseline stds): {z:.2f}"
+
+
+register(DatasetDriftDetectorTool())