khaled-dsml 0.1.0__py3-none-any.whl

khaled_dsml/__init__.py

@@ -0,0 +1,17 @@
+"""KhaledDSML: educational statistics, ML and big data in Python."""
+__version__ = "0.1.0"
+
+from .statistics import (normality_test, shapiro_test, jarque_bera_test,
+    independent_t_test, paired_t_test, mann_whitney_test, wilcoxon_test,
+    anova_test, kruskal_test, levene_test, chi_square_test, pearson_test,
+    spearman_test, ks_two_sample_test, descriptive_summary)
+from .quality import (quality_report, missingness_report, iqr_outliers,
+    zscore_outliers, fill_missing_numeric, memory_report, optimize_memory,
+    distribution_drift_test)
+from .ml import (split_data, make_regressor, make_classifier, regression_metrics,
+    classification_metrics, cross_validation_summary, cluster_data, pca_summary)
+from .bigdata import (generate_transactions, csv_chunk_summary, csv_chunk_missing,
+    csv_chunk_monthly, map_reduce_word_count, json_documents_to_table)
+from .quizzes import Question, get_questions, grade_quiz
+from .timeseries import lagged_features
+from .sqltools import sqlite_import_frame, sqlite_group_summary

khaled_dsml/_results.py

@@ -0,0 +1,39 @@
+"""Simple, teaching-oriented results for statistical hypothesis tests."""
+from dataclasses import asdict, dataclass
+import math
+
+
+@dataclass(frozen=True)
+class TestResult:
+    test: str
+    statistic: float
+    p_value: float
+    alpha: float
+    reject_null: bool
+    interpretation_ar: str
+    null_hypothesis_ar: str
+    sample_size: int
+    notes: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def make_result(
+    name: str, statistic: float, p_value: float, n: int,
+    alpha: float, null_ar: str, notes: str = "",
+) -> TestResult:
+    if not (0 < alpha < 1):
+        raise ValueError("alpha must be between 0 and 1 (exclusive)")
+    if not math.isfinite(float(p_value)) or not (0 <= float(p_value) <= 1):
+        raise ValueError("Test returned an invalid p-value; inspect assumptions and data")
+    reject = bool(p_value < alpha)
+    interpretation = (
+        f"نرفض الفرضية الصفرية عند مستوى دلالة {alpha:g}؛ "
+        "توجد أدلة إحصائية ضدها."
+        if reject else
+        f"لا نرفض الفرضية الصفرية عند مستوى دلالة {alpha:g}؛ "
+        "لا يعني ذلك إثبات صحتها."
+    )
+    return TestResult(name, float(statistic), float(p_value), float(alpha),
+                      reject, interpretation, null_ar, int(n), notes)

khaled_dsml/bigdata.py

@@ -0,0 +1,107 @@
+"""Memory-aware big-data teaching tools (local, chunked, non-distributed)."""
+from __future__ import annotations
+
+from collections import Counter
+from pathlib import Path
+import re
+
+import numpy as np
+import pandas as pd
+
+
+def generate_transactions(n_rows=10_000, seed=42, missing_rate=.02) -> pd.DataFrame:
+    """Reproducible synthetic Algerian transactions, amounts in Algerian dinars."""
+    if n_rows < 1 or not 0 <= missing_rate < 1:
+        raise ValueError("Require n_rows >= 1 and 0 <= missing_rate < 1")
+    rng = np.random.default_rng(seed)
+    base = pd.Timestamp("2025-01-01")
+    df = pd.DataFrame({
+        "id_client": rng.integers(1, 50_000, n_rows),
+        "wilaya": rng.choice(["Alger", "Oran", "Setif", "Constantine", "Annaba", "Batna"], n_rows),
+        "categorie": rng.choice(["Alimentation", "Electronique", "Vetements", "Transport"], n_rows),
+        "montant_da": np.round(rng.lognormal(7.5, .9, n_rows), 2),
+        "date": base + pd.to_timedelta(rng.integers(0, 365*86400, n_rows), unit="s"),
+    })
+    miss = rng.random(n_rows) < missing_rate
+    df.loc[miss, "montant_da"] = np.nan
+    return df
+
+
+def csv_chunk_summary(path: str | Path, *, amount_column="montant_da",
+                      group_by="wilaya", chunksize=100_000) -> pd.DataFrame:
+    """Streaming count/sum/mean by category; never loads the whole CSV."""
+    if chunksize < 1:
+        raise ValueError("chunksize must be positive")
+    usecols = [amount_column] if group_by is None else [group_by, amount_column]
+    sums: dict = {}
+    counts: dict = {}
+    for chunk in pd.read_csv(path, usecols=usecols, chunksize=chunksize):
+        chunk[amount_column] = pd.to_numeric(chunk[amount_column], errors="coerce")
+        if group_by is None:
+            pairs = [("ALL", (int(chunk[amount_column].count()),
+                               float(chunk[amount_column].sum())))]
+        else:
+            group = chunk.groupby(group_by, dropna=False)[amount_column].agg(["count", "sum"])
+            pairs = [(k, (int(row["count"]), float(row["sum"]))) for k, row in group.iterrows()]
+        for key, (count, total) in pairs:
+            # Use a stable string key also for missing group values.
+            k = "<MISSING>" if pd.isna(key) else str(key)
+            sums[k] = sums.get(k, 0.0) + total
+            counts[k] = counts.get(k, 0) + count
+    result = pd.DataFrame({"group": list(sums),
+                           "count": [counts[k] for k in sums],
+                           "sum": [sums[k] for k in sums]})
+    if result.empty:
+        return pd.DataFrame(columns=["count", "sum", "mean"],
+                            index=pd.Index([], name=group_by or "group"))
+    result["mean"] = result["sum"] / result["count"].replace(0, np.nan)
+    return result.set_index("group").rename_axis(group_by or "group").sort_values("sum", ascending=False)
+
+
+def csv_chunk_missing(path: str | Path, *, chunksize=100_000) -> pd.DataFrame:
+    """Count missing observations over all CSV chunks."""
+    if chunksize < 1:
+        raise ValueError("chunksize must be positive")
+    total = 0
+    missing = None
+    for chunk in pd.read_csv(path, chunksize=chunksize):
+        total += len(chunk)
+        counts = chunk.isna().sum()
+        missing = counts if missing is None else missing.add(counts, fill_value=0)
+    if missing is None:
+        return pd.DataFrame(columns=["count", "percent"])
+    return pd.DataFrame({"count": missing.astype(int), "percent": missing / total * 100})
+
+
+def csv_chunk_monthly(path: str | Path, *, date_column="date",
+                      amount_column="montant_da", chunksize=100_000) -> pd.Series:
+    if chunksize < 1:
+        raise ValueError("chunksize must be positive")
+    sums = {}
+    for chunk in pd.read_csv(path, usecols=[date_column, amount_column], chunksize=chunksize):
+        dates = pd.to_datetime(chunk[date_column], errors="coerce")
+        amounts = pd.to_numeric(chunk[amount_column], errors="coerce")
+        grouped = amounts.groupby(dates.dt.to_period("M")).sum()
+        for period, value in grouped.items():
+            if pd.notna(period):
+                sums[period] = sums.get(period, 0.0) + float(value)
+    result = pd.Series(sums, dtype=float, name="total_da").sort_index()
+    result.index.name = "month"
+    return result
+
+
+def map_reduce_word_count(documents, *, min_length=2) -> dict[str, int]:
+    """Small local MapReduce illustration, not an Apache Spark implementation."""
+    if min_length < 1:
+        raise ValueError("min_length must be >= 1")
+    mapped = (Counter(w.lower() for w in re.findall(r"\b\w+\b", str(doc), flags=re.UNICODE)
+                      if len(w) >= min_length) for doc in documents)
+    reduced: Counter = Counter()
+    for part in mapped:
+        reduced.update(part)
+    return dict(reduced.most_common())
+
+
+def json_documents_to_table(documents: list[dict]) -> pd.DataFrame:
+    """Flatten document-style JSON as an introduction to NoSQL data."""
+    return pd.json_normalize(documents)

khaled_dsml/cli.py

@@ -0,0 +1,36 @@
+"""Command line interface: demo, inspect, and quiz."""
+from __future__ import annotations
+
+import argparse
+import json
+from .bigdata import generate_transactions
+from .quality import quality_report
+from .quizzes import get_questions
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="khaled-dsml", description="KhaledDSML educational toolkit")
+    commands = parser.add_subparsers(dest="command", required=True)
+    demo = commands.add_parser("demo", help="Run a synthetic dataset example")
+    demo.add_argument("--rows", type=int, default=1000)
+    inspect = commands.add_parser("inspect", help="Inspect a CSV file (loads it into RAM)")
+    inspect.add_argument("csv")
+    quiz = commands.add_parser("quiz", help="Print questions without exposing the solutions")
+    quiz.add_argument("--course", choices=["all", "data_science", "machine_learning", "big_data"], default="all")
+    quiz.add_argument("--count", type=int, default=5)
+    args = parser.parse_args(argv)
+    if args.command == "demo":
+        df = generate_transactions(n_rows=args.rows)
+        print(json.dumps(quality_report(df), ensure_ascii=False, indent=2))
+    elif args.command == "inspect":
+        import pandas as pd
+        print(json.dumps(quality_report(pd.read_csv(args.csv)), ensure_ascii=False, indent=2))
+    else:
+        for q in get_questions(args.course, args.count):
+            print(f"\n{q.id}: {q.question}")
+            for label, choice in zip("ABCD", q.choices):
+                print(f"  {label}. {choice}")
+
+
+if __name__ == "__main__":
+    main()

khaled_dsml/distributed.py

@@ -0,0 +1,32 @@
+"""Optional adapters for Dask and Spark; real engines required when used."""
+from __future__ import annotations
+
+
+def dask_csv_summary(path, *, group_by="wilaya", amount_column="montant_da",
+                     blocksize="64MB"):
+    """Lazy distributed CSV computation; returns a small pandas result table."""
+    try:
+        import dask.dataframe as dd
+    except ImportError as exc:
+        raise ImportError('Install with: pip install "khaled-dsml[dask]"') from exc
+    frame = dd.read_csv(str(path), blocksize=blocksize)
+    result = frame.groupby(group_by)[amount_column].agg(["count", "sum", "mean"])
+    return result.compute().sort_values("sum", ascending=False)
+
+
+def spark_group_summary(spark_df, *, group_by="wilaya", amount_column="montant_da"):
+    """Return aggregated *Spark DataFrame* without collecting it to driver RAM."""
+    try:
+        from pyspark.sql import functions as F
+        from pyspark.sql import DataFrame as SparkDataFrame
+    except ImportError as exc:
+        raise ImportError('Install with: pip install "khaled-dsml[spark]"; Java is also required') from exc
+    if not isinstance(spark_df, SparkDataFrame):
+        raise TypeError("spark_df must be a pyspark.sql.DataFrame")
+    if group_by not in spark_df.columns or amount_column not in spark_df.columns:
+        raise KeyError("Requested columns are absent")
+    return spark_df.groupBy(group_by).agg(
+        F.count(amount_column).alias("count"),
+        F.sum(amount_column).alias("sum"),
+        F.avg(amount_column).alias("mean"),
+    ).orderBy(F.col("sum").desc())

khaled_dsml/econometrics.py

@@ -0,0 +1,79 @@
+"""Optional statsmodels diagnostics for quantitative economics."""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from ._results import make_result, TestResult
+
+
+def _require_statsmodels():
+    try:
+        import statsmodels.api as sm
+    except ImportError as exc:
+        raise ImportError('Install with: pip install "khaled-dsml[econometrics]"') from exc
+    return sm
+
+
+def adf_test(series, alpha=0.05) -> TestResult:
+    sm = _require_statsmodels()
+    from statsmodels.tsa.stattools import adfuller
+    x = pd.to_numeric(pd.Series(series), errors="raise").dropna().to_numpy(dtype=float)
+    if len(x) < 10:
+        raise ValueError("Need at least 10 observations for this educational ADF check")
+    stat, p, *_ = adfuller(x, autolag="AIC")
+    return make_result("Augmented Dickey-Fuller", stat, p, len(x), alpha,
+                       "توجد جذور وحدة (السلسلة غير مستقرة بالمعنى المختبر).",
+                       "اختيار المكون الحتمي وطول الإبطاء يؤثر في الاستنتاج.")
+
+
+def vif_table(frame: pd.DataFrame) -> pd.DataFrame:
+    sm = _require_statsmodels()
+    from statsmodels.stats.outliers_influence import variance_inflation_factor
+    if not isinstance(frame, pd.DataFrame) or frame.shape[1] < 2:
+        raise ValueError("Provide numeric DataFrame with at least 2 features")
+    arr = frame.to_numpy(dtype=float)
+    if not np.all(np.isfinite(arr)):
+        raise ValueError("VIF requires complete finite numeric data")
+    design = sm.add_constant(arr, has_constant="add")
+    values = [float(variance_inflation_factor(design, i + 1)) for i in range(arr.shape[1])]
+    return pd.DataFrame({"feature": frame.columns.tolist(), "vif": values})
+
+
+def breusch_pagan_test(residuals, features, alpha=.05) -> TestResult:
+    sm = _require_statsmodels()
+    from statsmodels.stats.diagnostic import het_breuschpagan
+    residuals = np.asarray(residuals, dtype=float)
+    x = np.asarray(features, dtype=float)
+    if x.ndim != 2 or len(residuals) != len(x) or not np.all(np.isfinite(x)):
+        raise ValueError("Features must be a finite 2D matrix aligned with residuals")
+    if x.shape[1] < 1 or len(x) <= x.shape[1] + 1:
+        raise ValueError("Insufficient observations for Breusch-Pagan test")
+    design = sm.add_constant(x, has_constant="add")
+    stat, p, *_ = het_breuschpagan(residuals, design)
+    return make_result("Breusch-Pagan", stat, p, len(residuals), alpha,
+                       "تباين الأخطاء ثابت.")
+
+
+def ljung_box_test(series, lags=10, alpha=0.05) -> TestResult:
+    sm = _require_statsmodels()
+    from statsmodels.stats.diagnostic import acorr_ljungbox
+    x = pd.to_numeric(pd.Series(series), errors="raise").dropna().to_numpy(dtype=float)
+    if lags < 1 or len(x) <= lags:
+        raise ValueError("Need 1 <= lags < number of observations")
+    r = acorr_ljungbox(x, lags=[lags], return_df=True)
+    return make_result("Ljung-Box", r["lb_stat"].iloc[0], r["lb_pvalue"].iloc[0],
+                       len(x), alpha, "لا يوجد ارتباط ذاتي حتى عدد الإبطاءات المختار.")
+
+
+def cronbach_alpha(items: pd.DataFrame) -> float:
+    """Internal-consistency coefficient; no claim of unidimensionality."""
+    if not isinstance(items, pd.DataFrame) or items.shape[1] < 2:
+        raise ValueError("Need >=2 item columns")
+    x = items.astype(float).dropna().to_numpy()
+    if len(x) < 2:
+        raise ValueError("Need >=2 complete cases")
+    k = x.shape[1]
+    total = np.var(x.sum(axis=1), ddof=1)
+    if total <= 0:
+        raise ValueError("Total-score variance is zero")
+    return float((k / (k - 1)) * (1 - np.var(x, axis=0, ddof=1).sum() / total))

khaled_dsml/ml.py

@@ -0,0 +1,155 @@
+"""Leakage-aware ML workflows and metrics for classroom use."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import numpy as np
+import pandas as pd
+from sklearn.cluster import AgglomerativeClustering, KMeans
+from sklearn.compose import ColumnTransformer
+from sklearn.decomposition import PCA
+from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import Lasso, LinearRegression, LogisticRegression, Ridge
+from sklearn.metrics import (accuracy_score, confusion_matrix, f1_score,
+    mean_absolute_error, mean_squared_error, precision_score, recall_score,
+    r2_score, silhouette_score, davies_bouldin_score)
+from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score, train_test_split
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
+
+
+def split_data(X, y, *, test_size=.2, random_state=42, stratify=False):
+    """Always split before computing any preprocessing statistics."""
+    return train_test_split(X, y, test_size=test_size, random_state=random_state,
+                            stratify=y if stratify else None)
+
+
+def _preprocessor(X):
+    if isinstance(X, pd.DataFrame):
+        numeric = X.select_dtypes(include="number").columns.tolist()
+        categorical = [c for c in X.columns if c not in numeric]
+        transformers = []
+        if numeric:
+            transformers.append(("numeric", Pipeline([
+                ("impute", SimpleImputer(strategy="median")),
+                ("scale", StandardScaler()),
+            ]), numeric))
+        if categorical:
+            transformers.append(("categorical", Pipeline([
+                ("impute", SimpleImputer(strategy="most_frequent")),
+                ("encode", OneHotEncoder(handle_unknown="ignore")),
+            ]), categorical))
+        if not transformers:
+            raise ValueError("X must contain at least one feature")
+        return ColumnTransformer(transformers)
+    return Pipeline([("impute", SimpleImputer(strategy="median")),
+                     ("scale", StandardScaler())])
+
+
+def make_regressor(X, algorithm="linear", random_state=42, **kwargs) -> Pipeline:
+    """Unfitted pipeline. Fit on training data only."""
+    algorithms = {
+        "linear": LinearRegression,
+        "ridge": Ridge,
+        "lasso": Lasso,
+        "decision_tree": DecisionTreeRegressor,
+        "random_forest": RandomForestRegressor,
+    }
+    if algorithm not in algorithms:
+        raise ValueError("algorithm must be: " + ", ".join(algorithms))
+    cls = algorithms[algorithm]
+    if algorithm in ("decision_tree", "random_forest"):
+        kwargs.setdefault("random_state", random_state)
+    return Pipeline([("preprocess", _preprocessor(X)), ("model", cls(**kwargs))])
+
+
+def make_classifier(X, algorithm="logistic", random_state=42, **kwargs) -> Pipeline:
+    """Unfitted pipeline with imputation and categoricals inside CV folds."""
+    algorithms = {
+        "logistic": LogisticRegression,
+        "knn": KNeighborsClassifier,
+        "decision_tree": DecisionTreeClassifier,
+        "random_forest": RandomForestClassifier,
+    }
+    if algorithm not in algorithms:
+        raise ValueError("algorithm must be: " + ", ".join(algorithms))
+    cls = algorithms[algorithm]
+    if algorithm in ("logistic", "decision_tree", "random_forest"):
+        kwargs.setdefault("random_state", random_state)
+    if algorithm == "logistic":
+        kwargs.setdefault("max_iter", 1000)
+    return Pipeline([("preprocess", _preprocessor(X)), ("model", cls(**kwargs))])
+
+
+def regression_metrics(y_true, y_pred) -> dict:
+    return {"mae": float(mean_absolute_error(y_true, y_pred)),
+            "mse": float(mean_squared_error(y_true, y_pred)),
+            "rmse": float(np.sqrt(mean_squared_error(y_true, y_pred))),
+            "r2": float(r2_score(y_true, y_pred))}
+
+
+def classification_metrics(y_true, y_pred) -> dict:
+    """Macro averaging treats every class equally, including minority classes."""
+    return {"accuracy": float(accuracy_score(y_true, y_pred)),
+            "precision_macro": float(precision_score(y_true, y_pred, average="macro", zero_division=0)),
+            "recall_macro": float(recall_score(y_true, y_pred, average="macro", zero_division=0)),
+            "f1_macro": float(f1_score(y_true, y_pred, average="macro", zero_division=0)),
+            "confusion_matrix": confusion_matrix(y_true, y_pred).tolist()}
+
+
+def cross_validation_summary(estimator, X, y, *, task="regression", folds=5,
+                             random_state=42, scoring=None) -> dict:
+    if folds < 2:
+        raise ValueError("folds must be >= 2")
+    if task not in ("regression", "classification"):
+        raise ValueError("task must be regression or classification")
+    cv = (KFold(n_splits=folds, shuffle=True, random_state=random_state)
+          if task == "regression" else
+          StratifiedKFold(n_splits=folds, shuffle=True, random_state=random_state))
+    score = scoring or ("r2" if task == "regression" else "f1_macro")
+    values = cross_val_score(estimator, X, y, cv=cv, scoring=score)
+    return {"scoring": score, "fold_scores": values.tolist(),
+            "mean": float(np.mean(values)), "std": float(np.std(values))}
+
+
+@dataclass
+class ClusterResult:
+    labels: np.ndarray
+    silhouette: float | None
+    davies_bouldin: float | None
+    model: object
+
+
+def cluster_data(X, *, method="kmeans", n_clusters=3, random_state=42,
+                 scale=True) -> ClusterResult:
+    x = np.asarray(X, dtype=float)
+    if x.ndim != 2 or len(x) < 2 or not np.all(np.isfinite(x)):
+        raise ValueError("Clustering requires a finite numeric 2D matrix")
+    if not 2 <= n_clusters <= len(x):
+        raise ValueError("Need 2 <= n_clusters <= number of rows")
+    data = StandardScaler().fit_transform(x) if scale else x
+    if method == "kmeans":
+        model = KMeans(n_clusters=n_clusters, random_state=random_state, n_init=10)
+    elif method == "hierarchical":
+        model = AgglomerativeClustering(n_clusters=n_clusters)
+    else:
+        raise ValueError("method must be kmeans or hierarchical")
+    labels = model.fit_predict(data)
+    n_unique = len(np.unique(labels))
+    silhouette = (float(silhouette_score(data, labels))
+                  if 2 <= n_unique < len(x) else None)
+    db = float(davies_bouldin_score(data, labels)) if 2 <= n_unique < len(x) else None
+    return ClusterResult(labels, silhouette, db, model)
+
+
+def pca_summary(X, n_components=2, *, scale=True) -> dict:
+    x = np.asarray(X, dtype=float)
+    if x.ndim != 2 or not np.all(np.isfinite(x)):
+        raise ValueError("PCA needs complete finite numeric 2D data")
+    data = StandardScaler().fit_transform(x) if scale else x
+    model = PCA(n_components=n_components)
+    components = model.fit_transform(data)
+    return {"scores": components, "explained_variance_ratio": model.explained_variance_ratio_.tolist(),
+            "components": model.components_, "model": model}

khaled_dsml/plots.py

@@ -0,0 +1,41 @@
+"""Optional classroom figures; matplotlib is installed through [plots]."""
+from __future__ import annotations
+
+
+def _plt():
+    try:
+        import matplotlib.pyplot as plt
+    except ImportError as exc:
+        raise ImportError('Install with: pip install "khaled-dsml[plots]"') from exc
+    return plt
+
+
+def plot_missingness(df):
+    plt = _plt()
+    fig, ax = plt.subplots()
+    (100 * df.isna().mean()).plot.bar(ax=ax)
+    ax.set(ylabel="Missing values (%)", title="Missingness by column")
+    fig.tight_layout()
+    return fig, ax
+
+
+def plot_monthly(monthly):
+    plt = _plt()
+    fig, ax = plt.subplots()
+    monthly.plot(ax=ax)
+    ax.set(ylabel="Total (DA)", title="Monthly transaction amounts")
+    fig.tight_layout()
+    return fig, ax
+
+
+def plot_confusion(confusion):
+    import numpy as np
+    plt = _plt()
+    cm = np.asarray(confusion)
+    fig, ax = plt.subplots()
+    ax.imshow(cm)
+    for (i, j), value in np.ndenumerate(cm):
+        ax.text(j, i, str(value), ha="center", va="center")
+    ax.set(xlabel="Predicted", ylabel="Actual", title="Confusion matrix")
+    fig.tight_layout()
+    return fig, ax

khaled_dsml/quality.py

@@ -0,0 +1,109 @@
+"""Data inspection, cleaning helpers and simple distribution drift diagnostics."""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+from ._results import make_result, TestResult
+
+
+def quality_report(df: pd.DataFrame) -> dict:
+    """Return shape, missing cells, duplicate rows, dtypes and memory usage."""
+    _require_frame(df)
+    return {
+        "rows": len(df), "columns": len(df.columns),
+        "missing_cells": int(df.isna().sum().sum()),
+        "missing_by_column": df.isna().sum().astype(int).to_dict(),
+        "duplicate_rows": int(df.duplicated().sum()),
+        "dtypes": {k: str(v) for k, v in df.dtypes.items()},
+        "memory_mb": round(float(df.memory_usage(deep=True).sum()) / 1e6, 4),
+    }
+
+
+def missingness_report(df: pd.DataFrame) -> pd.DataFrame:
+    _require_frame(df)
+    result = pd.DataFrame({"count": df.isna().sum()})
+    result["percent"] = (100 * result["count"] / len(df)) if len(df) else 0.0
+    return result
+
+
+def _require_frame(df):
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("Expected a pandas DataFrame")
+
+
+def iqr_outliers(data: pd.DataFrame, column: str, multiplier: float = 1.5) -> pd.Series:
+    """Boolean mask of outliers; NaNs are False. Returns original row index."""
+    _require_frame(data)
+    if multiplier <= 0:
+        raise ValueError("multiplier must be positive")
+    x = pd.to_numeric(data[column], errors="raise")
+    if x.notna().sum() == 0:
+        raise ValueError("Column has no observed values")
+    q1, q3 = x.quantile([.25, .75])
+    spread = q3 - q1
+    return ((x < q1 - multiplier * spread) | (x > q3 + multiplier * spread)).fillna(False)
+
+
+def zscore_outliers(data: pd.DataFrame, column: str, threshold: float = 3.0) -> pd.Series:
+    _require_frame(data)
+    if threshold <= 0:
+        raise ValueError("threshold must be positive")
+    x = pd.to_numeric(data[column], errors="raise")
+    std = x.std(ddof=0)
+    if not pd.notna(std) or std == 0:
+        return pd.Series(False, index=data.index)
+    return ((x - x.mean()).abs() / std > threshold).fillna(False)
+
+
+def fill_missing_numeric(df: pd.DataFrame, strategy: str = "median") -> pd.DataFrame:
+    """Fill numeric missing values; return a copy. For ML use train-only imputation."""
+    _require_frame(df)
+    if strategy not in ("median", "mean", "zero"):
+        raise ValueError("strategy must be median, mean or zero")
+    out = df.copy()
+    for c in out.select_dtypes(include="number").columns:
+        if out[c].notna().sum() == 0:
+            continue
+        value = {"median": out[c].median, "mean": out[c].mean, "zero": lambda: 0}[strategy]()
+        out[c] = out[c].fillna(value)
+    return out
+
+
+def memory_report(df: pd.DataFrame) -> pd.DataFrame:
+    _require_frame(df)
+    usage = df.memory_usage(deep=True, index=False)
+    return pd.DataFrame({"bytes": usage, "megabytes": usage / 1e6}).sort_values("bytes", ascending=False)
+
+
+def optimize_memory(df: pd.DataFrame, *, category_ratio: float = .5,
+                    allow_float_downcast: bool = False) -> pd.DataFrame:
+    """Downcast ints, optionally float64, and low-cardinality text columns.
+
+    Float downcasting can be lossy and is disabled by default.
+    Original dataframe is not changed.
+    """
+    _require_frame(df)
+    if not 0 < category_ratio <= 1:
+        raise ValueError("category_ratio must be in (0, 1]")
+    out = df.copy()
+    for c in out.columns:
+        s = out[c]
+        if pd.api.types.is_integer_dtype(s.dtype):
+            out[c] = pd.to_numeric(s, downcast="integer")
+        elif allow_float_downcast and pd.api.types.is_float_dtype(s.dtype):
+            out[c] = pd.to_numeric(s, downcast="float")
+        elif (pd.api.types.is_object_dtype(s.dtype) or pd.api.types.is_string_dtype(s.dtype)):
+            observed = s.dropna()
+            if len(observed) and observed.nunique() / len(observed) <= category_ratio:
+                out[c] = s.astype("category")
+    return out
+
+
+def distribution_drift_test(reference, current, alpha=.05) -> TestResult:
+    from .statistics import ks_two_sample_test
+    result = ks_two_sample_test(reference, current, alpha=alpha)
+    return TestResult("Distribution drift / KS", result.statistic, result.p_value,
+                      result.alpha, result.reject_null, result.interpretation_ar,
+                      result.null_hypothesis_ar, result.sample_size,
+                      "اختبار اختلاف التوزيعات وليس اختبارًا للسببية.")

khaled_dsml/quizzes.py

@@ -0,0 +1,62 @@
+"""Optional classroom multiple-choice quizzes with reproducible selection."""
+from __future__ import annotations
+from dataclasses import dataclass
+from random import Random
+
+
+@dataclass(frozen=True)
+class Question:
+    id: str
+    course: str
+    question: str
+    choices: tuple[str, str, str, str]
+    answer: str
+    explanation_ar: str
+
+
+BANK = [
+    Question("DS01", "data_science", "ما المقصود بوحدة المشاهدة؟", ("سطر يمثل حالة", "اسم الجدول", "نوع الرسم", "ملف البرنامج"), "A", "المشاهدة هي الحالة التي يمثلها صف الجدول."),
+    Question("DS02", "data_science", "أي مقياس أقل تأثرًا بالقيم المتطرفة؟", ("المتوسط", "الوسيط", "المدى", "التباين"), "B", "الوسيط مقاوم نسبيًا للقيم المتطرفة."),
+    Question("DS03", "data_science", "ما المشكلة الأساسية في حذف جميع القيم المفقودة؟", ("زيادة حجم البيانات", "فقد عينة واحتمال التحيز", "تحسين التمثيل دائمًا", "تقليل الدقة العددية فقط"), "B", "الحذف قد يقلص العينة ويسبب تحيزًا."),
+    Question("DS04", "data_science", "أي اختبار شائع لفحص اعتدالية توزيع البيانات؟", ("Shapiro-Wilk", "K-means", "F1", "MAE"), "A", "Shapiro-Wilk يختبر فرضية التوزيع الطبيعي."),
+    Question("DS05", "data_science", "ماذا يعني الارتباط الموجب؟", ("السببية مؤكدة", "اتجاه التغير المشترك غالبًا واحد", "عدم وجود علاقة", "تساوي المتوسطات"), "B", "الارتباط لا يثبت السببية."),
+    Question("DS06", "data_science", "ماذا يعني عدم رفض H0؟", ("أثبتنا صحة H0", "لا توجد أدلة كافية للرفض", "H1 صحيحة حتمًا", "حجم العينة كبير"), "B", "عدم الرفض لا يثبت الفرضية الصفرية."),
+    Question("DS07", "data_science", "الهدف من IQR في التنظيف؟", ("قياس ارتباط خطي", "كشف قيم بعيدة وفق الربيعات", "ترتيب الأشهر", "تقسيم التدريب"), "B", "قاعدة IQR إحدى طرق الإشارة إلى الشواذ."),
+    Question("DS08", "data_science", "ماذا تتطلب معالجة البيانات الأخلاقية؟", ("نشر بيانات العملاء", "حماية الخصوصية وتوثيق المصدر", "حذف مصدر البيانات", "إلغاء النسخ الاحتياطية"), "B", "إخفاء البيانات الحساسة وأخذ الموافقات من الأسس."),
+    Question("ML01", "machine_learning", "ما دور مجموعة الاختبار؟", ("ضبط كل المعاملات حتى أفضل نتيجة", "قياس التعميم النهائي", "تدريب النموذج", "حذف الفئات"), "B", "مجموعة الاختبار تعطي تقديرًا على بيانات غير مستخدمة بالتدريب/الضبط."),
+    Question("ML02", "machine_learning", "ما مثال على تسرب البيانات؟", ("التقييس قبل تقسيم كامل البيانات", "استخدام Pipeline داخل CV", "فصل البيانات أولًا", "تثبيت random_state"), "A", "التقييس قبل التقسيم ينقل معلومات الاختبار إلى التدريب."),
+    Question("ML03", "machine_learning", "أي مقياس يستخدم للتنبؤ العددي؟", ("Recall", "MAE", "Accuracy", "F1"), "B", "MAE متوسط الخطأ المطلق."),
+    Question("ML04", "machine_learning", "ما وظيفة Ridge؟", ("عقوبة L2", "تجميع غير خاضع للإشراف", "تشخيص بيانات نصية", "تحويل JSON"), "A", "Ridge يطبق تنظيم L2 في الانحدار."),
+    Question("ML05", "machine_learning", "أي خوارزمية تستخدم لتجميع عملاء دون تسميات؟", ("K-means", "LogisticRegression", "LinearRegression", "Welch t-test"), "A", "K-means تعلم غير خاضع للإشراف."),
+    Question("ML06", "machine_learning", "ما سبب استخدام F1 مع الفئات غير المتوازنة؟", ("يلغي تحيز البيانات", "يوفق بين Precision وRecall", "يقيس المبيعات", "يلغي الحاجة إلى الاختبار"), "B", "F1 متوسط توافقي للدقة الإيجابية والاسترجاع."),
+    Question("ML07", "machine_learning", "كيف تختبر عدة تقسيمات للتدريب والتقييم؟", ("Cross-validation", "Excel فقط", "إزالة المتغير الهدف", "تقريب الأرقام"), "A", "التحقق المتقاطع يعيد التقييم عبر طيات متعددة."),
+    Question("ML08", "machine_learning", "ما المقصود بـ PCA؟", ("اختبار قيمة مفقودة", "خفض الأبعاد بمركبات رئيسية", "تصنيف موجه", "توليد SQL"), "B", "PCA يستخرج مركبات تفسر تباينًا من البيانات."),
+    Question("BD01", "big_data", "ما المقصود بـ Volume في 5Vs؟", ("حجم البيانات", "دقتها", "مصدرها", "قيمتها الاقتصادية"), "A", "Volume يشير إلى حجم البيانات."),
+    Question("BD02", "big_data", "ماذا تفيد chunksize في read_csv؟", ("تحميل الملف كله دائمًا", "قراءة دفعات بدل الملف كاملًا", "إزالة الارتباط", "ترميز متغير الهدف"), "B", "chunksize يعيد أجزاء متعاقبة للمعالجة."),
+    Question("BD03", "big_data", "ما المقصود بـ MapReduce؟", ("رسم النقاط", "توزيع المعالجة ثم تجميع النتائج", "إعادة تسمية الأعمدة", "فحص الاعتدالية"), "B", "Map ثم Reduce من مفاهيم المعالجة الموزعة."),
+    Question("BD04", "big_data", "أي نوع تمثله مستندات JSON غالبًا؟", ("شبه منظمة", "صور خام", "غير قابلة للمعالجة", "بيانات مقاسة بالثواني"), "A", "JSON يستخدم بنية مفاتيح وقيم مرنة."),
+    Question("BD05", "big_data", "كيف نقارن استهلاك الذاكرة لـ DataFrame؟", ("memory_usage(deep=True)", "shape وحده", "head", "unique"), "A", "deep=True يحسب أيضًا الذاكرة المخزنة في كائنات النصوص."),
+    Question("BD06", "big_data", "ما الفرق بين SQL وNoSQL؟", ("NoSQL دائمًا أسرع", "NoSQL تدعم نماذج تخزين متعددة", "SQL لا تدعم الفهارس", "SQL تعني الصور"), "B", "NoSQL تشمل المستندات والمفاتيح/القيم وأشكالًا أخرى."),
+    Question("BD07", "big_data", "ما خطر استخلاص استنتاجات عامة من بيانات اصطناعية؟", ("لا توجد مخاطر", "قد لا تمثل الواقع الاقتصادي", "تزيد السرعة", "تزيد عدد الأعمدة"), "B", "بيانات المحاكاة توضح المنهج لكنها ليست دليلًا واقعيًا."),
+    Question("BD08", "big_data", "ما الاسم الشائع للتنبؤ الآني بمؤشر اقتصادي؟", ("Nowcasting", "OneHot", "Shapiro", "KNN"), "A", "Nowcasting تقدير للحالة الحالية قبل إصدار الإحصاءات النهائية."),
+]
+
+
+def get_questions(course="all", limit=10, seed=42) -> list[Question]:
+    courses = ("all", "data_science", "machine_learning", "big_data")
+    if course not in courses:
+        raise ValueError("course must be: " + ", ".join(courses))
+    if limit < 1:
+        raise ValueError("limit must be positive")
+    questions = [q for q in BANK if course == "all" or q.course == course]
+    return Random(seed).sample(questions, min(limit, len(questions)))
+
+
+def grade_quiz(questions: list[Question], answers: dict[str, str]) -> dict:
+    """Grade a list of questions. Answers map question IDs to A/B/C/D."""
+    results = [{"id": q.id, "correct": answers.get(q.id, "").strip().upper() == q.answer,
+                "answer": q.answer, "explanation_ar": q.explanation_ar} for q in questions]
+    count = sum(r["correct"] for r in results)
+    return {"correct": count, "total": len(results),
+            "percentage": round(count / len(results) * 100, 2) if results else 0,
+            "details": results}

khaled_dsml/sqltools.py

@@ -0,0 +1,35 @@
+"""SQLite demonstrations for TD2: SQL versus document-style data."""
+from __future__ import annotations
+import re
+import sqlite3
+from pathlib import Path
+import pandas as pd
+
+
+def _safe_name(name: str) -> str:
+    if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
+        raise ValueError("SQL identifier must be ASCII letters, digits or underscores")
+    return '"' + name + '"'
+
+
+def sqlite_import_frame(df: pd.DataFrame, database: str | Path,
+                        table="transactions", if_exists="replace") -> int:
+    """Store a pandas DataFrame in a local SQLite database."""
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("df must be a pandas DataFrame")
+    _safe_name(table)
+    if if_exists not in {"replace", "append", "fail"}:
+        raise ValueError("if_exists must be replace, append or fail")
+    with sqlite3.connect(str(database)) as conn:
+        count = df.to_sql(table, conn, if_exists=if_exists, index=False)
+    return int(count)
+
+
+def sqlite_group_summary(database: str | Path, *, table="transactions",
+                         group_by="wilaya", amount_column="montant_da") -> pd.DataFrame:
+    """Return count/sum/mean, computed by SQLite using a safe fixed SQL pattern."""
+    t, g, a = _safe_name(table), _safe_name(group_by), _safe_name(amount_column)
+    sql = (f"SELECT {g} AS category, COUNT({a}) AS count, SUM({a}) AS total, "
+           f"AVG({a}) AS mean FROM {t} GROUP BY {g} ORDER BY total DESC")
+    with sqlite3.connect(str(database)) as conn:
+        return pd.read_sql_query(sql, conn)

khaled_dsml/statistics.py

@@ -0,0 +1,162 @@
+"""Introductory inferential statistics. Each test explicitly reports H0."""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+from ._results import TestResult, make_result
+
+
+def _sample(x, minimum=2):
+    s = pd.to_numeric(pd.Series(x), errors="raise").dropna().to_numpy(dtype=float)
+    if s.size < minimum or not np.all(np.isfinite(s)):
+        raise ValueError(f"Need at least {minimum} finite numeric observations")
+    return s
+
+
+def shapiro_test(x, alpha=0.05) -> TestResult:
+    a = _sample(x, 3)
+    if np.ptp(a) == 0:
+        raise ValueError("Shapiro-Wilk is undefined for constant data")
+    result = stats.shapiro(a)
+    note = "قد تكون p-value غير دقيقة للعينات أكبر من 5000." if len(a) > 5000 else ""
+    return make_result("Shapiro-Wilk", result.statistic, result.pvalue, len(a), alpha,
+                       "تتبع البيانات توزيعًا طبيعيًا.", note)
+
+
+def jarque_bera_test(x, alpha=0.05) -> TestResult:
+    a = _sample(x, 3)
+    if np.ptp(a) == 0:
+        raise ValueError("Jarque-Bera is undefined for constant data")
+    r = stats.jarque_bera(a)
+    return make_result("Jarque-Bera", r.statistic, r.pvalue, len(a), alpha,
+                       "تتفق الالتواء والتفرطح مع التوزيع الطبيعي.")
+
+
+def normality_test(x, method="shapiro", alpha=0.05) -> TestResult:
+    """method='shapiro' or 'jarque_bera'."""
+    if method == "shapiro":
+        return shapiro_test(x, alpha)
+    if method == "jarque_bera":
+        return jarque_bera_test(x, alpha)
+    raise ValueError("method must be 'shapiro' or 'jarque_bera'")
+
+
+def independent_t_test(a, b, alpha=0.05, equal_var=False) -> TestResult:
+    x, y = _sample(a), _sample(b)
+    r = stats.ttest_ind(x, y, equal_var=equal_var)
+    name = "Student t-test" if equal_var else "Welch t-test"
+    return make_result(name, r.statistic, r.pvalue, len(x)+len(y), alpha,
+                       "متوسطا المجموعتين متساويان.")
+
+
+def paired_t_test(a, b, alpha=0.05) -> TestResult:
+    x, y = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
+    if x.shape != y.shape:
+        raise ValueError("Paired samples must have the same shape")
+    good = np.isfinite(x) & np.isfinite(y)
+    x, y = x[good], y[good]
+    if len(x) < 2:
+        raise ValueError("Need at least two valid pairs")
+    r = stats.ttest_rel(x, y)
+    return make_result("Paired t-test", r.statistic, r.pvalue, len(x), alpha,
+                       "متوسط الفروق بين القياسين يساوي صفرًا.")
+
+
+def mann_whitney_test(a, b, alpha=0.05) -> TestResult:
+    x, y = _sample(a), _sample(b)
+    r = stats.mannwhitneyu(x, y, alternative="two-sided")
+    return make_result("Mann-Whitney U", r.statistic, r.pvalue, len(x)+len(y), alpha,
+                       "توزيعا المجموعتين متماثلان.",
+                       "لا يُفسَّر تلقائيًا على أنه اختبار تساوي الوسيطين.")
+
+
+def wilcoxon_test(a, b, alpha=0.05) -> TestResult:
+    x, y = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
+    if x.shape != y.shape:
+        raise ValueError("Paired samples must have the same shape")
+    valid = np.isfinite(x) & np.isfinite(y)
+    x, y = x[valid], y[valid]
+    if len(x) < 2 or np.allclose(x-y, 0):
+        raise ValueError("Need at least two pairs and a nonzero difference")
+    r = stats.wilcoxon(x, y)
+    return make_result("Wilcoxon signed-rank", r.statistic, r.pvalue, len(x), alpha,
+                       "توزيع الفروق متناظر حول الصفر.")
+
+
+def anova_test(*groups, alpha=0.05) -> TestResult:
+    if len(groups) < 2:
+        raise ValueError("Provide at least two groups")
+    arrays = [_sample(group) for group in groups]
+    r = stats.f_oneway(*arrays)
+    return make_result("One-way ANOVA", r.statistic, r.pvalue,
+                       sum(map(len, arrays)), alpha, "متوسطات كل المجموعات متساوية.")
+
+
+def kruskal_test(*groups, alpha=0.05) -> TestResult:
+    if len(groups) < 2:
+        raise ValueError("Provide at least two groups")
+    arrays = [_sample(group) for group in groups]
+    r = stats.kruskal(*arrays)
+    return make_result("Kruskal-Wallis", r.statistic, r.pvalue,
+                       sum(map(len, arrays)), alpha, "توزيعات المجموعات متماثلة.")
+
+
+def levene_test(*groups, alpha=0.05) -> TestResult:
+    if len(groups) < 2:
+        raise ValueError("Provide at least two groups")
+    arrays = [_sample(group) for group in groups]
+    r = stats.levene(*arrays, center="median")
+    return make_result("Brown-Forsythe (Levene)", r.statistic, r.pvalue,
+                       sum(map(len, arrays)), alpha, "تباينات المجموعات متساوية.")
+
+
+def chi_square_test(contingency_table, alpha=0.05) -> TestResult:
+    table = np.asarray(contingency_table, dtype=float)
+    if table.ndim != 2 or min(table.shape) < 2 or not np.all(np.isfinite(table)):
+        raise ValueError("Expected a finite 2D contingency table >= 2x2")
+    if np.any(table < 0) or np.any(table.sum(axis=0) == 0) or np.any(table.sum(axis=1) == 0):
+        raise ValueError("Table needs nonnegative counts and nonzero margins")
+    r = stats.chi2_contingency(table, correction=False)
+    notes = "راجع التكرارات المتوقعة؛ قد يلزم اختبار دقيق عند ندرتها."
+    return make_result("Pearson Chi-square independence", r.statistic, r.pvalue,
+                       int(table.sum()), alpha, "المتغيران الفئويان مستقلان.", notes)
+
+
+def pearson_test(a, b, alpha=0.05) -> TestResult:
+    x, y = _paired(a, b)
+    r = stats.pearsonr(x, y)
+    return make_result("Pearson correlation", r.statistic, r.pvalue,
+                       len(x), alpha, "الارتباط الخطي السكاني يساوي صفرًا.")
+
+
+def spearman_test(a, b, alpha=0.05) -> TestResult:
+    x, y = _paired(a, b)
+    r = stats.spearmanr(x, y)
+    return make_result("Spearman correlation", r.statistic, r.pvalue,
+                       len(x), alpha, "الارتباط الرتبي السكاني يساوي صفرًا.")
+
+
+def _paired(a, b):
+    x, y = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
+    if x.shape != y.shape or x.ndim != 1:
+        raise ValueError("Need aligned one-dimensional samples")
+    valid = np.isfinite(x) & np.isfinite(y)
+    x, y = x[valid], y[valid]
+    if len(x) < 3 or np.std(x) == 0 or np.std(y) == 0:
+        raise ValueError("Need at least 3 variable paired observations")
+    return x, y
+
+
+def ks_two_sample_test(a, b, alpha=0.05) -> TestResult:
+    x, y = _sample(a), _sample(b)
+    r = stats.ks_2samp(x, y)
+    return make_result("Two-sample Kolmogorov-Smirnov", r.statistic, r.pvalue,
+                       len(x)+len(y), alpha, "المجموعتان تتبعان التوزيع نفسه.")
+
+
+def descriptive_summary(data: pd.DataFrame) -> pd.DataFrame:
+    """Descriptive statistics for numeric columns only."""
+    if not isinstance(data, pd.DataFrame):
+        raise TypeError("data must be a pandas DataFrame")
+    return data.describe(include="number").T

khaled_dsml/timeseries.py

@@ -0,0 +1,22 @@
+"""Time-safe lagged features for classroom forecasting exercises."""
+from __future__ import annotations
+import pandas as pd
+
+
+def lagged_features(series, *, lags=(1, 2, 3), rolling_window=None) -> pd.DataFrame:
+    """Create past-only features: lag t-n and optional mean up to t-1.
+
+    Input must already be sorted in ascending time order. No random splitting
+    should be used to validate temporal forecasts.
+    """
+    values = pd.Series(series)
+    if len(lags) < 1 or any(not isinstance(lag, int) or lag < 1 for lag in lags):
+        raise ValueError("lags must be positive integers")
+    out = pd.DataFrame(index=values.index)
+    for lag in lags:
+        out[f"lag_{lag}"] = values.shift(lag)
+    if rolling_window is not None:
+        if not isinstance(rolling_window, int) or rolling_window < 2:
+            raise ValueError("rolling_window must be an integer >= 2")
+        out[f"rolling_mean_{rolling_window}"] = values.shift(1).rolling(rolling_window).mean()
+    return out

khaled_dsml-0.1.0.dist-info/METADATA

@@ -0,0 +1,188 @@
+Metadata-Version: 2.4
+Name: khaled-dsml
+Version: 0.1.0
+Summary: Educational data science, machine learning and big-data analysis toolkit with Arabic interpretations
+Author-email: Khaled Ali <ali-khaled@univ-eloued.dz>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/noble2013/KhaledDSML
+Project-URL: Repository, https://github.com/noble2013/KhaledDSML
+Project-URL: Issues, https://github.com/noble2013/KhaledDSML/issues
+Keywords: data-science,machine-learning,big-data,education,statistics,arabic
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas>=2.1
+Requires-Dist: scipy>=1.11
+Requires-Dist: scikit-learn>=1.3
+Provides-Extra: econometrics
+Requires-Dist: statsmodels>=0.14; extra == "econometrics"
+Provides-Extra: dask
+Requires-Dist: dask[dataframe]>=2024.1; extra == "dask"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.5; extra == "spark"
+Provides-Extra: plots
+Requires-Dist: matplotlib>=3.8; extra == "plots"
+Provides-Extra: notebooks
+Requires-Dist: jupyterlab>=4; extra == "notebooks"
+Requires-Dist: ipykernel>=6; extra == "notebooks"
+Requires-Dist: openpyxl>=3.1; extra == "notebooks"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: build>=1; extra == "dev"
+Requires-Dist: twine>=5; extra == "dev"
+Provides-Extra: all
+Requires-Dist: statsmodels>=0.14; extra == "all"
+Requires-Dist: matplotlib>=3.8; extra == "all"
+Requires-Dist: jupyterlab>=4; extra == "all"
+Requires-Dist: ipykernel>=6; extra == "all"
+Requires-Dist: openpyxl>=3.1; extra == "all"
+Dynamic: license-file
+
+# KhaledDSML — علم البيانات وتعلّم الآلة وتحليل البيانات الضخمة
+
+**الإعداد والتطوير: الأستاذ الدكتور خالد علي (Khaled Ali)**  
+**الجامعة:** جامعة الشهيد حمه لخضر – الوادي، الجزائر (University of El Oued, Algeria)  
+**الكلية:** كلية العلوم الاقتصادية والتجارية وعلوم التسيير (Faculty of Economic, Commercial and Management Sciences)  
+**البريد الإلكتروني المهني:** [ali-khaled@univ-eloued.dz](mailto:ali-khaled@univ-eloued.dz)  
+**الإصدار التجريبي: 0.1.0 — أكتوبر 2026**
+
+حزمة Python تعليمية للمقاييس التالية:
+
+- علم البيانات وتعلّم الآلة — السنة الثالثة ليسانس اقتصاد وتسيير المؤسسات.
+- تحليل البيانات الضخمة — السنة الثانية ماستر اقتصاد كمي.
+
+> تنبيه: تم تصميم المشروع وفق البرامج البيداغوجية التي أعدّها المؤلف؛ لا يُدَّعى تطابقه مع بطاقة وزارية رسمية دون مراجعتها. البيانات الاصطناعية للأغراض التعليمية فقط. الإحصاء الوصفي لا يثبت السببية، وعدم رفض الفرضية الصفرية لا يثبت صحتها.
+
+## لماذا هذه المكتبة؟
+
+- اختبارات فرضيات شائعة مع `statistic`، و`p_value`، وفرضية صفرية، وتفسير عربي.
+- فحص جودة البيانات واكتشاف القيم الشاذة وتحسين الذاكرة.
+- أنابيب تدريب آمنة من تسرب البيانات، وتقييم الانحدار والتصنيف والتجميع.
+- معالجة CSV بالأجزاء، وتجميع شهري وجغرافي، وتوليد مليون معاملة اصطناعية إذا طُلب ذلك.
+- 24 سؤال اختيار متعدد للتقويم التكويني مع التصحيح والتفسير.
+- دعم Windows وJupyterLab وGoogle Colab بعد تثبيت الاعتمادات.
+
+## التثبيت من المجلد المصدر — Windows PowerShell
+
+افتح PowerShell من داخل المجلد الذي يحتوي `pyproject.toml`:
+
+```powershell
+py -3.13 -m venv .venv
+.\.venv\Scripts\python.exe -m pip install --upgrade pip
+.\.venv\Scripts\python.exe -m pip install -e ".[all,dev]"
+.\.venv\Scripts\python.exe -m pytest
+.\.venv\Scripts\khaled-dsml.exe demo --rows 1000
+.\.venv\Scripts\khaled-dsml.exe quiz --course big_data --count 5
+```
+
+بعض بيئات Windows تستعمل `py` بدل `py -3.13` عند عدم توفر الفرع 3.13.
+
+*لاحقًا وبعد النشر الفعلي على PyPI فقط:*
+
+```powershell
+py -m pip install khaled-dsml
+```
+
+## أمثلة عملية
+
+### اختبار اعتدالية
+
+```python
+from khaled_dsml import normality_test
+result = normality_test([12, 13, 15, 17, 16, 14, 13, 18])
+print(result.interpretation_ar)
+print(result.p_value)
+```
+
+### كشف مشاكل البيانات
+
+```python
+from khaled_dsml import generate_transactions, quality_report, missingness_report
+transactions = generate_transactions(100_000)
+print(quality_report(transactions))
+print(missingness_report(transactions))
+```
+
+### تعلم الآلة (منع تسرب البيانات)
+
+```python
+from sklearn.datasets import make_regression
+from khaled_dsml import split_data, make_regressor, regression_metrics
+X, y = make_regression(n_samples=200, n_features=3, noise=3, random_state=42)
+X_train, X_test, y_train, y_test = split_data(X, y)
+model = make_regressor(X_train, "ridge", alpha=1.0)
+model.fit(X_train, y_train)
+print(regression_metrics(y_test, model.predict(X_test)))
+```
+
+### مليون معاملة والتجميع بالأجزاء
+
+```python
+from khaled_dsml import generate_transactions, csv_chunk_summary, csv_chunk_monthly
+transactions = generate_transactions(1_000_000)
+transactions.to_csv("transactions.csv", index=False)
+print(csv_chunk_summary("transactions.csv", group_by="wilaya"))
+print(csv_chunk_monthly("transactions.csv"))
+```
+
+`read_csv(chunksize=...)` لا يحتاج تحميل الملف بكامله. **لكن** توليد مليون صف في `generate_transactions` يحتفظ بها في الذاكرة قبل الحفظ؛ اختر عدد صفوف أصغر لجهاز بذاكرة محدودة.
+
+### أسئلة تصحيح آلي
+
+```python
+from khaled_dsml import get_questions, grade_quiz
+questions = get_questions("machine_learning", limit=3, seed=3)
+for question in questions:
+    print(question.id, question.question)
+answers = {q.id: "A" for q in questions}  # إجابات افتراضية
+print(grade_quiz(questions, answers))
+```
+
+## واجهات المكتبة
+
+| الوحدة | أمثلة الدوال |
+|---|---|
+| `statistics` | `normality_test`, `independent_t_test`, `paired_t_test`, `anova_test`, `kruskal_test`, `chi_square_test`, `pearson_test`, `spearman_test`, `levene_test`, `ks_two_sample_test` |
+| `quality` | `quality_report`, `missingness_report`, `iqr_outliers`, `fill_missing_numeric`, `optimize_memory`, `distribution_drift_test` |
+| `ml` | `make_regressor`, `make_classifier`, `cross_validation_summary`, `classification_metrics`, `regression_metrics`, `cluster_data`, `pca_summary` |
+| `bigdata` | `generate_transactions`, `csv_chunk_summary`, `csv_chunk_missing`, `csv_chunk_monthly`, `map_reduce_word_count`, `json_documents_to_table` |
+| `sqltools` | `sqlite_import_frame`, `sqlite_group_summary` (SQLite محلي لتجارب SQL) |
+| `timeseries` | `lagged_features` (خصائص مؤخرة دون تسرب للمستقبل) |
+| `distributed` | `dask_csv_summary`, `spark_group_summary` (تثبيت `[dask]` أو `[spark]` واستخدام محرك فعلي) |
+| `econometrics` | `adf_test`, `vif_table`, `breusch_pagan_test`, `ljung_box_test`, `cronbach_alpha` (تثبيت `econometrics`) |
+| `plots` | `plot_missingness`, `plot_monthly`, `plot_confusion` (تثبيت `plots`) |
+| `quizzes` | `get_questions`, `grade_quiz` |
+
+لخريطة المقياسين والتغطية التفصيلية، راجع [docs/COURSE_MAPPING_AR.md](docs/COURSE_MAPPING_AR.md). للحصول على أمثلة مكتملة راجع `examples/`.
+
+**لإنشاء المستودع ونشر الحزمة بأمان عبر GitHub Actions وTrusted Publishing (بدون حفظ مفتاح API)، راجع [دليل النشر العربي](docs/PUBLISH_AR.md).**
+
+## نشر الحزمة على TestPyPI وPyPI باستخدام GitHub Actions
+
+هذه المكتبة **غير منشورة بعد** على PyPI أو TestPyPI؛ لا تستخدم أمر `pip install khaled-dsml` من PyPI إلى حين نشرها.
+
+1. تابع نتائج الاختبارات والبناء في [GitHub Actions](https://github.com/noble2013/KhaledDSML/actions).
+2. أنشئ حسابًا منفصلًا على [TestPyPI](https://test.pypi.org/account/register/) وبيئة `testpypi` في `Settings → Environments` بمستودع GitHub.
+3. أضف في [TestPyPI Trusted Publishing](https://test.pypi.org/manage/account/publishing/) ناشرًا معلقًا للاسم `khaled-dsml` ومعلومات المستودع `noble2013/KhaledDSML`، وملف `testpypi.yml`، والبيئة `testpypi`.
+4. بعد نجاح البناء، نفذ `Actions → Publish to TestPyPI (manual) → Run workflow`.
+5. بعد نجاح التجربة، أنشئ البيئة `pypi` وناشرًا موثوقًا للاسم نفسه على [PyPI](https://pypi.org/manage/account/publishing/) باستخدام `release.yml` والبيئة `pypi`، ثم انشر GitHub Release يحمل الوسم `v0.1.0`.
+
+يوصى بحماية بيئة `pypi` بمراجعة يدوية. لا تضع كلمات مرور أو رموز API داخل GitHub. اقرأ [دليل النشر العربي](docs/PUBLISH_AR.md) للمزيد.
+
+## Notes (English)
+
+KhaledDSML is an **educational** Python toolkit; it wraps proven NumPy, pandas, SciPy and scikit-learn primitives rather than reinventing statistical algorithms. Optional statistics are backed by statsmodels. It is not a distributed processing engine: the MapReduce example runs locally; Optional Dask and Spark adapters are included, but need real engines and are not validated on a Spark cluster in version 0.1.0. Do not fit preprocessing on test data. Hypothesis-test interpretations should be checked against sampling design and assumptions. An MIT license covers this original wrapper code, not a claim of authorship over third-party algorithms.
+
+## حقوق الاستخدام
+
+الترخيص MIT. الاسم المقترح للنشر هو `khaled-dsml` (يلزم التحقق من توفره عند رفع الحزمة). كل الإضافات يمكن مراجعتها عبر اختبارات `pytest` قبل النشر. حقوق المنهجيات والخوارزميات الأساسية محفوظة لمؤلفيها ومشروعاتها الأصلية.

khaled_dsml-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+khaled_dsml/__init__.py,sha256=iRLVRN4MK57sdy9y2gEktx_p-TZAUL3-BxU_lbfKGk8,1026
+khaled_dsml/_results.py,sha256=ZIMEhQwUu7OxfLBp7FG9-IDinQOVoc23ys3QT8X9RnA,1357
+khaled_dsml/bigdata.py,sha256=8i1wUjgtRk3vZJu7B-MuBOzMVVTG5ifRCdHSQ7Tp3TE,4982
+khaled_dsml/cli.py,sha256=kXLfODk_fozZc2nSdDSusGpYHUIGZeXcxPXhSM49cNk,1557
+khaled_dsml/distributed.py,sha256=3yoe84VQUOyRJYR4i0HajcOn7t9PbG3zDkLSzlQthp8,1565
+khaled_dsml/econometrics.py,sha256=nMFLh2sC3K6xHbZPdSDISo1fbVSCugDlAIm3H4JqseE,3753
+khaled_dsml/ml.py,sha256=_9lPEl9K37vHZWMyi64MUdEaNwxeRtpgNdbUNZuQhGE,7144
+khaled_dsml/plots.py,sha256=o3Tvc-logeaZNtndQIVd8t4M84lbuH2pLSU33omexUE,1118
+khaled_dsml/quality.py,sha256=iO-7UH0aPdQD0tv91HXxjquNrLYlALRt5i1t13Jho6o,4513
+khaled_dsml/quizzes.py,sha256=6qUjsu7WuHHoPFSdFBcOdx_3BSRFUAM03-Q1RbmkCUM,8578
+khaled_dsml/sqltools.py,sha256=DMgh7iq_Q22BIyHMi6ZPHSFjVV3H9aOXzNdS22evOQg,1580
+khaled_dsml/statistics.py,sha256=jjMlLFw-br97xv762Q0gGb3LZGe4cNIVQdW1mdbtwiA,7200
+khaled_dsml/timeseries.py,sha256=CNl0xmPf8hUCafXAx4qJZnT_JKraRidjdoZ57GLwnM0,988
+khaled_dsml-0.1.0.dist-info/licenses/LICENSE,sha256=f7as3Fpf2T2Z9GAKUuCt7dzd2fdQCh8FldlmMGs3Hx4,1067
+khaled_dsml-0.1.0.dist-info/METADATA,sha256=GnWw6sPdwAzA8qHahFv6K3IShbTe10Y8GTJbZqqDrf4,11102
+khaled_dsml-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+khaled_dsml-0.1.0.dist-info/entry_points.txt,sha256=igTVGou4rpnn0Ik3oZKnB4REyvSzNh1QelHldW5Mqkk,53
+khaled_dsml-0.1.0.dist-info/top_level.txt,sha256=d9Gk3AokLxRE8eFeC417BYZBC3JmSBxWF1denSkt5fA,12
+khaled_dsml-0.1.0.dist-info/RECORD,,

khaled_dsml-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

khaled_dsml-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+khaled-dsml = khaled_dsml.cli:main

khaled_dsml-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Khaled Ali
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

khaled_dsml-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+khaled_dsml