libz 0.4__tar.gz → 0.6__tar.gz

{libz-0.4 → libz-0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: libz
-Version: 0.4
+Version: 0.6
 Summary: A lightweight Python utility toolkit.
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown

{libz-0.4 → libz-0.6}/libz/DLG/__init__.py

@@ -1,15 +1,15 @@
-from libz._core import register
-
-TITLES = {
-    1: "Deep feed forward (any CSV)",
-    2: "Regularization and optimization (any CSV)",
-    3: "Semi-supervised learning (any text CSV)",
-    4: "Multitasking (any CSV)",
-    5: "CNN-based handwriting recognition (any image folder / MNIST)",
-    6: "CNN - unsupervised feature learning (any image folder / digits)",
-    7: "Recurrent neural network (any text CSV / IMDB)",
-    8: "Recursive neural network (tree) (any text CSV / sample)",
-    9: "PCA vs linear autoencoder (any numeric CSV / synthetic)",
-}
-
-register(__name__, globals(), TITLES)
+from libz._core import register
+
+TITLES = {
+    1: "Deep feed forward (any CSV)",
+    2: "Regularization and optimization (any CSV)",
+    3: "Semi-supervised learning (any text CSV)",
+    4: "Multitasking (any CSV)",
+    5: "CNN-based handwriting recognition (any image folder / CSV / MNIST)",
+    6: "CNN - unsupervised feature learning (any image folder / CSV / digits)",
+    7: "Recurrent neural network (any text CSV / IMDB)",
+    8: "Recursive neural network (tree) (any text CSV / sample)",
+    9: "PCA vs linear autoencoder (any numeric CSV / synthetic)",
+}
+
+register(__name__, globals(), TITLES)

libz-0.6/libz/DLG/prac1.py

@@ -0,0 +1,137 @@
+# Deep Feed Forward Neural Network - works with any CSV classification dataset
+# The target column and the feature columns are picked automatically.
+# If accuracy is low, you are asked which columns to use and the model is retrained.
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.metrics import classification_report, confusion_matrix
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Input, Dense, Dropout
+from tensorflow.keras.callbacks import EarlyStopping
+
+MIN_ACCURACY = 0.70  # below this you get to choose the columns yourself
+TARGET_NAMES = {"target", "label", "labels", "class", "output", "outcome", "result", "y"}
+
+
+# ---------- helpers ----------
+def read_csv(path):
+    """Read a CSV with any separator (, ; tab |) and encoding."""
+    with open(path, encoding="latin-1") as f:
+        first_line = f.readline()
+    sep = max([",", ";", "\t", "|"], key=first_line.count)
+    try:
+        df = pd.read_csv(path, sep=sep)
+    except UnicodeDecodeError:
+        df = pd.read_csv(path, sep=sep, encoding="latin-1")
+    df = df.loc[:, ~df.columns.astype(str).str.startswith("Unnamed")]
+    return df.dropna(axis=1, how="all")
+
+
+def is_id(col):
+    name = str(col.name).lower()
+    return (name in ("id", "index") or name.endswith("_id")
+            or (pd.api.types.is_integer_dtype(col) and col.nunique() == len(col)))
+
+
+def guess_target(df):
+    for c in df.columns:
+        if str(c).lower() in TARGET_NAMES:
+            return c
+    return [c for c in df.columns if not is_id(df[c]) and df[c].nunique() >= 2][-1]
+
+
+def make_labels(col):
+    """Numeric columns with many values become High / Low (split at the median)."""
+    if pd.api.types.is_numeric_dtype(col) and col.nunique() > 10:
+        col = pd.Series(np.where(col >= col.median(), "High", "Low"), index=col.index)
+    encoder = LabelEncoder()
+    return encoder.fit_transform(col.astype(str)), [str(c) for c in encoder.classes_]
+
+
+def make_features(X):
+    """Drop ID / constant / free-text columns, fill missing values, one-hot encode."""
+    keep = [c for c in X.columns
+            if not is_id(X[c]) and X[c].nunique() > 1
+            and (pd.api.types.is_numeric_dtype(X[c]) or X[c].nunique() <= 50)]
+    X = X[keep].copy()
+    for c in X.columns:
+        if pd.api.types.is_numeric_dtype(X[c]):
+            X[c] = X[c].fillna(X[c].median())
+        else:
+            X[c] = X[c].fillna(X[c].mode()[0])
+    return pd.get_dummies(X, drop_first=True, dtype=int), keep
+
+
+def ask_columns(df, target, dropped):
+    print(f"\nAccuracy is below {MIN_ACCURACY:.0%}. You can choose the columns yourself.")
+    print("Columns:", list(df.columns))
+    new_target = input(f"Target column (Enter to keep '{target}'): ").strip() or target
+    if new_target not in df.columns:
+        print(f"'{new_target}' not found, keeping '{target}'.")
+        new_target = target
+    drop = [c.strip() for c in input("Columns to leave out, comma separated (Enter for none): ").split(",")]
+    drop = [c for c in drop if c in df.columns and c != new_target]
+    changed = new_target != target or set(drop) != set(dropped)
+    return new_target, drop, changed
+
+
+# ---------- 1. Load dataset ----------
+path = input("Enter CSV file path: ").strip().strip('"')
+df = read_csv(path)
+print(df.head())
+print("Shape:", df.shape)
+
+target = guess_target(df)
+dropped = []
+
+# ---------- 2. Train (repeats if accuracy is low and you choose other columns) ----------
+while True:
+    data = df.dropna(subset=[target]).drop_duplicates()
+    y, class_names = make_labels(data[target])
+    X, used = make_features(data.drop(columns=[target] + dropped))
+    print(f"\nTarget column: {target} -> classes {class_names}")
+    print("Feature columns used:", used)
+
+    stratify = y if np.bincount(y).min() >= 2 else None
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42, stratify=stratify
+    )
+    scaler = StandardScaler()
+    X_train = scaler.fit_transform(X_train)
+    X_test = scaler.transform(X_test)
+
+    # Deep feed forward network
+    model = Sequential([
+        Input(shape=(X_train.shape[1],)),
+        Dense(128, activation="relu"),
+        Dropout(0.3),
+        Dense(64, activation="relu"),
+        Dropout(0.2),
+        Dense(32, activation="relu"),
+        Dense(len(class_names), activation="softmax"),
+    ])
+    model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
+    model.fit(X_train, y_train, epochs=50, batch_size=32, validation_split=0.2, verbose=0,
+              callbacks=[EarlyStopping(monitor="val_loss", patience=5, restore_best_weights=True)])
+
+    test_acc = model.evaluate(X_test, y_test, verbose=0)[1]
+    print(f"Testing Accuracy: {test_acc * 100:.2f}%")
+    if test_acc >= MIN_ACCURACY:
+        break
+    target, dropped, changed = ask_columns(df, target, dropped)
+    if not changed:
+        break
+
+# ---------- 3. Final results ----------
+model.summary()
+train_acc = model.evaluate(X_train, y_train, verbose=0)[1]
+y_pred = model.predict(X_test, verbose=0).argmax(axis=1)
+
+print(f"\nTraining Accuracy: {train_acc * 100:.2f}%")
+print(f"Testing Accuracy: {test_acc * 100:.2f}%")
+print("\nClassification Report:")
+print(classification_report(y_test, y_pred, labels=range(len(class_names)),
+                            target_names=class_names, zero_division=0))
+print("Confusion Matrix:")
+print(confusion_matrix(y_test, y_pred))

libz-0.6/libz/DLG/prac2.py

@@ -0,0 +1,172 @@
+# Regularization and Optimization - works with any CSV classification dataset
+# Compares: no regularization vs L2 + Dropout + Early Stopping,
+# and the optimizers Adam, SGD and RMSprop.
+# The target column and the feature columns are picked automatically.
+# If accuracy is low, you are asked which columns to use and the models are retrained.
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.metrics import classification_report, confusion_matrix
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Input, Dense, Dropout
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras.callbacks import EarlyStopping
+
+MIN_ACCURACY = 0.70  # below this you get to choose the columns yourself
+TARGET_NAMES = {"target", "label", "labels", "class", "output", "outcome", "result", "y"}
+
+
+# ---------- helpers ----------
+def read_csv(path):
+    """Read a CSV with any separator (, ; tab |) and encoding."""
+    with open(path, encoding="latin-1") as f:
+        first_line = f.readline()
+    sep = max([",", ";", "\t", "|"], key=first_line.count)
+    try:
+        df = pd.read_csv(path, sep=sep)
+    except UnicodeDecodeError:
+        df = pd.read_csv(path, sep=sep, encoding="latin-1")
+    df = df.loc[:, ~df.columns.astype(str).str.startswith("Unnamed")]
+    return df.dropna(axis=1, how="all")
+
+
+def is_id(col):
+    name = str(col.name).lower()
+    return (name in ("id", "index") or name.endswith("_id")
+            or (pd.api.types.is_integer_dtype(col) and col.nunique() == len(col)))
+
+
+def guess_target(df):
+    for c in df.columns:
+        if str(c).lower() in TARGET_NAMES:
+            return c
+    return [c for c in df.columns if not is_id(df[c]) and df[c].nunique() >= 2][-1]
+
+
+def make_labels(col):
+    """Numeric columns with many values become High / Low (split at the median)."""
+    if pd.api.types.is_numeric_dtype(col) and col.nunique() > 10:
+        col = pd.Series(np.where(col >= col.median(), "High", "Low"), index=col.index)
+    encoder = LabelEncoder()
+    return encoder.fit_transform(col.astype(str)), [str(c) for c in encoder.classes_]
+
+
+def make_features(X):
+    """Drop ID / constant / free-text columns, fill missing values, one-hot encode."""
+    keep = [c for c in X.columns
+            if not is_id(X[c]) and X[c].nunique() > 1
+            and (pd.api.types.is_numeric_dtype(X[c]) or X[c].nunique() <= 50)]
+    X = X[keep].copy()
+    for c in X.columns:
+        if pd.api.types.is_numeric_dtype(X[c]):
+            X[c] = X[c].fillna(X[c].median())
+        else:
+            X[c] = X[c].fillna(X[c].mode()[0])
+    return pd.get_dummies(X, drop_first=True, dtype=int), keep
+
+
+def ask_columns(df, target, dropped):
+    print(f"\nBest accuracy is below {MIN_ACCURACY:.0%}. You can choose the columns yourself.")
+    print("Columns:", list(df.columns))
+    new_target = input(f"Target column (Enter to keep '{target}'): ").strip() or target
+    if new_target not in df.columns:
+        print(f"'{new_target}' not found, keeping '{target}'.")
+        new_target = target
+    drop = [c.strip() for c in input("Columns to leave out, comma separated (Enter for none): ").split(",")]
+    drop = [c for c in drop if c in df.columns and c != new_target]
+    changed = new_target != target or set(drop) != set(dropped)
+    return new_target, drop, changed
+
+
+def build_model(n_inputs, n_classes, regularize, optimizer):
+    reg = l2(0.001) if regularize else None
+    drop = 0.3 if regularize else 0.0
+    model = Sequential([
+        Input(shape=(n_inputs,)),
+        Dense(128, activation="relu", kernel_regularizer=reg),
+        Dropout(drop),
+        Dense(64, activation="relu", kernel_regularizer=reg),
+        Dropout(drop),
+        Dense(n_classes, activation="softmax"),
+    ])
+    model.compile(optimizer=optimizer, loss="sparse_categorical_crossentropy", metrics=["accuracy"])
+    return model
+
+
+EXPERIMENTS = [
+    ("No regularization (Adam)", False, "adam"),
+    ("L2 + Dropout + EarlyStop (Adam)", True, "adam"),
+    ("L2 + Dropout + EarlyStop (SGD)", True, "sgd"),
+    ("L2 + Dropout + EarlyStop (RMSprop)", True, "rmsprop"),
+]
+
+# ---------- 1. Load dataset ----------
+path = input("Enter CSV file path: ").strip().strip('"')
+df = read_csv(path)
+print(df.head())
+print("Shape:", df.shape)
+
+target = guess_target(df)
+dropped = []
+
+# ---------- 2. Train all experiments (repeats if accuracy is low and you choose other columns) ----------
+while True:
+    data = df.dropna(subset=[target]).drop_duplicates()
+    y, class_names = make_labels(data[target])
+    X, used = make_features(data.drop(columns=[target] + dropped))
+    print(f"\nTarget column: {target} -> classes {class_names}")
+    print("Feature columns used:", used)
+
+    stratify = y if np.bincount(y).min() >= 2 else None
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42, stratify=stratify
+    )
+    scaler = StandardScaler()
+    X_train = scaler.fit_transform(X_train)
+    X_test = scaler.transform(X_test)
+
+    results, histories, models = [], {}, {}
+    for name, regularize, optimizer in EXPERIMENTS:
+        print("Training:", name)
+        model = build_model(X_train.shape[1], len(class_names), regularize, optimizer)
+        callbacks = [EarlyStopping(monitor="val_loss", patience=5, restore_best_weights=True)] if regularize else []
+        history = model.fit(X_train, y_train, epochs=50, batch_size=32,
+                            validation_split=0.2, callbacks=callbacks, verbose=0)
+        train_acc = model.evaluate(X_train, y_train, verbose=0)[1]
+        test_acc = model.evaluate(X_test, y_test, verbose=0)[1]
+        results.append([name, train_acc * 100, test_acc * 100, len(history.history["loss"])])
+        histories[name] = history
+        models[name] = model
+
+    results_df = pd.DataFrame(results, columns=["Model", "Train Acc", "Test Acc", "Epochs"])
+    best_acc = results_df["Test Acc"].max() / 100
+    print(f"Best Test Accuracy: {best_acc * 100:.2f}%")
+    if best_acc >= MIN_ACCURACY:
+        break
+    target, dropped, changed = ask_columns(df, target, dropped)
+    if not changed:
+        break
+
+# ---------- 3. Compare results ----------
+print("\nComparison:")
+print(results_df.round(2).to_string(index=False))
+
+plt.figure(figsize=(9, 5))
+for name, history in histories.items():
+    plt.plot(history.history["val_loss"], label=name)
+plt.title("Validation Loss: Regularization and Optimizers")
+plt.xlabel("Epoch")
+plt.ylabel("Validation Loss")
+plt.legend()
+plt.grid(True)
+plt.show()
+
+best = results_df.sort_values("Test Acc", ascending=False).iloc[0]["Model"]
+y_pred = models[best].predict(X_test, verbose=0).argmax(axis=1)
+print(f"\nBest model: {best}")
+print(classification_report(y_test, y_pred, labels=range(len(class_names)),
+                            target_names=class_names, zero_division=0))
+print("Confusion Matrix:")
+print(confusion_matrix(y_test, y_pred))

libz-0.6/libz/DLG/prac3.py

@@ -0,0 +1,183 @@
+# Semi-Supervised Learning (self-training with pseudo-labels) - works with any text CSV
+# Only part of the training data keeps its labels. The model labels the rest itself
+# (only predictions it is confident about) and retrains on the bigger dataset.
+# The text column (longest average text) and the label column are picked automatically.
+# If the semi-supervised accuracy is low, you are asked which columns to use and it retrains.
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.preprocessing import LabelEncoder
+from sklearn.metrics import classification_report, confusion_matrix
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Input, Dense, Dropout
+from tensorflow.keras.callbacks import EarlyStopping
+
+MIN_ACCURACY = 0.70  # below this you get to choose the columns yourself
+TARGET_NAMES = {"target", "label", "labels", "class", "output", "outcome", "result", "y"}
+LABELED_RATIO = 0.3  # fraction of the training data that keeps its labels
+THRESHOLD = 0.90     # confidence needed to accept a pseudo-label
+
+
+# ---------- helpers ----------
+def read_csv(path):
+    """Read a CSV with any separator (, ; tab |) and encoding."""
+    with open(path, encoding="latin-1") as f:
+        first_line = f.readline()
+    sep = max([",", ";", "\t", "|"], key=first_line.count)
+    try:
+        df = pd.read_csv(path, sep=sep)
+    except UnicodeDecodeError:
+        df = pd.read_csv(path, sep=sep, encoding="latin-1")
+    df = df.loc[:, ~df.columns.astype(str).str.startswith("Unnamed")]
+    return df.dropna(axis=1, how="all")
+
+
+def pick_column(df, prompt):
+    while True:
+        col = input(prompt).strip()
+        if col in df.columns:
+            return col
+        print(f"'{col}' not found. Columns: {list(df.columns)}")
+
+
+def guess_text_label(df):
+    """Text = non-numeric column with the longest average text; label = a target-like column."""
+    text_cols = [c for c in df.columns if not pd.api.types.is_numeric_dtype(df[c])]
+    if not text_cols or len(df.columns) < 2:
+        print("Could not find a text column and a label column automatically.")
+        print("Columns:", list(df.columns))
+        text_col = pick_column(df, "Text column: ")
+        label_col = pick_column(df, "Label column: ")
+        return text_col, label_col
+
+    text_col = max(text_cols, key=lambda c: df[c].dropna().astype(str).str.len().mean())
+    others = [c for c in df.columns if c != text_col]
+    label_col = next((c for c in others if str(c).lower() in TARGET_NAMES), None)
+    if label_col is None:
+        label_col = next((c for c in others if 2 <= df[c].nunique() <= 20), others[0])
+    return text_col, label_col
+
+
+def ask_columns(df, text_col, label_col):
+    print(f"\nAccuracy is below {MIN_ACCURACY:.0%}. You can choose the columns yourself.")
+    print("Columns:", list(df.columns))
+    new_text = input(f"Text column (Enter to keep '{text_col}'): ").strip() or text_col
+    if new_text not in df.columns:
+        print(f"'{new_text}' not found, keeping '{text_col}'.")
+        new_text = text_col
+    new_label = input(f"Label column (Enter to keep '{label_col}'): ").strip() or label_col
+    if new_label not in df.columns:
+        print(f"'{new_label}' not found, keeping '{label_col}'.")
+        new_label = label_col
+    if new_text == new_label:
+        print("Text and label columns must be different, keeping the previous choice.")
+        new_text, new_label = text_col, label_col
+    changed = new_text != text_col or new_label != label_col
+    return new_text, new_label, changed
+
+
+def train_model(X, y, n_classes):
+    model = Sequential([
+        Input(shape=(X.shape[1],)),
+        Dense(64, activation="relu"),
+        Dropout(0.3),
+        Dense(32, activation="relu"),
+        Dropout(0.3),
+        Dense(n_classes, activation="softmax"),
+    ])
+    model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
+    # Shuffle first: validation_split takes the LAST 20%, which would otherwise be only pseudo-labels
+    order = np.random.permutation(len(X))
+    X, y = X[order], y[order]
+    model.fit(X, y, epochs=20, batch_size=32, validation_split=0.2, verbose=0,
+              callbacks=[EarlyStopping(monitor="val_loss", patience=3, restore_best_weights=True)])
+    return model
+
+
+# ---------- 1. Load dataset ----------
+path = input("Enter CSV file path: ").strip().strip('"')
+df = read_csv(path)
+print(df.head())
+print("Shape:", df.shape)
+
+text_col, label_col = guess_text_label(df)
+
+# ---------- 2. Train (repeats if accuracy is low and you choose other columns) ----------
+while True:
+    # Clean and encode
+    data = df[[text_col, label_col]].dropna().drop_duplicates()
+    encoder = LabelEncoder()
+    y = encoder.fit_transform(data[label_col].astype(str))
+    class_names = [str(c) for c in encoder.classes_]
+    n_classes = len(class_names)
+    print(f"\nText column: {text_col} | Label column: {label_col} -> classes {class_names}")
+    print(data[label_col].value_counts())
+
+    stratify = y if np.bincount(y).min() >= 2 else None
+    X_train_text, X_test_text, y_train, y_test = train_test_split(
+        data[text_col].astype(str), y, test_size=0.2, random_state=42, stratify=stratify
+    )
+
+    # TF-IDF features
+    vectorizer = TfidfVectorizer(max_features=5000, stop_words="english")
+    X_train = vectorizer.fit_transform(X_train_text).toarray().astype("float32")
+    X_test = vectorizer.transform(X_test_text).toarray().astype("float32")
+
+    # Split training data into labelled and unlabelled parts
+    stratify = y_train if np.bincount(y_train).min() >= 2 else None
+    X_lab, X_unlab, y_lab, _ = train_test_split(
+        X_train, y_train, train_size=LABELED_RATIO, random_state=42, stratify=stratify
+    )
+    print("Labelled samples:", len(X_lab))
+    print("Unlabelled samples:", len(X_unlab))
+
+    # Baseline: supervised model trained on the labelled part only
+    baseline = train_model(X_lab, y_lab, n_classes)
+    baseline_acc = baseline.evaluate(X_test, y_test, verbose=0)[1]
+    print(f"\nBaseline (labelled data only) Test Accuracy: {baseline_acc * 100:.2f}%")
+
+    # Self-training: add confident pseudo-labels and retrain
+    model = baseline
+    X_cur, y_cur, pool = X_lab, y_lab, X_unlab
+
+    for round_no in range(1, 4):
+        probs = model.predict(pool, verbose=0)
+        confident = probs.max(axis=1) >= THRESHOLD
+        if confident.sum() == 0:
+            print("No more confident predictions.")
+            break
+
+        X_cur = np.vstack([X_cur, pool[confident]])
+        y_cur = np.concatenate([y_cur, probs[confident].argmax(axis=1)])
+        pool = pool[~confident]
+        print(f"Round {round_no}: added {confident.sum()} pseudo-labels "
+              f"(training size {len(X_cur)}, {len(pool)} unlabelled left)")
+
+        model = train_model(X_cur, y_cur, n_classes)
+
+    ssl_acc = model.evaluate(X_test, y_test, verbose=0)[1]
+    print(f"Semi-Supervised Test Accuracy: {ssl_acc * 100:.2f}%")
+    if ssl_acc >= MIN_ACCURACY:
+        break
+    text_col, label_col, changed = ask_columns(df, text_col, label_col)
+    if not changed:
+        break
+
+# ---------- 3. Compare ----------
+print(f"\nBaseline Test Accuracy:        {baseline_acc * 100:.2f}%")
+print(f"Semi-Supervised Test Accuracy: {ssl_acc * 100:.2f}%")
+
+if ssl_acc > baseline_acc:
+    print("Semi-supervised learning improved the model.")
+elif ssl_acc < baseline_acc:
+    print("Baseline model performed better.")
+else:
+    print("Both models performed equally.")
+
+y_pred = model.predict(X_test, verbose=0).argmax(axis=1)
+print("\nClassification Report (semi-supervised model):")
+print(classification_report(y_test, y_pred, labels=range(len(class_names)),
+                            target_names=class_names, zero_division=0))
+print("Confusion Matrix:")
+print(confusion_matrix(y_test, y_pred))