quick-sentiments 0.1.9__py3-none-any.whl

demo/sentiment_prediction.py

@@ -0,0 +1,67 @@
+! pip install .\dist\quick_sentiments-0.1.6-py3-none-any.whl
+#load the packages
+import polars as pl
+#make sure that this is the main file
+import sys
+import os
+project_root = os.getcwd()
+sys.path.insert(0, project_root)
+
+# here I have three python script I built to pre_process the data and running the pipeline
+# you can find the code in the tools/preprocess.py file
+# you can find  the code in the tools/pipeline.py file
+# the pre_process function is used to clean the text data, there are various options available, please check the tools/preprocess.py file for details
+# the run_pipeline function is used to run the sentimental analysis pipeline, it takes the training data and the vectorizer and machine learning methods as input, and returns the results
+
+# the run_pipeline function is used to run the sentimental analysis pipeline, it takes the training data and the vectorizer and machine learning methods as input, and returns the results
+from quick_sentiments import pre_process
+#this function will run the sentimental analysis in the training data and return the results
+from quick_sentiments import run_pipeline
+# this function will run the sentimental analysis in the new data and return the predictions
+from quick_sentiments import make_predictions
+
+# ENTER YOUR PATHS HERE FOR THE TRAINING DATA SET
+path1 = "training_data/train.csv" #give path to the training data
+df_train = pl.read_csv(path1, has_header=True, encoding="utf8")
+
+# ENTER YOUR PATHS HERE FOR THE TESTING DATA SET
+path2 = "new_data/test.csv" #give path to the test data
+df_test = pl.read_csv(path2, has_header=True, encoding="utf8")
+
+
+# ENTER THE COLUMN NAMES HERE
+response_column = "reviewText" # feel free to change the column name to your text column name
+sentiment_column = "sentiment" # feel free to change the column name to your label column name   
+
+df_train = df_train.with_columns(
+    pl.col(response_column).map_elements(lambda x: pre_process(x)).alias("processed")  #add inside the map_elements
+)
+
+dt= run_pipeline(
+    vectorizer_name="BOW", # BOW, tf, tfidf, wv
+    model_name="logit", # logit, rf, XGB .#XGB takes long time, can not recommend using it on normal case
+    df=df_train,
+    text_column_name="processed",  # this is the column name of the text data, 
+    sentiment_column_name = "sentiment",
+    perform_tuning = False # make this true if you want to perform hyperparameter tuning, it will take longer time and 
+                            # may run out of memory if the dataset is large,
+)
+
+new_data = df_test.with_columns(
+    pl.col(response_column).map_elements(lambda x: pre_process(x).alias("processed")  #add inside the map_elements
+))
+
+# MAKE PREDICTIONS ON THE NEW DATA
+sentiments_prediction= make_predictions(
+    new_data=new_data,
+    text_column_name="processed",
+    vectorizer=dt["vectorizer_object"],
+    best_model=dt["model_object"],
+    label_encoder=dt["label_encoder"],
+    prediction_column_name="sentiment_predictions"  # Optional custom name
+)
+
+# SAVE THE PREDICTIONS TO A CSV FILE
+sentiments_prediction.write_csv("new_data/sentiments_prediction.csv")
+
+# THE END

quick_sentiments/__init__.py

@@ -0,0 +1,7 @@
+# quick_sentiments/__init__.py
+from .pipeline import run_pipeline       # Expose pipeline function
+from .predict import make_predictions # Expose prediction function
+from .preprocess import pre_process  # Expose preprocessing function
+
+__all__ = ['run_pipeline', 'make_predictions']  # Controls what's available in *
+

quick_sentiments/ml_algo/XGB.py

@@ -0,0 +1,91 @@
+from xgboost import XGBClassifier # Import XGBClassifier
+from sklearn.model_selection import GridSearchCV # Import GridSearchCV
+from sklearn.metrics import classification_report # For evaluation metrics
+import numpy as np # For type hinting
+
+def train_and_predict(X_train, y_train, X_test, perform_tuning: bool = True): # <--- Added perform_tuning flag
+    """
+    Trains XGBoostClassifier model (with optional hyperparameter tuning) and predicts on test data.
+
+    Args:
+        X_train: training features (e.g., NumPy array or sparse matrix).
+        y_train: training labels (numerical, e.g., 0, 1, 2...).
+        X_test: test features (e.g., NumPy array or sparse matrix).
+        perform_tuning (bool): If True, performs GridSearchCV. If False, trains
+                               the model with default parameters. Defaults to True.
+
+    Returns:
+        y_pred: predicted labels for test set.
+        best_model: The best trained XGBoostClassifier model (either from GridSearchCV or simple fit).
+    """
+    print("   - Starting XGBoost training...")
+
+    # Determine objective and eval_metric based on number of unique classes
+    num_classes = len(np.unique(y_train))
+    
+    if num_classes == 2:
+        xgb_objective = 'binary:logistic'
+        xgb_eval_metric = 'logloss'
+        scoring_metric = 'f1_weighted'
+    else:
+        xgb_objective = 'multi:softmax'
+        xgb_eval_metric = 'mlogloss'
+        scoring_metric = 'f1_weighted' # Or 'accuracy'
+
+    # Base XGBClassifier model (used for both tuning and simple fit)
+    # verbosity=0 to suppress excessive output from XGBoost itself during GridSearchCV
+    xgb_model = XGBClassifier(
+        objective=xgb_objective,
+        eval_metric=xgb_eval_metric,
+        use_label_encoder=False, # Suppress warning for newer versions
+        random_state=42,
+        num_class=num_classes if num_classes > 2 else None,
+        verbosity=0 # Suppress XGBoost internal verbosity during grid search
+    )
+
+
+    if perform_tuning:
+        print("   - Performing GridSearchCV for hyperparameter tuning...")
+
+       
+        param_grid = {
+            'n_estimators': [100, 200],
+            'learning_rate': [0.05, 0.1, 0.2],
+            'max_depth': [3, 5, 7],
+            'subsample': [0.8, 1.0],
+            'colsample_bytree': [0.8, 1.0],
+        
+        
+        }
+        print("   - Using default parameter grid for tuning:", param_grid)
+
+        grid_search = GridSearchCV(
+            estimator=xgb_model, # Use the base xgb_model here
+            param_grid=param_grid,
+            cv=5,
+            scoring=scoring_metric,
+            n_jobs=-1,
+            verbose=1 # Print progress messages from GridSearchCV
+        )
+
+        # Fit GridSearchCV to the training data
+        grid_search.fit(X_train, y_train)
+
+        # Get the best model found by GridSearchCV
+        best_model = grid_search.best_estimator_
+
+        print("\n   - Best Hyperparameters found:")
+        print(grid_search.best_params_)
+        print(f"   - Best Cross-Validation Score ({scoring_metric}): {grid_search.best_score_:.4f}")
+    else:
+        
+        print("   - Training XGBoost with default parameters (no hyperparameter tuning)...")
+        best_model = xgb_model # Use the base model directly
+        best_model.fit(X_train, y_train) # Fit it on X_train, y_train
+        print("   - Model trained with default parameters.")
+
+    # Make predictions on the test set using the best model (tuned or default)
+    y_pred = best_model.predict(X_test)
+
+    # Return both the predictions and the best model object
+    return y_pred, best_model

quick_sentiments/ml_algo/logit.py

@@ -0,0 +1,60 @@
+# MLAlgo/logistic_regression_model.py
+
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import GridSearchCV # Import GridSearchCV
+from sklearn.metrics import classification_report # For evaluation metrics
+import numpy as np # For type hinting
+
+def train_and_predict(X_train, y_train, X_test,perform_tuning = False):
+    """
+        Trains Logistic Regression model (with optional hyperparameter tuning) and predicts on test data.
+
+    Args:
+        X_train: training features (e.g., NumPy array or sparse matrix).
+        y_train: training labels (list or NumPy array).
+        X_test: test features (e.g., NumPy array or sparse matrix).
+        perform_tuning (bool): If True, performs GridSearchCV. If False, trains
+                               the model with default parameters. Defaults to True.
+
+    Returns:
+        y_pred: predicted labels for test set.
+        best_model: The best trained LogisticRegression model (either from GridSearchCV or simple fit).
+    """
+    lr_model = LogisticRegression(random_state=42) # Base model for training
+
+    if perform_tuning:
+        print("   - Starting Logistic Regression training with GridSearchCV for hyperparameter tuning...")
+
+        # Define the parameter grid to search (default grid, as no custom grid is passed here)
+        param_grid = {
+            'solver': ['liblinear', 'lbfgs'],
+            'C': [0.1, 1.0, 10.0],
+            'class_weight': [None, 'balanced'],
+            'max_iter': [500, 1000]
+        }
+        print("   - Using default parameter grid for tuning:", param_grid)
+
+        grid_search = GridSearchCV(
+            estimator=lr_model,
+            param_grid=param_grid,
+            cv=5,
+            scoring='f1_weighted',
+            n_jobs=-1,
+            verbose=1
+        )
+
+        grid_search.fit(X_train, y_train)
+        best_model = grid_search.best_estimator_
+        print("\n   - Best Hyperparameters found:")
+        print(grid_search.best_params_)
+        print(f"   - Best Cross-Validation Score (F1-weighted): {grid_search.best_score_:.4f}")
+    else:
+        print("   - Training Logistic Regression with default parameters (no hyperparameter tuning)...")
+        best_model = lr_model # Use the base model directly
+        best_model.fit(X_train, y_train) # Fit it on X_train, y_train
+        print("   - Model trained with default parameters.")
+
+    y_pred = best_model.predict(X_test)
+    print("Best model parameters:", best_model.get_params())
+
+    return y_pred, best_model

quick_sentiments/ml_algo/rf.py

@@ -0,0 +1,67 @@
+# MLAlgo/random_forest_model.py
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import GridSearchCV # Import GridSearchCV
+from sklearn.metrics import classification_report # For evaluation metrics
+import numpy as np # For type hinting
+
+def train_and_predict(X_train, y_train, X_test, perform_tuning = False):
+    
+    """
+    Trains RandomForestClassifier model (with optional hyperparameter tuning) and predicts on test data.
+
+    Args:
+        X_train: training features (e.g., NumPy array or sparse matrix).
+        y_train: training labels (list or NumPy array).
+        X_test: test features (e.g., NumPy array or sparse matrix).
+        perform_tuning (bool): If True, performs GridSearchCV. If False, trains
+                               the model with default parameters. Defaults to True.
+
+    Returns:
+        y_pred: predicted labels for test set.
+        best_model: The best trained RandomForestClassifier model (either from GridSearchCV or simple fit).
+    """
+    rf_model = RandomForestClassifier(random_state=42) # Base model for training
+
+    if perform_tuning:
+        print("   - Starting Random Forest training with GridSearchCV for hyperparameter tuning...")
+
+        # Define the parameter grid to search for RandomForestClassifier (default grid)
+        param_grid = {
+            'n_estimators': [100, 200, 300],
+            'max_depth': [10, 20, None],
+            'min_samples_split': [2, 5],
+            'min_samples_leaf': [1, 2],
+            'class_weight': [None, 'balanced']
+        }
+        print("   - Using default parameter grid for tuning:", param_grid)
+
+        grid_search = GridSearchCV(
+            estimator=rf_model, # Use the base rf_model here
+            param_grid=param_grid,
+            cv=5,
+            scoring='f1_weighted',
+            n_jobs=-1,
+            verbose=1
+        )
+
+        # Fit GridSearchCV to the training data
+        grid_search.fit(X_train, y_train)
+
+        # Get the best model found by GridSearchCV
+        best_model = grid_search.best_estimator_
+
+        print("\n   - Best Hyperparameters found:")
+        print(grid_search.best_params_)
+        print(f"   - Best Cross-Validation Score (F1-weighted): {grid_search.best_score_:.4f}")
+    else:
+        print("   - Training Random Forest with default parameters (no hyperparameter tuning)...")
+        best_model = rf_model # Use the base model directly
+        best_model.fit(X_train, y_train) # Fit it on X_train, y_train
+        print("   - Model trained with default parameters.")
+
+    # Make predictions on the test set using the best model (tuned or default)
+    y_pred = best_model.predict(X_test)
+
+    # Return both the predictions and the best model object
+    return y_pred, best_model

quick_sentiments/pipeline.py

@@ -0,0 +1,128 @@
+# POSITIVELY DO NOT CHANGE (This comment is for the user, I am making necessary fixes based on our conversation)
+
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import classification_report, accuracy_score
+from collections import Counter
+from sklearn.preprocessing import LabelEncoder
+import polars as pl
+import importlib
+import numpy as np
+import pandas as pd
+from typing import Union
+
+def run_pipeline(
+    vectorizer_name: str,
+    model_name: str,
+    df: Union[pl.DataFrame, pd.DataFrame],
+    text_column_name: str,
+    sentiment_column_name: str,
+    perform_tuning: bool = False
+):
+    """
+    Runs the full pipeline:
+      - vectorize
+      - train model
+      - evaluate
+
+    Args:
+        vectorizer_name (str): Name of the vectorizer (e.g., 'tfidf', 'word_embedding').
+        model_name (str): Name of the ML model (e.g., 'logistic_regression', 'random_forest').
+        df (pl.DataFrame): Your Polars DataFrame containing the text and sentiment columns.
+        text_column_name (str): The name of the column in `df` that contains the processed text.
+        sentiment_column_name (str): The name of the column in `df` that contains the sentiment labels.
+
+    Returns:
+        dict: A dictionary containing the trained model, fitted vectorizer, label encoder, and evaluation results.
+    """
+    print(f"--- Running Pipeline for {vectorizer_name.replace('_', ' ').title()} + {model_name.replace('_', ' ').title()} ---")
+
+    # Import vectorizer from vect folder
+    try:
+        vec_module = importlib.import_module(f"quick_sentiments.vect.{vectorizer_name}")
+        vectorize_function = getattr(vec_module, "vectorize")
+    except (ImportError, AttributeError) as e:
+        print(f"Error loading vectorizer module/function: {e}")
+        return None
+
+    # Import ML model from ml_algo folder
+    try:
+        model_module = importlib.import_module(f"quick_sentiments.ml_algo.{model_name}")
+        train_and_predict_function = getattr(model_module, "train_and_predict")
+    except (ImportError, AttributeError) as e:
+        print(f"Error loading ML model module/function: {e}")
+        return None
+
+    """
+    Modified to handle both Polars and pandas DataFrames.
+    """
+    # Convert to Polars if input is pandas
+    if isinstance(df, pd.DataFrame):
+        df = pl.from_pandas(df)
+    elif not isinstance(df, pl.DataFrame):
+        raise TypeError(f"Expected Polars or pandas DataFrame, got {type(df)}")
+    
+    # Polars DataFrame handling
+    X_text = df[text_column_name].to_list()
+    y_raw = df[sentiment_column_name].to_list()
+    
+    # --- NEW: Check for and drop None values in X_text and y_raw ---
+    initial_data_len = len(X_text)
+    
+    # Filter out pairs where either X_text element or y_raw element is None
+    # Use zip to iterate over both lists simultaneously and filter
+    filtered_data = [(x, y_val) for x, y_val in zip(X_text, y_raw) if x is not None and y_val is not None]
+    
+    # Unzip the filtered data back into X_text and y_raw
+    if filtered_data: # Check if filtered_data is not empty to avoid unpacking error
+        X_text, y_raw = zip(*filtered_data)
+        X_text = list(X_text) # Convert back to list
+        y_raw = list(y_raw)   # Convert back to list
+    else:
+        # Handle case where all data might be None
+        print("WARNING: All data rows contained missing values after initial extraction. Cannot proceed with training.")
+        return None
+
+    dropped_rows_count = initial_data_len - len(X_text)
+    if dropped_rows_count > 0:
+        print(f"WARNING: Dropped {dropped_rows_count} rows due to missing values (None) in '{text_column_name}' or '{sentiment_column_name}' columns. Original rows: {initial_data_len}, Rows after dropping: {len(X_text)}")
+    else:
+        print("No missing values (None) found in text or sentiment columns. Proceeding with all rows.")
+    # ------------------------------------------------------------------
+
+    # Label Encoding for y_raw
+    label_encoder = LabelEncoder()
+    y = label_encoder.fit_transform(y_raw)
+    print(f"Labels encoded: Original -> {label_encoder.classes_}, Encoded -> {np.unique(y)}")
+
+    # Vectorize the entire dataset (X)
+    print("1. Vectorizing entire dataset (X)...")
+    X_vectorized, fitted_vectorizer_object = vectorize_function(X_text)
+
+    # Split data AFTER vectorization
+    print("2. Splitting data into train/test...")
+    X_train, X_test, y_train, y_test = train_test_split(
+        X_vectorized, y, test_size=0.2, random_state=42, stratify=y
+    )
+
+    # Train + predict
+    print("3. Training and predicting...")
+    y_pred, trained_model_object = train_and_predict_function(X_train, y_train, X_test, perform_tuning=perform_tuning)
+
+    # Evaluate
+    print("4. Evaluating model...")
+    print("\nClassification Report:")
+    print(classification_report(y_test, y_pred, target_names=label_encoder.classes_))
+    print("True labels distribution:", Counter(y_test))
+    print("Predicted labels distribution:", Counter(y_pred))
+
+    # Return results including all necessary objects for future predictions
+    return {
+        "model_object": trained_model_object,
+        "vectorizer_name": vectorizer_name,
+        "vectorizer_object": fitted_vectorizer_object,
+        "label_encoder": label_encoder,
+        "y_test": y_test,
+        "y_pred": y_pred,
+        "accuracy": accuracy_score(y_test, y_pred),
+        "report": classification_report(y_test, y_pred, output_dict=True, target_names=label_encoder.classes_)
+    }

quick_sentiments/predict.py

@@ -0,0 +1,56 @@
+from typing import Union
+import polars as pl
+import pandas as pd
+import numpy as np
+
+def make_predictions(
+        new_data: Union[pl.DataFrame, pd.DataFrame],
+        text_column_name: str,
+        vectorizer,
+        best_model,
+        label_encoder,
+        prediction_column_name: str = "predictions") -> pl.DataFrame:
+    """
+    Makes predictions and adds them as a new column with original labels.
+    
+    Args:
+        new_data: Input DataFrame (Polars or pandas)
+        text_column_name: Name of column containing text to predict on
+        vectorizer: Fitted vectorizer (TF-IDF/BOW) or word embeddings model
+        best_model: Trained model (must have classes_ attribute)
+        label_encoder: Fitted LabelEncoder for inverse transform
+        prediction_column_name: Name for new prediction column
+        
+    Returns:
+        Polars DataFrame with label predictions added
+    """
+    # Convert pandas to Polars if needed
+    if isinstance(new_data, pd.DataFrame):
+        new_data = pl.from_pandas(new_data)
+    elif not isinstance(new_data, pl.DataFrame):
+        raise TypeError(f"Expected Polars or pandas DataFrame, got {type(new_data)}")
+
+    # Drop nulls in the text column
+    new_data = new_data.drop_nulls(subset=[text_column_name])
+    texts = new_data[text_column_name].to_list()
+    
+    # Generate features
+    if hasattr(vectorizer, 'transform'):
+        new_features = vectorizer.transform(texts)
+    else:
+        def text_to_vector(text):
+            words = text.split()
+            vectors = [vectorizer[word] for word in words if word in vectorizer]
+            return np.mean(vectors, axis=0) if vectors else np.zeros(vectorizer.vector_size)
+        new_features = np.array([text_to_vector(text) for text in texts])
+    
+    # Get numerical predictions
+    numeric_predictions = best_model.predict(new_features)
+    
+    # Convert to original labels
+    predictions = label_encoder.inverse_transform(numeric_predictions)
+    
+    # Add predictions as new column
+    return new_data.with_columns(
+        pl.Series(prediction_column_name, predictions)
+    )

quick_sentiments/preprocess.py

@@ -0,0 +1,195 @@
+import string
+import re
+import nltk
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+import unicodedata
+
+# This block ensures NLTK data is downloaded if not found.
+# It's good practice to have this check in a module that relies on NLTK data.
+# --- NLTK Downloads (Run these once, preferably outside the script or with a check) ---
+# This block ensures NLTK data is downloaded if not found.
+# It's good practice to have this check in a module that relies on NLTK data.
+def _download_nltk_data():
+    """Helper function to download NLTK data if not present."""
+    # List of NLTK datasets required by this module
+    # 'punkt' for word_tokenize, 'stopwords' for stopwords, 'wordnet' for WordNetLemmatizer
+    # 'omw-1.4' is often recommended for WordNetLemmatizer for broader coverage.
+    datasets = ['punkt', 'stopwords', 'wordnet', 'omw-1.4'] # <--- ADDED 'omw-1.4'
+
+    for dataset in datasets:
+        try:
+            # NLTK data paths vary, 'punkt' is a tokenizer, others are corpora.
+            # This check tries to find it based on common NLTK data structures.
+            if dataset == 'punkt':
+                nltk.data.find(f'tokenizers/{dataset}')
+            elif dataset == 'omw-1.4': # omw-1.4 is part of corpora
+                nltk.data.find(f'corpora/{dataset}')
+            else: # stopwords, wordnet are also in corpora
+                nltk.data.find(f'corpora/{dataset}')
+            print(f"NLTK data '{dataset}' already present.")
+        except Exception as e: # Catch any exception if data is not found
+            print(f"Downloading NLTK data: {dataset}...")
+            nltk.download(dataset)
+            print(f"NLTK data '{dataset}' downloaded.")
+        # Note regarding 'punkt_tab': There isn't a standard 'punkt_tab' dataset.
+        # It's likely a typo for 'punkt'. 'punkt' is already included.
+
+_download_nltk_data() # Call this helper function when the module is imported
+
+# --- Initialize NLTK components once ---
+# These are global to this module
+lemmatizer = WordNetLemmatizer()
+stop_words = set(stopwords.words('english'))
+
+# --- Initialize NLTK components once ---
+# These are global to this module
+lemmatizer = WordNetLemmatizer()
+stop_words = set(stopwords.words('english'))
+
+
+
+# --- Helper Cleaning Functions ---
+def remove_square_brackets(text):
+    """Removes text enclosed in square brackets."""
+    cleaned_text = re.sub(r'\[.*?\]', '', text)
+    return cleaned_text.strip()
+
+def remove_urls_emails(text):
+    """Removes URLs and email addresses from text."""
+    text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE)
+    text = re.sub(r'\S*@\S*\s?', '', text)
+    return text
+
+def remove_html_tags(text):
+    """Removes HTML tags from text."""
+    clean = re.compile('<.*?>')
+    return re.sub(clean, '', text)
+
+def remove_numbers(text):
+    """Removes all digits from text."""
+    return re.sub(r'\d+', '', text)
+
+def remove_extra_spaces(text):
+    """Replaces multiple spaces with a single space and strips leading/trailing spaces."""
+    return re.sub(r'\s+', ' ', text).strip()
+
+def remove_punctuation_from_token(token):
+    """Removes punctuation characters from an individual token."""
+    translator = str.maketrans('', '', string.punctuation)
+    return token.translate(translator)
+
+def remove_emojis(text):
+    """Removes common emojis from text using a regex pattern."""
+    emoji_pattern = re.compile(
+        "["
+        "\U0001F600-\U0001F64F"  # emoticons
+        "\U0001F300-\U0001F5FF"  # symbols & pictographs
+        "\U0001F680-\U0001F6FF"  # transport & map symbols
+        "\U0001F1E0-\U0001F1FF"  # flags (iOS)
+        "\U00002702-\U000027B0"
+        "\U000024C2-\U0001F251"
+        "]+", flags=re.UNICODE)
+    return emoji_pattern.sub(r'', text)
+
+def normalize_unicode_characters(text): # <--- NEW HELPER FUNCTION
+    """
+    Normalizes unicode characters (e.g., smart quotes, accented chars)
+    to their closest ASCII equivalents and removes non-ASCII.
+    """
+    # Normalize to NFKD form (decomposes characters like é to e + accent)
+    # Then encode to ASCII, ignoring characters that can't be mapped
+    return unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
+
+
+# --- Main Preprocessing Function with Options ---
+def pre_process(doc,
+                remove_brackets=True,
+                remove_urls=True,
+                remove_html=True,
+                remove_nums=False,
+                remove_emojis_flag=False,
+                normalize_unicode=True,
+                to_lowercase=True,
+                tokenize=True,
+                remove_punct_tokens=True,
+                remove_stop_words=True,
+                lemmatize=True,
+                remove_extra_space=True,
+                return_string=True):
+    """
+    
+    Preprocesses a text document with configurable cleaning steps.
+
+    Args:
+        doc (str): The input text document.
+        remove_brackets (bool): If True, remove text in square brackets.
+        remove_urls (bool): If True, remove URLs and email addresses.
+        remove_html (bool): If True, remove HTML tags.
+        remove_nums (bool): If True, remove all numeric digits.
+        remove_emojis_flag (bool): If True, remove common emojis.
+        normalize_unicode (bool): If True, normalize unicode characters to ASCII.
+        to_lowercase (bool): If True, convert text to lowercase.
+        tokenize (bool): If True, tokenize the text using NLTK's word_tokenize.
+        remove_punct_tokens (bool): If True, remove punctuation from individual tokens.
+        remove_stop_words (bool): If True, remove common English stop words.
+        lemmatize (bool): If True, perform lemmatization on tokens.
+        remove_extra_space (bool): If True, replace multiple spaces with single spaces.
+        return_string (bool): If True, join tokens back into a string; otherwise, return a list of tokens.
+
+    Returns:
+        str or list: The preprocessed text as a string or a list of tokens.
+    """
+        
+
+    # Stage 1: Text-level cleaning (before tokenization)
+    if remove_brackets:
+        doc = remove_square_brackets(doc)
+    if remove_urls:
+        doc = remove_urls_emails(doc)
+    if remove_html:
+        doc = remove_html_tags(doc)
+    if remove_nums:
+        doc = remove_numbers(doc)
+    if remove_emojis_flag:
+        doc = remove_emojis(doc)
+    if normalize_unicode: # <--- NEW STEP IN PIPELINE
+        doc = normalize_unicode_characters(doc)
+    if to_lowercase:
+        doc = doc.lower()
+    if remove_extra_space:
+        doc = remove_extra_spaces(doc)
+
+    # Stage 2: Tokenization
+    tokens = []
+    if tokenize:
+        tokens = word_tokenize(doc)
+    else:
+        # If not tokenizing, just return the string after initial cleaning
+        # Ensure it's a list if return_string is False, even if it's a single item
+        return [doc.strip()] if not return_string else doc.strip()
+
+    # Stage 3: Token-level cleaning and normalization
+    processed_tokens = []
+    for token in tokens:
+        if remove_punct_tokens:
+            token = remove_punctuation_from_token(token)
+
+        if not token: # Skip if token became empty after cleaning punctuation
+            continue
+
+        if remove_stop_words:
+            if token in stop_words:
+                continue
+
+        if lemmatize:
+            token = lemmatizer.lemmatize(token)
+
+        if token: # Final check in case lemmatization or other steps create empty strings
+            processed_tokens.append(token)
+
+    if return_string:
+        return " ".join(processed_tokens)
+    else:
+        return processed_tokens