scikit-lab-kit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scikit_lab_kit-0.1.0/PKG-INFO +46 -0
- scikit_lab_kit-0.1.0/README.md +39 -0
- scikit_lab_kit-0.1.0/pyproject.toml +19 -0
- scikit_lab_kit-0.1.0/scikit_lab/__init__.py +3 -0
- scikit_lab_kit-0.1.0/scikit_lab/__main__.py +4 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/01_data_cleaning_missing_values.txt +72 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/02_data_cleaning_noisy_data.txt +69 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/03_data_cleaning_on_dataset.txt +70 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/04_preprocessing_contingency.txt +42 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/05_eda.txt +252 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/06_decision_tree.txt +75 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/07_compare_classification_models.txt +372 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/08_classification_improvement.txt +162 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/09_clustering.txt +101 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/10_association_apriori_groceries.txt +65 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/11_association_apriori_own_dataset.txt +140 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/12_fp_tree_association.txt +36 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/13_classifiers_explanation.txt +51 -0
- scikit_lab_kit-0.1.0/scikit_lab/practicals/14_pca.txt +16 -0
- scikit_lab_kit-0.1.0/scikit_lab/viewer.py +106 -0
- scikit_lab_kit-0.1.0/scikit_lab_kit.egg-info/PKG-INFO +46 -0
- scikit_lab_kit-0.1.0/scikit_lab_kit.egg-info/SOURCES.txt +24 -0
- scikit_lab_kit-0.1.0/scikit_lab_kit.egg-info/dependency_links.txt +1 -0
- scikit_lab_kit-0.1.0/scikit_lab_kit.egg-info/entry_points.txt +2 -0
- scikit_lab_kit-0.1.0/scikit_lab_kit.egg-info/top_level.txt +1 -0
- scikit_lab_kit-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scikit-lab-kit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Numbered DAML (scikit-learn) practicals: steps, code and short explanation
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
|
|
8
|
+
# scikit-lab-kit
|
|
9
|
+
|
|
10
|
+
Numbered viewer for the DAML practicals (steps, code and a short explanation), shown in **red**.
|
|
11
|
+
Type a number to show a practical, `m` to show the menu, `0` to exit.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
```
|
|
15
|
+
python -m pip install scikit-lab-kit
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Run
|
|
19
|
+
```
|
|
20
|
+
python -m scikit_lab
|
|
21
|
+
```
|
|
22
|
+
or in IDLE / Python:
|
|
23
|
+
```python
|
|
24
|
+
import scikit_lab
|
|
25
|
+
scikit_lab.main()
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Menu
|
|
29
|
+
| No. | Practical |
|
|
30
|
+
|-----|-----------|
|
|
31
|
+
| 1 | Data cleaning - missing values |
|
|
32
|
+
| 2 | Data cleaning - noisy data (outliers) |
|
|
33
|
+
| 3 | Data cleaning on a dataset (loan credit) |
|
|
34
|
+
| 4 | Preprocessing - correlation and contingency table |
|
|
35
|
+
| 5 | Exploratory data analysis (EDA) |
|
|
36
|
+
| 6 | Decision tree |
|
|
37
|
+
| 7 | Comparing classification models |
|
|
38
|
+
| 8 | Classification improvement (bagging, boosting) |
|
|
39
|
+
| 9 | Clustering (K-Means, hierarchical) |
|
|
40
|
+
| 10 | Association - Apriori (groceries) |
|
|
41
|
+
| 11 | Association - Apriori (own dataset) |
|
|
42
|
+
| 12 | FP-Tree association |
|
|
43
|
+
| 13 | Classifiers explanation (PCA, SVM, Random Forest) |
|
|
44
|
+
| 14 | Principal Component Analysis (PCA) |
|
|
45
|
+
|
|
46
|
+
This package only displays text; it does not run the practical code.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# scikit-lab-kit
|
|
2
|
+
|
|
3
|
+
Numbered viewer for the DAML practicals (steps, code and a short explanation), shown in **red**.
|
|
4
|
+
Type a number to show a practical, `m` to show the menu, `0` to exit.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
```
|
|
8
|
+
python -m pip install scikit-lab-kit
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Run
|
|
12
|
+
```
|
|
13
|
+
python -m scikit_lab
|
|
14
|
+
```
|
|
15
|
+
or in IDLE / Python:
|
|
16
|
+
```python
|
|
17
|
+
import scikit_lab
|
|
18
|
+
scikit_lab.main()
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Menu
|
|
22
|
+
| No. | Practical |
|
|
23
|
+
|-----|-----------|
|
|
24
|
+
| 1 | Data cleaning - missing values |
|
|
25
|
+
| 2 | Data cleaning - noisy data (outliers) |
|
|
26
|
+
| 3 | Data cleaning on a dataset (loan credit) |
|
|
27
|
+
| 4 | Preprocessing - correlation and contingency table |
|
|
28
|
+
| 5 | Exploratory data analysis (EDA) |
|
|
29
|
+
| 6 | Decision tree |
|
|
30
|
+
| 7 | Comparing classification models |
|
|
31
|
+
| 8 | Classification improvement (bagging, boosting) |
|
|
32
|
+
| 9 | Clustering (K-Means, hierarchical) |
|
|
33
|
+
| 10 | Association - Apriori (groceries) |
|
|
34
|
+
| 11 | Association - Apriori (own dataset) |
|
|
35
|
+
| 12 | FP-Tree association |
|
|
36
|
+
| 13 | Classifiers explanation (PCA, SVM, Random Forest) |
|
|
37
|
+
| 14 | Principal Component Analysis (PCA) |
|
|
38
|
+
|
|
39
|
+
This package only displays text; it does not run the practical code.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scikit-lab-kit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Numbered DAML (scikit-learn) practicals: steps, code and short explanation"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
|
|
12
|
+
[project.scripts]
|
|
13
|
+
scikit-lab = "scikit_lab.viewer:main"
|
|
14
|
+
|
|
15
|
+
[tool.setuptools.packages.find]
|
|
16
|
+
include = ["scikit_lab*"]
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.package-data]
|
|
19
|
+
scikit_lab = ["practicals/*.txt"]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
PRACTICAL 1: DATA CLEANING - MISSING VALUES
|
|
2
|
+
|
|
3
|
+
STEPS:
|
|
4
|
+
1. Install the libraries: pip install pandas scikit-learn
|
|
5
|
+
2. Keep the data.csv file in the same folder as the program.
|
|
6
|
+
3. Load the csv file into a DataFrame and print the columns.
|
|
7
|
+
4. Find the missing values (count, any-missing rows, all-missing rows).
|
|
8
|
+
5. Fill the missing values (Name -> "Unknown", City -> forward/backward fill, Marks -> mean).
|
|
9
|
+
6. Scale the Marks column using Min-Max and Z-score scaling.
|
|
10
|
+
7. Print the statistics of Marks (mean, median, max, min).
|
|
11
|
+
8. Drop the rows with missing values and save the cleaned data to student_cleaned.csv.
|
|
12
|
+
|
|
13
|
+
CODE:
|
|
14
|
+
import pandas as pd
|
|
15
|
+
from sklearn.preprocessing import MinMaxScaler, StandardScaler
|
|
16
|
+
try:
|
|
17
|
+
df = pd.read_csv("data.csv")
|
|
18
|
+
print("Columns detected in csv file:", df.columns.tolist())
|
|
19
|
+
except Exception as e:
|
|
20
|
+
print(f"Error loading csv file: {e}")
|
|
21
|
+
exit()
|
|
22
|
+
print("\nOriginal DataFrame:")
|
|
23
|
+
print(df)
|
|
24
|
+
print("\nMissing Values Summary:")
|
|
25
|
+
print(df.isna().sum())
|
|
26
|
+
print("\nRows with Any missing values:")
|
|
27
|
+
print(df[df.isna().any(axis=1)])
|
|
28
|
+
print("\nRows with ALL missing values:")
|
|
29
|
+
print(df[df.isna().all(axis=1)])
|
|
30
|
+
print("\n################ FILL MISSING DATA ###################")
|
|
31
|
+
if "Name" in df.columns:
|
|
32
|
+
df["Name"] = df["Name"].fillna("Unknown")
|
|
33
|
+
if "City" in df.columns:
|
|
34
|
+
df["City"] = df["City"].ffill().bfill()
|
|
35
|
+
if 'Dob' in df.columns:
|
|
36
|
+
df['Dob'] = df['Dob'].fillna('1900-01-01')
|
|
37
|
+
if 'Marks' in df.columns:
|
|
38
|
+
df['Marks'] = pd.to_numeric(df['Marks'], errors='coerce')
|
|
39
|
+
df['Marks'] = df['Marks'].fillna(df['Marks'].mean())
|
|
40
|
+
print("\nDataFrame After Filling Missing Values:")
|
|
41
|
+
print(df)
|
|
42
|
+
if 'Marks' in df.columns and not df['Marks'].isna().all():
|
|
43
|
+
# Min-Max Scaling
|
|
44
|
+
scaler = MinMaxScaler()
|
|
45
|
+
df['Marks_MINMAX'] = scaler.fit_transform(df[['Marks']])
|
|
46
|
+
zscaler = StandardScaler()
|
|
47
|
+
df['Marks_ZSCORE'] = zscaler.fit_transform(df[['Marks']])
|
|
48
|
+
print("\nDataFrame with Scaled Marks:")
|
|
49
|
+
print(df[['Name', 'Roll_no', 'Marks', 'Marks_MINMAX', 'Marks_ZSCORE']])
|
|
50
|
+
else:
|
|
51
|
+
print("\nNo valid Marks column available for scaling")
|
|
52
|
+
if 'Marks' in df.columns:
|
|
53
|
+
print("\nMarks Statistics:")
|
|
54
|
+
print(f"Mean: {df['Marks'].mean():.2f}")
|
|
55
|
+
print(f"Median: {df['Marks'].median():.2f}")
|
|
56
|
+
print(f"Max: {df['Marks'].max():.2f}")
|
|
57
|
+
print(f"Min: {df['Marks'].min():.2f}")
|
|
58
|
+
df_clean = df.dropna(how='any')
|
|
59
|
+
print("\nDataFrame after dropping rows with ANY missing values:")
|
|
60
|
+
print(df_clean)
|
|
61
|
+
df_clean_all = df.dropna(how='all')
|
|
62
|
+
print("\nDataFrame after dropping rows with ALL missing values:")
|
|
63
|
+
print(df_clean_all)
|
|
64
|
+
df.to_csv('student_cleaned.csv', index=False)
|
|
65
|
+
print("\nCleaned data saved to 'student_cleaned.csv'")
|
|
66
|
+
|
|
67
|
+
EXPLANATION:
|
|
68
|
+
- The program reads the student csv and uses isna() to show how many values are missing in each column.
|
|
69
|
+
- Missing Name is filled with "Unknown", City is filled from the neighbouring rows and Marks with the column mean.
|
|
70
|
+
- MinMaxScaler brings Marks into the range 0-1 and StandardScaler converts it to a Z-score.
|
|
71
|
+
- dropna(how='any') removes rows with any gap, dropna(how='all') removes only completely empty rows.
|
|
72
|
+
- The final cleaned data is saved to student_cleaned.csv.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
PRACTICAL 2: DATA CLEANING - NOISY DATA
|
|
2
|
+
|
|
3
|
+
STEPS:
|
|
4
|
+
1. Install the libraries: pip install pandas numpy matplotlib
|
|
5
|
+
2. Load noisy_data.csv (with the date column) and strip the spaces from the column names.
|
|
6
|
+
3. Draw a boxplot of price to see the outliers before cleaning.
|
|
7
|
+
4. Convert the date column to the dd-mm-yy format.
|
|
8
|
+
5. Find the outliers of price and quantity with the IQR method and replace them with NaN.
|
|
9
|
+
6. Draw the boxplot again after cleaning.
|
|
10
|
+
7. Save the cleaned data to a csv file and print its shape and first 5 rows.
|
|
11
|
+
|
|
12
|
+
CODE:
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import numpy as np
|
|
15
|
+
import matplotlib.pyplot as plt
|
|
16
|
+
def load_and_clean_data():
|
|
17
|
+
try:
|
|
18
|
+
df=pd.read_csv('/content/sample_data/noisy_data.csv',parse_dates=['date'])
|
|
19
|
+
print('column detected succesfully')
|
|
20
|
+
except:
|
|
21
|
+
print('error finding file')
|
|
22
|
+
df.columns=df.columns.str.strip()
|
|
23
|
+
return df
|
|
24
|
+
print("Loading Data...")
|
|
25
|
+
df=load_and_clean_data()
|
|
26
|
+
print("\nOg Data (5 rows )")
|
|
27
|
+
print(df.head())
|
|
28
|
+
plt.figure(figsize=(6, 4))
|
|
29
|
+
plt.boxplot(df['price'])
|
|
30
|
+
plt.title("Boxplot - Product Prices (Before Cleaning)")
|
|
31
|
+
plt.ylabel("Price ($)")
|
|
32
|
+
plt.tight_layout()
|
|
33
|
+
plt.show()
|
|
34
|
+
print("\nConverting date format...")
|
|
35
|
+
df['date'] = pd.to_datetime(df['date'], format='%d-%m-%y')
|
|
36
|
+
print("\n Identifying and Removing Outliers..")
|
|
37
|
+
def remove_outliers(column):
|
|
38
|
+
column=pd.to_numeric(column,errors='coerce')
|
|
39
|
+
Q1=column.quantile(0.25)
|
|
40
|
+
Q3=column.quantile(0.75)
|
|
41
|
+
IQR=Q3-Q1
|
|
42
|
+
lower=Q1-1.5*IQR
|
|
43
|
+
upper=Q3+1.5*IQR
|
|
44
|
+
print(f"\nColumn statistics:Q1={Q1},Q3={Q3},IQR={IQR}")
|
|
45
|
+
print(f"Outlier bounds: lower={lower}, upper={upper}")
|
|
46
|
+
return column.apply(lambda x: np.nan if (x < lower or x > upper) else x)
|
|
47
|
+
price_no_outliers = remove_outliers(df['price'])
|
|
48
|
+
quantity_no_outliers = remove_outliers(df['quantity'])
|
|
49
|
+
df = df.assign(price=price_no_outliers, quantity=quantity_no_outliers)
|
|
50
|
+
plt.figure(figsize=(6, 4))
|
|
51
|
+
plt.boxplot(df['price'].dropna())
|
|
52
|
+
plt.title("Boxplot - Product Prices (After Cleaning)")
|
|
53
|
+
plt.ylabel("Price ($)")
|
|
54
|
+
plt.tight_layout()
|
|
55
|
+
plt.show()
|
|
56
|
+
cleaned_file_path = "/content/sample_data/cleaned_sales_data.csv"
|
|
57
|
+
df.to_csv(cleaned_file_path, index=False)
|
|
58
|
+
print("\nCleaning complete!")
|
|
59
|
+
print(f"Cleaned data saved to: {cleaned_file_path}")
|
|
60
|
+
print(f"\nFinal cleaned data shape: {df.shape}")
|
|
61
|
+
print("\nPreview of Cleaned Data (first 5 rows):")
|
|
62
|
+
print(df.head())
|
|
63
|
+
|
|
64
|
+
EXPLANATION:
|
|
65
|
+
- Noisy data means values that are wrong or far away from the rest (outliers), for example a very high price.
|
|
66
|
+
- The boxplot before cleaning shows the outliers as points outside the whiskers.
|
|
67
|
+
- remove_outliers() calculates Q1, Q3 and IQR and treats values below Q1-1.5*IQR or above Q3+1.5*IQR as outliers.
|
|
68
|
+
- The outliers are replaced with NaN, so the boxplot after cleaning is without the extreme points.
|
|
69
|
+
- The cleaned dataset is saved to cleaned_sales_data.csv.
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
PRACTICAL 3: DATA CLEANING ON DATASET (LOAN CREDIT SCORE)
|
|
2
|
+
|
|
3
|
+
Dataset taken: Loan_Credit_Score (loan_credit_dataset_240_rows_with_outliers.csv)
|
|
4
|
+
|
|
5
|
+
STEPS:
|
|
6
|
+
1. Install the libraries: pip install pandas numpy matplotlib
|
|
7
|
+
2. Load the loan credit dataset and view head(), info() and describe().
|
|
8
|
+
3. Check the missing values, duplicate rows and data types.
|
|
9
|
+
4. Draw a boxplot of each numeric column (before removing outliers).
|
|
10
|
+
5. Remove the outliers of each column using the IQR method.
|
|
11
|
+
6. Draw the boxplots again (after removing outliers).
|
|
12
|
+
|
|
13
|
+
CODE:
|
|
14
|
+
import pandas as pd
|
|
15
|
+
import numpy as np
|
|
16
|
+
import matplotlib.pyplot as plt
|
|
17
|
+
df=pd.read_csv('/content/sample_data/loan_credit_dataset_240_rows_with_outliers.csv')
|
|
18
|
+
print(df.head())
|
|
19
|
+
print(df.info())
|
|
20
|
+
print(df.describe())
|
|
21
|
+
print(df.isnull().sum()) #missing value
|
|
22
|
+
print("Duplicate rows:",df.duplicated().sum()) #duplicate values
|
|
23
|
+
print(df.dtypes)
|
|
24
|
+
#finding outliers in data
|
|
25
|
+
print("Before removing outliers:")
|
|
26
|
+
columns = [
|
|
27
|
+
"AnnualIncome",
|
|
28
|
+
"CreditScore",
|
|
29
|
+
"LoanAmount",
|
|
30
|
+
"DebtToIncomeRatio",
|
|
31
|
+
"EmploymentYears"
|
|
32
|
+
]
|
|
33
|
+
for col in columns:
|
|
34
|
+
plt.figure(figsize=(6,4))
|
|
35
|
+
plt.boxplot(df[col])
|
|
36
|
+
plt.title(f"Box Plot of {col}")
|
|
37
|
+
plt.ylabel(col)
|
|
38
|
+
plt.grid(True)
|
|
39
|
+
plt.show()
|
|
40
|
+
print("identifying and removing outliers")
|
|
41
|
+
def remove_outliers(column):
|
|
42
|
+
Q1=column.quantile(0.25)
|
|
43
|
+
Q3=column.quantile(0.75)
|
|
44
|
+
IQR=Q3-Q1
|
|
45
|
+
lower=Q1-1.5*IQR
|
|
46
|
+
upper=Q3+1.5*IQR
|
|
47
|
+
print(f"\nColumn statistics:Q1={Q1},Q3={Q3},IQR={IQR}")
|
|
48
|
+
print(f"Outlier bounds: lower={lower}, upper={upper}")
|
|
49
|
+
return column.apply(lambda x: np.nan if (x < lower or x > upper) else x)
|
|
50
|
+
for col in columns:
|
|
51
|
+
df[col] = remove_outliers(df[col])
|
|
52
|
+
print("After removing outliers:")
|
|
53
|
+
for col in columns:
|
|
54
|
+
plt.figure(figsize=(6,4))
|
|
55
|
+
plt.boxplot(df[col].dropna())
|
|
56
|
+
plt.title(f"Box Plot of {col}")
|
|
57
|
+
plt.ylabel(col)
|
|
58
|
+
plt.grid(True)
|
|
59
|
+
plt.show()
|
|
60
|
+
|
|
61
|
+
EXPLANATION:
|
|
62
|
+
- The first part inspects the dataset: shape of data, statistics, missing values, duplicates and data types.
|
|
63
|
+
- Five numeric columns (income, credit score, loan amount, debt-to-income ratio, employment years) are checked for outliers.
|
|
64
|
+
- Boxplots drawn before cleaning show the extreme values in each column.
|
|
65
|
+
- remove_outliers() uses the IQR rule (Q1-1.5*IQR, Q3+1.5*IQR) and replaces the outliers with NaN.
|
|
66
|
+
- The boxplots after cleaning show the same columns without the extreme values.
|
|
67
|
+
|
|
68
|
+
OUTPUT:
|
|
69
|
+
Before removing outliers: box plots with outlier points.
|
|
70
|
+
After removing outliers: box plots with the outliers removed.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
PRACTICAL 4: DATA PREPROCESSING - CORRELATION AND CONTINGENCY TABLE
|
|
2
|
+
|
|
3
|
+
STEPS:
|
|
4
|
+
1. Install the libraries: pip install pandas seaborn matplotlib scipy
|
|
5
|
+
2. Load Match.csv and select only the numeric columns.
|
|
6
|
+
3. Calculate the correlation matrix and draw it as a heatmap.
|
|
7
|
+
4. Create a contingency table of Toss_Decision against Match_Winner_Id.
|
|
8
|
+
5. Apply the Chi-square test on the table (statistic, p-value, degrees of freedom, expected frequencies).
|
|
9
|
+
6. If p-value < 0.05 the variables are dependent, otherwise they are independent.
|
|
10
|
+
|
|
11
|
+
CODE:
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import seaborn as sns
|
|
14
|
+
import matplotlib.pyplot as plt
|
|
15
|
+
from scipy.stats import chi2_contingency
|
|
16
|
+
df=pd.read_csv("Match.csv")
|
|
17
|
+
num_df=df.select_dtypes(include=["int64","float64"])
|
|
18
|
+
correlation_matrix=num_df.corr()
|
|
19
|
+
print("correlation matrix:\n",correlation_matrix)
|
|
20
|
+
plt.figure(figsize=(10,8))
|
|
21
|
+
sns.heatmap(correlation_matrix,annot=True,cmap="coolwarm",fmt=".2f")
|
|
22
|
+
plt.title("correlation matrix")
|
|
23
|
+
plt.show()
|
|
24
|
+
if 'Toss_Decision' in df.columns:
|
|
25
|
+
contingency_table=pd.crosstab(df['Toss_Decision'],df['Match_Winner_Id'])
|
|
26
|
+
print("Contigency Table:\n",contingency_table)
|
|
27
|
+
chi2,p,dof,expected=chi2_contingency(contingency_table)
|
|
28
|
+
print(f"\nChi-square Statistic: {chi2}")
|
|
29
|
+
print(f"P-value: {p}")
|
|
30
|
+
print(f"Degrees of Freedom: {dof}")
|
|
31
|
+
print("Expected Frequencies:\n",expected)
|
|
32
|
+
if p< 0.05:
|
|
33
|
+
print("\n Variables are dependent (reject H0)")
|
|
34
|
+
else:
|
|
35
|
+
print("\nVariables are independent (fail to reject H0) ")
|
|
36
|
+
|
|
37
|
+
EXPLANATION:
|
|
38
|
+
- corr() finds how strongly every pair of numeric columns is related (+1 strong positive, -1 strong negative, 0 none).
|
|
39
|
+
- The heatmap shows the correlation matrix with colours, so the related columns are easy to spot.
|
|
40
|
+
- pd.crosstab() builds a contingency table that counts matches for each toss decision and match winner.
|
|
41
|
+
- chi2_contingency() tests whether the two categorical variables are related.
|
|
42
|
+
- If the p-value is below 0.05 the variables are dependent (H0 rejected), otherwise they are independent.
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
PRACTICAL 5: EXPLORATORY DATA ANALYSIS (EDA)
|
|
2
|
+
|
|
3
|
+
STEPS:
|
|
4
|
+
1. Install the libraries: pip install pandas numpy seaborn matplotlib
|
|
5
|
+
2. Load Match.csv.
|
|
6
|
+
3. For every numeric column find mean, median, mode, skewness and kurtosis and tell the type of skew.
|
|
7
|
+
4. Plot the venue-wise wins of the teams at the top 10 venues (bar plot).
|
|
8
|
+
5. Plot the toss decision frequency (bar plot).
|
|
9
|
+
6. Map the team ids to team names and plot the home ground wins (pie chart).
|
|
10
|
+
7. Plot the venue-wise wins (scatter plot), winning margin by toss decision (box plot and violin plot).
|
|
11
|
+
8. Plot the frequency of win types (line plot) and winner by toss decision (swarm plot).
|
|
12
|
+
9. Plot the home wins as a donut chart and venue, toss decision and win type as a 3D scatter plot.
|
|
13
|
+
|
|
14
|
+
CODE:
|
|
15
|
+
from math import isclose
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import numpy as np
|
|
18
|
+
import seaborn as sns
|
|
19
|
+
import matplotlib.pyplot as plt
|
|
20
|
+
df=pd.read_csv("Match.csv")
|
|
21
|
+
num_cols=df.select_dtypes(include=[np.number]).columns
|
|
22
|
+
for col in num_cols:
|
|
23
|
+
mean_val=df[col].mean()
|
|
24
|
+
meadian_val=df[col].median()
|
|
25
|
+
mode_val=df[col].mode().iloc[0]
|
|
26
|
+
skew_val=df[col].skew()
|
|
27
|
+
kurtosis_val=df[col].kurtosis()
|
|
28
|
+
print(f"Mean of {col}: {mean_val}")
|
|
29
|
+
print(f"Median of {col}: {meadian_val}")
|
|
30
|
+
print(f"Mode of {col}: {mode_val}")
|
|
31
|
+
print(f"Skewness of {col}: {skew_val}")
|
|
32
|
+
print(f"Kurtosis of {col}: {kurtosis_val}")
|
|
33
|
+
print("\n")
|
|
34
|
+
if np.isclose(skew_val,0,atol=0.1):
|
|
35
|
+
print("Data is symmetric")
|
|
36
|
+
elif skew_val>0:
|
|
37
|
+
print("Data is positively skewed")
|
|
38
|
+
else:
|
|
39
|
+
print("Data is negatively skewed")
|
|
40
|
+
print("-"*70)
|
|
41
|
+
venue_wins = (
|
|
42
|
+
df.groupby(['Venue_Name', 'Match_Winner_Id'])
|
|
43
|
+
.size()
|
|
44
|
+
.reset_index(name='Wins')
|
|
45
|
+
)
|
|
46
|
+
# Select top 10 venues by number of matches
|
|
47
|
+
top_venues = df['Venue_Name'].value_counts().head(10).index
|
|
48
|
+
venue_wins = venue_wins[venue_wins['Venue_Name'].isin(top_venues)]
|
|
49
|
+
plt.figure(figsize=(14,7))
|
|
50
|
+
sns.barplot(
|
|
51
|
+
data=venue_wins,
|
|
52
|
+
x='Venue_Name',
|
|
53
|
+
y='Wins',
|
|
54
|
+
hue='Match_Winner_Id'
|
|
55
|
+
)
|
|
56
|
+
plt.xticks(rotation=45, ha='right')
|
|
57
|
+
plt.title('Venue Advantage: Wins by Team at Top Venues')
|
|
58
|
+
plt.xlabel('Venue')
|
|
59
|
+
plt.ylabel('Number of Wins')
|
|
60
|
+
plt.tight_layout()
|
|
61
|
+
plt.show()
|
|
62
|
+
toss_counts = df['Toss_Decision'].value_counts()
|
|
63
|
+
toss_counts.plot(
|
|
64
|
+
kind='bar',
|
|
65
|
+
color=['skyblue', 'orange'],
|
|
66
|
+
figsize=(6,5)
|
|
67
|
+
)
|
|
68
|
+
plt.title('Frequency of Toss Decisions')
|
|
69
|
+
plt.xlabel('Toss Decision')
|
|
70
|
+
plt.ylabel('Number of Matches')
|
|
71
|
+
for i, v in enumerate(toss_counts):
|
|
72
|
+
plt.text(i, v + 2, str(v), ha='center')
|
|
73
|
+
plt.show()
|
|
74
|
+
team_id_name_pairs = pd.concat([
|
|
75
|
+
df[['Opponent_Team_Id', 'Team name']].rename(columns={'Opponent_Team_Id': 'Team_ID', 'Team name': 'Team_Name'}),
|
|
76
|
+
df[['Opponent_Team_Id', 'OpponentTeam name']].rename(columns={'Opponent_Team_Id': 'Team_ID', 'OpponentTeam name': 'Team_Name'}),
|
|
77
|
+
df[['Match_Winner_Id', 'Team name']].rename(columns={'Match_Winner_Id': 'Team_ID', 'Team name': 'Team_Name'}),
|
|
78
|
+
df[['Match_Winner_Id', 'OpponentTeam name']].rename(columns={'Match_Winner_Id': 'Team_ID', 'OpponentTeam name': 'Team_Name'})
|
|
79
|
+
]).drop_duplicates().dropna(subset=['Team_ID', 'Team_Name'])
|
|
80
|
+
team_map = team_id_name_pairs.groupby('Team_ID')['Team_Name'].first()
|
|
81
|
+
# Add winner name column
|
|
82
|
+
df['Match_Winner_Id'] = df['Match_Winner_Id'].map(team_map)
|
|
83
|
+
# Home wins
|
|
84
|
+
home_wins = df[df['OpponentTeam name'] == df['Match_Winner_Id']]
|
|
85
|
+
home_win_counts = home_wins['OpponentTeam name'].value_counts()
|
|
86
|
+
plt.figure(figsize=(8,8))
|
|
87
|
+
plt.pie(
|
|
88
|
+
home_win_counts,
|
|
89
|
+
labels=home_win_counts.index,
|
|
90
|
+
autopct='%1.1f%%',
|
|
91
|
+
startangle=90
|
|
92
|
+
)
|
|
93
|
+
plt.title('Home Ground Wins by Team')
|
|
94
|
+
plt.axis('equal')
|
|
95
|
+
plt.show()
|
|
96
|
+
venue_wins = (
|
|
97
|
+
df.groupby(['Venue_Name', 'Match_Winner_Id'])
|
|
98
|
+
.size()
|
|
99
|
+
.reset_index(name='Wins')
|
|
100
|
+
)
|
|
101
|
+
# Top 10 venues
|
|
102
|
+
top_venues = df['Venue_Name'].value_counts().head(10).index
|
|
103
|
+
venue_wins = venue_wins[venue_wins['Venue_Name'].isin(top_venues)]
|
|
104
|
+
# Convert venue names to numeric positions
|
|
105
|
+
venue_positions = {venue: i for i, venue in enumerate(top_venues)}
|
|
106
|
+
venue_wins['Venue_Pos'] = venue_wins['Venue_Name'].map(venue_positions)
|
|
107
|
+
# Convert 'Match_Winner_Id' (team names) to numerical categories for plotting
|
|
108
|
+
venue_wins['Team_Numeric_ID'], unique_teams = pd.factorize(venue_wins['Match_Winner_Id'])
|
|
109
|
+
# Scatter Plot
|
|
110
|
+
plt.figure(figsize=(14,7))
|
|
111
|
+
scatter = plt.scatter(
|
|
112
|
+
venue_wins['Venue_Pos'],
|
|
113
|
+
venue_wins['Wins'],
|
|
114
|
+
s=120,
|
|
115
|
+
c=venue_wins['Team_Numeric_ID'], # Use numerical IDs for coloring
|
|
116
|
+
cmap='tab20',
|
|
117
|
+
alpha=0.8
|
|
118
|
+
)
|
|
119
|
+
plt.xticks(
|
|
120
|
+
range(len(top_venues)),
|
|
121
|
+
top_venues,
|
|
122
|
+
rotation=45,
|
|
123
|
+
ha='right'
|
|
124
|
+
)
|
|
125
|
+
plt.xlabel('Venue')
|
|
126
|
+
plt.ylabel('Number of Wins')
|
|
127
|
+
plt.title('Venue-wise Wins by Teams (Scatter Plot)')
|
|
128
|
+
# Create a custom colorbar to show team names
|
|
129
|
+
cbar = plt.colorbar(scatter, ticks=range(len(unique_teams)))
|
|
130
|
+
cbar.set_ticklabels(unique_teams)
|
|
131
|
+
cbar.set_label('Match Winner Team')
|
|
132
|
+
plt.tight_layout()
|
|
133
|
+
plt.show()
|
|
134
|
+
plt.figure(figsize=(8,6))
|
|
135
|
+
sns.boxplot(
|
|
136
|
+
data=df,
|
|
137
|
+
x='Toss_Decision',
|
|
138
|
+
y='Won_By',
|
|
139
|
+
palette='Set2'
|
|
140
|
+
)
|
|
141
|
+
plt.title('Winning Margin by Toss Decision')
|
|
142
|
+
plt.xlabel('Toss Decision')
|
|
143
|
+
plt.ylabel('Winning Margin')
|
|
144
|
+
plt.show()
|
|
145
|
+
win_counts = df['Win_Type'].value_counts().sort_index()
|
|
146
|
+
plt.figure(figsize=(8,5))
|
|
147
|
+
plt.plot(
|
|
148
|
+
win_counts.index,
|
|
149
|
+
win_counts.values,
|
|
150
|
+
marker='o',
|
|
151
|
+
linewidth=2
|
|
152
|
+
)
|
|
153
|
+
plt.title('Frequency of Win Types')
|
|
154
|
+
plt.xlabel('Win Type')
|
|
155
|
+
plt.ylabel('Number of Matches')
|
|
156
|
+
plt.grid(True)
|
|
157
|
+
plt.show()
|
|
158
|
+
plt.figure(figsize=(8,6))
|
|
159
|
+
sns.violinplot(
|
|
160
|
+
data=df,
|
|
161
|
+
x='Toss_Decision',
|
|
162
|
+
y='Won_By',
|
|
163
|
+
inner='quartile'
|
|
164
|
+
)
|
|
165
|
+
plt.title('Winning Margin by Toss Decision')
|
|
166
|
+
plt.xlabel('Toss Decision')
|
|
167
|
+
plt.ylabel('Winning Margin')
|
|
168
|
+
plt.show()
|
|
169
|
+
plt.figure(figsize=(10,6))
|
|
170
|
+
sns.swarmplot(
|
|
171
|
+
data=df,
|
|
172
|
+
x='Toss_Decision',
|
|
173
|
+
y='Match_Winner_Id',
|
|
174
|
+
size=5
|
|
175
|
+
)
|
|
176
|
+
plt.title('Match Winner IDs by Toss Decision')
|
|
177
|
+
plt.xlabel('Toss Decision')
|
|
178
|
+
plt.ylabel('Match Winner ID')
|
|
179
|
+
plt.show()
|
|
180
|
+
team_id_name_pairs = pd.concat([
|
|
181
|
+
df[['Opponent_Team_Id', 'Team name']].rename(columns={'Opponent_Team_Id': 'Team_ID', 'Team name': 'Team_Name'}),
|
|
182
|
+
df[['Opponent_Team_Id', 'OpponentTeam name']].rename(columns={'Opponent_Team_Id': 'Team_ID', 'OpponentTeam name': 'Team_Name'}),
|
|
183
|
+
df[['Match_Winner_Id', 'Team name']].rename(columns={'Match_Winner_Id': 'Team_ID', 'Team name': 'Team_Name'}),
|
|
184
|
+
df[['Match_Winner_Id', 'OpponentTeam name']].rename(columns={'Match_Winner_Id': 'Team_ID', 'OpponentTeam name': 'Team_Name'})
|
|
185
|
+
]).drop_duplicates().dropna(subset=['Team_ID', 'Team_Name'])
|
|
186
|
+
team_map = team_id_name_pairs.groupby('Team_ID')['Team_Name'].first()
|
|
187
|
+
# Map winner IDs to team names
|
|
188
|
+
df['Match_Winner_Name'] = df['Match_Winner_Id'].map(team_map)
|
|
189
|
+
# Home wins
|
|
190
|
+
home_wins = df[df['OpponentTeam name'] == df['Match_Winner_Name']]
|
|
191
|
+
home_win_counts = home_wins['OpponentTeam name'].value_counts()
|
|
192
|
+
plt.figure(figsize=(9,9))
|
|
193
|
+
plt.pie(
|
|
194
|
+
home_win_counts,
|
|
195
|
+
labels=home_win_counts.index,
|
|
196
|
+
autopct='%1.1f%%',
|
|
197
|
+
startangle=90,
|
|
198
|
+
pctdistance=0.8,
|
|
199
|
+
wedgeprops=dict(width=0.4, edgecolor='white')
|
|
200
|
+
)
|
|
201
|
+
plt.text(
|
|
202
|
+
0, 0,
|
|
203
|
+
"Home\nWins",
|
|
204
|
+
ha='center',
|
|
205
|
+
va='center',
|
|
206
|
+
fontsize=16,
|
|
207
|
+
fontweight='bold'
|
|
208
|
+
)
|
|
209
|
+
plt.title('Home Ground Wins by Team', fontsize=16, fontweight='bold')
|
|
210
|
+
plt.axis('equal')
|
|
211
|
+
plt.tight_layout()
|
|
212
|
+
plt.show()
|
|
213
|
+
df['Venue_Code'] = df['Venue_Name'].astype('category').cat.codes
|
|
214
|
+
df['Toss_Code'] = df['Toss_Decision'].astype('category').cat.codes
|
|
215
|
+
df['WinType_Code'] = df['Win_Type'].astype('category').cat.codes
|
|
216
|
+
# Create 3D Scatter Plot
|
|
217
|
+
fig = plt.figure(figsize=(12,8))
|
|
218
|
+
ax = fig.add_subplot(111, projection='3d')
|
|
219
|
+
scatter = ax.scatter(
|
|
220
|
+
df['Venue_Code'],
|
|
221
|
+
df['Toss_Code'],
|
|
222
|
+
df['WinType_Code'],
|
|
223
|
+
c=df['WinType_Code'], # Color by Win Type
|
|
224
|
+
cmap='viridis',
|
|
225
|
+
s=40,
|
|
226
|
+
alpha=0.7
|
|
227
|
+
)
|
|
228
|
+
ax.set_title('3D Scatter Plot of Venue, Toss Decision and Win Type')
|
|
229
|
+
ax.set_xlabel('Venue')
|
|
230
|
+
ax.set_ylabel('Toss Decision')
|
|
231
|
+
ax.set_zlabel('Win Type')
|
|
232
|
+
# Show original category names on axes
|
|
233
|
+
venue_labels = df['Venue_Name'].astype('category').cat.categories
|
|
234
|
+
toss_labels = df['Toss_Decision'].astype('category').cat.categories
|
|
235
|
+
win_labels = df['Win_Type'].astype('category').cat.categories
|
|
236
|
+
# Limit x-axis labels to avoid clutter (top 10 venues)
|
|
237
|
+
ax.set_xticks(range(min(10, len(venue_labels))))
|
|
238
|
+
ax.set_xticklabels(venue_labels[:10], rotation=45, ha='right')
|
|
239
|
+
ax.set_yticks(range(len(toss_labels)))
|
|
240
|
+
ax.set_yticklabels(toss_labels)
|
|
241
|
+
ax.set_zticks(range(len(win_labels)))
|
|
242
|
+
ax.set_zticklabels(win_labels)
|
|
243
|
+
plt.colorbar(scatter, ax=ax, label='Win Type Code')
|
|
244
|
+
plt.tight_layout()
|
|
245
|
+
plt.show()
|
|
246
|
+
|
|
247
|
+
EXPLANATION:
|
|
248
|
+
- The first loop gives the mean, median, mode, skewness and kurtosis of every numeric column and tells if the data is symmetric, positively or negatively skewed.
|
|
249
|
+
- Bar, pie and donut charts show the categorical counts: wins at the top venues, toss decisions and home ground wins.
|
|
250
|
+
- A team-id to team-name map is built so the charts show names instead of ids.
|
|
251
|
+
- Box plot and violin plot compare the winning margin for each toss decision, the swarm plot shows each winner individually.
|
|
252
|
+
- The line plot shows win-type frequency and the 3D scatter plot shows venue, toss decision and win type together.
|