pyspan 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyspan-0.1.0/LICENSE +21 -0
- pyspan-0.1.0/PKG-INFO +201 -0
- pyspan-0.1.0/README.md +179 -0
- pyspan-0.1.0/pyspan.egg-info/PKG-INFO +201 -0
- pyspan-0.1.0/pyspan.egg-info/SOURCES.txt +9 -0
- pyspan-0.1.0/pyspan.egg-info/dependency_links.txt +1 -0
- pyspan-0.1.0/pyspan.egg-info/requires.txt +8 -0
- pyspan-0.1.0/pyspan.egg-info/top_level.txt +1 -0
- pyspan-0.1.0/pyspan.py +840 -0
- pyspan-0.1.0/setup.cfg +4 -0
- pyspan-0.1.0/setup.py +29 -0
pyspan-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 pyspan24
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pyspan-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pyspan
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Python package for efficient data cleaning and preprocessing with Pandas.
|
|
5
|
+
Home-page: https://github.com/yourusername/my_simple_lib
|
|
6
|
+
Author: Noor Surani, Amynah Reimoo
|
|
7
|
+
Author-email: nsurani@hotmail.com, amynahreimoo@gmail.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.12
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy==2.0.1
|
|
15
|
+
Requires-Dist: pandas==2.2.2
|
|
16
|
+
Requires-Dist: pyspellchecker==0.8.1
|
|
17
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
18
|
+
Requires-Dist: pytz==2024.1
|
|
19
|
+
Requires-Dist: setuptools==72.2.0
|
|
20
|
+
Requires-Dist: six==1.16.0
|
|
21
|
+
Requires-Dist: tzdata==2024.1
|
|
22
|
+
|
|
23
|
+
# pyspan
|
|
24
|
+
|
|
25
|
+
## 'pyspan' is a Python package designed to facilitate data cleaning and preprocessing using Pandas. It provides various functions to handle missing values, detect outliers, spell check data, and more. Additionally, it includes a logging utility to keep track of function calls and their parameters.
|
|
26
|
+
|
|
27
|
+
Installation
|
|
28
|
+
|
|
29
|
+
To use pyspan, simply install the package using pip:
|
|
30
|
+
|
|
31
|
+
``bash
|
|
32
|
+
pip install pyspan
|
|
33
|
+
|
|
34
|
+
# Functions
|
|
35
|
+
|
|
36
|
+
1. handle_nulls(data: pd.DataFrame, columns: List[str], method: str, value: Optional[Union[str, float]] = None) -> pd.DataFrame
|
|
37
|
+
Handles missing values in the specified columns of a DataFrame.
|
|
38
|
+
Parameters:
|
|
39
|
+
data: DataFrame with missing values.
|
|
40
|
+
columns: List of column names to apply the fill operation.
|
|
41
|
+
method: Strategy to use for imputing missing values ('mean', 'median', 'mode', 'interpolate', 'forward_fill', 'backward_fill').
|
|
42
|
+
value: Custom value to fill NaNs with (optional).
|
|
43
|
+
|
|
44
|
+
2. remove_duplicates(df: pd.DataFrame) -> pd.DataFrame
|
|
45
|
+
Removes duplicate rows from a DataFrame.
|
|
46
|
+
Parameters:
|
|
47
|
+
df: DataFrame to remove duplicates from.
|
|
48
|
+
|
|
49
|
+
3. remove_columns(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame
|
|
50
|
+
Removes specified columns from a DataFrame.
|
|
51
|
+
Parameters:
|
|
52
|
+
df: DataFrame to remove columns from.
|
|
53
|
+
columns: List of column names to remove.
|
|
54
|
+
|
|
55
|
+
4. auto_rename_columns(df: pd.DataFrame) -> pd.DataFrame
|
|
56
|
+
Automatically renames columns to remove spaces and special characters.
|
|
57
|
+
Parameters:
|
|
58
|
+
df: DataFrame to rename columns in.
|
|
59
|
+
|
|
60
|
+
5. rename_dataframe_columns(df: pd.DataFrame, rename_dict: dict) -> pd.DataFrame
|
|
61
|
+
Renames columns in a DataFrame using a provided dictionary mapping.
|
|
62
|
+
Parameters:
|
|
63
|
+
df: DataFrame to rename columns in.
|
|
64
|
+
rename_dict: Dictionary mapping current column names to new column names.
|
|
65
|
+
|
|
66
|
+
6. change_dt(df: pd.DataFrame, columns: Union[str, List[str]], date_format: str = "%Y-%m-%d", time_format: str = "%H:%M:%S", from_timezone: Optional[str] = None, to_timezone: Optional[str] = None) -> pd.DataFrame
|
|
67
|
+
Changes the format of date and time columns and handles timezone conversion.
|
|
68
|
+
Parameters:
|
|
69
|
+
df: DataFrame containing date/time columns to reformat.
|
|
70
|
+
columns: Name(s) of the column(s) to be reformatted.
|
|
71
|
+
date_format: Desired date format.
|
|
72
|
+
time_format: Desired time format.
|
|
73
|
+
from_timezone: Original timezone of the datetime column(s).
|
|
74
|
+
to_timezone: Desired timezone for the datetime column(s).
|
|
75
|
+
|
|
76
|
+
7. detect_delimiter(series: pd.Series) -> str
|
|
77
|
+
Detects the most common delimiter in a Series of strings.
|
|
78
|
+
Parameters:
|
|
79
|
+
series: Series containing strings to analyze.
|
|
80
|
+
|
|
81
|
+
8. split_column(df: pd.DataFrame, column_name: str, delimiter: str = None) -> pd.DataFrame
|
|
82
|
+
Splits a single column into multiple columns based on a delimiter.
|
|
83
|
+
Parameters:
|
|
84
|
+
df: DataFrame containing the column to split.
|
|
85
|
+
column_name: Name of the column to be split.
|
|
86
|
+
delimiter: Delimiter to use for splitting (optional).
|
|
87
|
+
|
|
88
|
+
9. impute(data, by=None, value=None, columns=None) -> pd.DataFrame
|
|
89
|
+
Handles missing values using specified strategy or custom value.
|
|
90
|
+
Parameters:
|
|
91
|
+
data: DataFrame or Series with missing values.
|
|
92
|
+
by: Strategy for imputing missing values ('mean', 'median', 'mode', 'interpolate', 'forward_fill', 'backward_fill').
|
|
93
|
+
value: Custom value to fill NaNs with.
|
|
94
|
+
columns: List of column names to apply the fill operation.
|
|
95
|
+
|
|
96
|
+
10. spell_check_dataframe(data: pd.DataFrame, dictionary='en_US', columns=None) -> dict
|
|
97
|
+
Performs spell check on specified columns of a DataFrame.
|
|
98
|
+
Parameters:
|
|
99
|
+
data: DataFrame containing columns to spell check.
|
|
100
|
+
dictionary: Dictionary to use for spell checking ('en_US', 'en_GB', 'en_AU', 'en_IE').
|
|
101
|
+
columns: List of column names to perform spell check on.
|
|
102
|
+
|
|
103
|
+
11. detect_invalid_dates(series: pd.Series) -> pd.Series
|
|
104
|
+
Detects invalid date values in a Series.
|
|
105
|
+
Parameters:
|
|
106
|
+
series: Series to check for invalid dates.
|
|
107
|
+
|
|
108
|
+
12. detect_data_entry_errors(data, spellcheck_dict='en_US', date_columns=None, numeric_columns=None, text_columns=None) -> pd.DataFrame
|
|
109
|
+
Detects and flags data entry errors including invalid dates and misspelled words.
|
|
110
|
+
Parameters:
|
|
111
|
+
data: DataFrame to analyze.
|
|
112
|
+
spellcheck_dict: Dictionary to use for spell checking.
|
|
113
|
+
date_columns: List of columns to check for invalid dates.
|
|
114
|
+
numeric_columns: List of columns to check for numeric format errors.
|
|
115
|
+
text_columns: List of text columns to perform spell checking on.
|
|
116
|
+
|
|
117
|
+
13. data_type_conversions(data, column=None) -> pd.DataFrame or pd.Series
|
|
118
|
+
Recommends and applies data type conversions based on the analysis of each column's data.
|
|
119
|
+
Parameters:
|
|
120
|
+
data: DataFrame or Series to analyze.
|
|
121
|
+
column: Specific column to analyze (optional).
|
|
122
|
+
|
|
123
|
+
14. detect_outliers(data, method='iqr', threshold=1.5, columns=None, handle_missing=True) -> pd.DataFrame
|
|
124
|
+
Detects outliers in a dataset using specified method and threshold.
|
|
125
|
+
Parameters:
|
|
126
|
+
data: DataFrame or Series to analyze.
|
|
127
|
+
method: Outlier detection method ('z-score', 'iqr').
|
|
128
|
+
threshold: Threshold for outlier detection.
|
|
129
|
+
columns: List of columns to apply the outlier detection on (optional).
|
|
130
|
+
handle_missing: Whether to handle missing values by dropping them or not.
|
|
131
|
+
|
|
132
|
+
Logging Functions
|
|
133
|
+
1. display_logs()
|
|
134
|
+
Prints stored log entries.
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
## Example Usage
|
|
138
|
+
Here are some examples to illustrate the usage of the functions provided in pyspan:
|
|
139
|
+
|
|
140
|
+
import pandas as pd
|
|
141
|
+
from pyspan import handle_nulls, remove_duplicates, remove_columns, auto_rename_columns, rename_dataframe_columns
|
|
142
|
+
from pyspan import change_dt, detect_delimiter, split_column, impute, spell_check_dataframe
|
|
143
|
+
from pyspan import detect_invalid_dates, detect_data_entry_errors, data_type_conversions, detect_outliers
|
|
144
|
+
from pyspan import log_function_call, display_logs
|
|
145
|
+
|
|
146
|
+
# Load a dataset
|
|
147
|
+
df = pd.read_csv('/content/GlobalSharkAttacks.csv')
|
|
148
|
+
|
|
149
|
+
# Example usage of handle_nulls
|
|
150
|
+
df_filled = handle_nulls(df, columns=['Column1', 'Column2'], method='mean')
|
|
151
|
+
|
|
152
|
+
# Example usage of remove_duplicates
|
|
153
|
+
df_unique = remove_duplicates(df)
|
|
154
|
+
|
|
155
|
+
# Example usage of remove_columns
|
|
156
|
+
df_reduced = remove_columns(df, columns=['ColumnToRemove'])
|
|
157
|
+
|
|
158
|
+
# Example usage of auto_rename_columns
|
|
159
|
+
auto_rename_columns(df)
|
|
160
|
+
|
|
161
|
+
# Example usage of rename_dataframe_columns
|
|
162
|
+
rename_dict = {'OldName': 'NewName'}
|
|
163
|
+
df_renamed_dict = rename_dataframe_columns(df, rename_dict)
|
|
164
|
+
|
|
165
|
+
# Example usage of change_dt
|
|
166
|
+
df_formatted = change_dt(df, columns=['Date'], date_format='%d-%m-%Y', time_format='%I:%M %p')
|
|
167
|
+
|
|
168
|
+
# Example usage of detect_delimiter
|
|
169
|
+
delimiter = detect_delimiter(df['ColumnWithDelimiters'])
|
|
170
|
+
|
|
171
|
+
# Example usage of split_column
|
|
172
|
+
df_split = split_column(df, column_name='ColumnWithDelimiters')
|
|
173
|
+
|
|
174
|
+
# Example usage of impute
|
|
175
|
+
df_imputed = impute(df, by='mean', columns=['Column1'])
|
|
176
|
+
|
|
177
|
+
# Example usage of spell_check_dataframe
|
|
178
|
+
misspelled = spell_check_dataframe(df, dictionary='en_US', columns=['TextColumn'])
|
|
179
|
+
|
|
180
|
+
# Example usage of detect_invalid_dates
|
|
181
|
+
invalid_dates = detect_invalid_dates(df['DateColumn'])
|
|
182
|
+
|
|
183
|
+
# Example usage of detect_data_entry_errors
|
|
184
|
+
errors = detect_data_entry_errors(df, spellcheck_dict='en_US', date_columns=['DateColumn'], numeric_columns=['NumericColumn'])
|
|
185
|
+
|
|
186
|
+
# Example usage of data_type_conversions
|
|
187
|
+
df_converted = data_type_conversions(df)
|
|
188
|
+
|
|
189
|
+
# Example usage of detect_outliers
|
|
190
|
+
outliers = detect_outliers(df, method='iqr', threshold=1.5)
|
|
191
|
+
|
|
192
|
+
# Example usage of display_logs
|
|
193
|
+
display_logs()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# License
|
|
197
|
+
This package is licensed under the MIT License. See the LICENSE file for more details.
|
|
198
|
+
|
|
199
|
+
# Contact
|
|
200
|
+
For issues or questions, please contact [amynahreimoo@gmail.com].
|
|
201
|
+
|
pyspan-0.1.0/README.md
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# pyspan
|
|
2
|
+
|
|
3
|
+
## 'pyspan' is a Python package designed to facilitate data cleaning and preprocessing using Pandas. It provides various functions to handle missing values, detect outliers, spell check data, and more. Additionally, it includes a logging utility to keep track of function calls and their parameters.
|
|
4
|
+
|
|
5
|
+
Installation
|
|
6
|
+
|
|
7
|
+
To use pyspan, simply install the package using pip:
|
|
8
|
+
|
|
9
|
+
``bash
|
|
10
|
+
pip install pyspan
|
|
11
|
+
|
|
12
|
+
# Functions
|
|
13
|
+
|
|
14
|
+
1. handle_nulls(data: pd.DataFrame, columns: List[str], method: str, value: Optional[Union[str, float]] = None) -> pd.DataFrame
|
|
15
|
+
Handles missing values in the specified columns of a DataFrame.
|
|
16
|
+
Parameters:
|
|
17
|
+
data: DataFrame with missing values.
|
|
18
|
+
columns: List of column names to apply the fill operation.
|
|
19
|
+
method: Strategy to use for imputing missing values ('mean', 'median', 'mode', 'interpolate', 'forward_fill', 'backward_fill').
|
|
20
|
+
value: Custom value to fill NaNs with (optional).
|
|
21
|
+
|
|
22
|
+
2. remove_duplicates(df: pd.DataFrame) -> pd.DataFrame
|
|
23
|
+
Removes duplicate rows from a DataFrame.
|
|
24
|
+
Parameters:
|
|
25
|
+
df: DataFrame to remove duplicates from.
|
|
26
|
+
|
|
27
|
+
3. remove_columns(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame
|
|
28
|
+
Removes specified columns from a DataFrame.
|
|
29
|
+
Parameters:
|
|
30
|
+
df: DataFrame to remove columns from.
|
|
31
|
+
columns: List of column names to remove.
|
|
32
|
+
|
|
33
|
+
4. auto_rename_columns(df: pd.DataFrame) -> pd.DataFrame
|
|
34
|
+
Automatically renames columns to remove spaces and special characters.
|
|
35
|
+
Parameters:
|
|
36
|
+
df: DataFrame to rename columns in.
|
|
37
|
+
|
|
38
|
+
5. rename_dataframe_columns(df: pd.DataFrame, rename_dict: dict) -> pd.DataFrame
|
|
39
|
+
Renames columns in a DataFrame using a provided dictionary mapping.
|
|
40
|
+
Parameters:
|
|
41
|
+
df: DataFrame to rename columns in.
|
|
42
|
+
rename_dict: Dictionary mapping current column names to new column names.
|
|
43
|
+
|
|
44
|
+
6. change_dt(df: pd.DataFrame, columns: Union[str, List[str]], date_format: str = "%Y-%m-%d", time_format: str = "%H:%M:%S", from_timezone: Optional[str] = None, to_timezone: Optional[str] = None) -> pd.DataFrame
|
|
45
|
+
Changes the format of date and time columns and handles timezone conversion.
|
|
46
|
+
Parameters:
|
|
47
|
+
df: DataFrame containing date/time columns to reformat.
|
|
48
|
+
columns: Name(s) of the column(s) to be reformatted.
|
|
49
|
+
date_format: Desired date format.
|
|
50
|
+
time_format: Desired time format.
|
|
51
|
+
from_timezone: Original timezone of the datetime column(s).
|
|
52
|
+
to_timezone: Desired timezone for the datetime column(s).
|
|
53
|
+
|
|
54
|
+
7. detect_delimiter(series: pd.Series) -> str
|
|
55
|
+
Detects the most common delimiter in a Series of strings.
|
|
56
|
+
Parameters:
|
|
57
|
+
series: Series containing strings to analyze.
|
|
58
|
+
|
|
59
|
+
8. split_column(df: pd.DataFrame, column_name: str, delimiter: str = None) -> pd.DataFrame
|
|
60
|
+
Splits a single column into multiple columns based on a delimiter.
|
|
61
|
+
Parameters:
|
|
62
|
+
df: DataFrame containing the column to split.
|
|
63
|
+
column_name: Name of the column to be split.
|
|
64
|
+
delimiter: Delimiter to use for splitting (optional).
|
|
65
|
+
|
|
66
|
+
9. impute(data, by=None, value=None, columns=None) -> pd.DataFrame
|
|
67
|
+
Handles missing values using specified strategy or custom value.
|
|
68
|
+
Parameters:
|
|
69
|
+
data: DataFrame or Series with missing values.
|
|
70
|
+
by: Strategy for imputing missing values ('mean', 'median', 'mode', 'interpolate', 'forward_fill', 'backward_fill').
|
|
71
|
+
value: Custom value to fill NaNs with.
|
|
72
|
+
columns: List of column names to apply the fill operation.
|
|
73
|
+
|
|
74
|
+
10. spell_check_dataframe(data: pd.DataFrame, dictionary='en_US', columns=None) -> dict
|
|
75
|
+
Performs spell check on specified columns of a DataFrame.
|
|
76
|
+
Parameters:
|
|
77
|
+
data: DataFrame containing columns to spell check.
|
|
78
|
+
dictionary: Dictionary to use for spell checking ('en_US', 'en_GB', 'en_AU', 'en_IE').
|
|
79
|
+
columns: List of column names to perform spell check on.
|
|
80
|
+
|
|
81
|
+
11. detect_invalid_dates(series: pd.Series) -> pd.Series
|
|
82
|
+
Detects invalid date values in a Series.
|
|
83
|
+
Parameters:
|
|
84
|
+
series: Series to check for invalid dates.
|
|
85
|
+
|
|
86
|
+
12. detect_data_entry_errors(data, spellcheck_dict='en_US', date_columns=None, numeric_columns=None, text_columns=None) -> pd.DataFrame
|
|
87
|
+
Detects and flags data entry errors including invalid dates and misspelled words.
|
|
88
|
+
Parameters:
|
|
89
|
+
data: DataFrame to analyze.
|
|
90
|
+
spellcheck_dict: Dictionary to use for spell checking.
|
|
91
|
+
date_columns: List of columns to check for invalid dates.
|
|
92
|
+
numeric_columns: List of columns to check for numeric format errors.
|
|
93
|
+
text_columns: List of text columns to perform spell checking on.
|
|
94
|
+
|
|
95
|
+
13. data_type_conversions(data, column=None) -> pd.DataFrame or pd.Series
|
|
96
|
+
Recommends and applies data type conversions based on the analysis of each column's data.
|
|
97
|
+
Parameters:
|
|
98
|
+
data: DataFrame or Series to analyze.
|
|
99
|
+
column: Specific column to analyze (optional).
|
|
100
|
+
|
|
101
|
+
14. detect_outliers(data, method='iqr', threshold=1.5, columns=None, handle_missing=True) -> pd.DataFrame
|
|
102
|
+
Detects outliers in a dataset using specified method and threshold.
|
|
103
|
+
Parameters:
|
|
104
|
+
data: DataFrame or Series to analyze.
|
|
105
|
+
method: Outlier detection method ('z-score', 'iqr').
|
|
106
|
+
threshold: Threshold for outlier detection.
|
|
107
|
+
columns: List of columns to apply the outlier detection on (optional).
|
|
108
|
+
handle_missing: Whether to handle missing values by dropping them or not.
|
|
109
|
+
|
|
110
|
+
Logging Functions
|
|
111
|
+
1. display_logs()
|
|
112
|
+
Prints stored log entries.
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
## Example Usage
|
|
116
|
+
Here are some examples to illustrate the usage of the functions provided in pyspan:
|
|
117
|
+
|
|
118
|
+
import pandas as pd
|
|
119
|
+
from pyspan import handle_nulls, remove_duplicates, remove_columns, auto_rename_columns, rename_dataframe_columns
|
|
120
|
+
from pyspan import change_dt, detect_delimiter, split_column, impute, spell_check_dataframe
|
|
121
|
+
from pyspan import detect_invalid_dates, detect_data_entry_errors, data_type_conversions, detect_outliers
|
|
122
|
+
from pyspan import log_function_call, display_logs
|
|
123
|
+
|
|
124
|
+
# Load a dataset
|
|
125
|
+
df = pd.read_csv('/content/GlobalSharkAttacks.csv')
|
|
126
|
+
|
|
127
|
+
# Example usage of handle_nulls
|
|
128
|
+
df_filled = handle_nulls(df, columns=['Column1', 'Column2'], method='mean')
|
|
129
|
+
|
|
130
|
+
# Example usage of remove_duplicates
|
|
131
|
+
df_unique = remove_duplicates(df)
|
|
132
|
+
|
|
133
|
+
# Example usage of remove_columns
|
|
134
|
+
df_reduced = remove_columns(df, columns=['ColumnToRemove'])
|
|
135
|
+
|
|
136
|
+
# Example usage of auto_rename_columns
|
|
137
|
+
auto_rename_columns(df)
|
|
138
|
+
|
|
139
|
+
# Example usage of rename_dataframe_columns
|
|
140
|
+
rename_dict = {'OldName': 'NewName'}
|
|
141
|
+
df_renamed_dict = rename_dataframe_columns(df, rename_dict)
|
|
142
|
+
|
|
143
|
+
# Example usage of change_dt
|
|
144
|
+
df_formatted = change_dt(df, columns=['Date'], date_format='%d-%m-%Y', time_format='%I:%M %p')
|
|
145
|
+
|
|
146
|
+
# Example usage of detect_delimiter
|
|
147
|
+
delimiter = detect_delimiter(df['ColumnWithDelimiters'])
|
|
148
|
+
|
|
149
|
+
# Example usage of split_column
|
|
150
|
+
df_split = split_column(df, column_name='ColumnWithDelimiters')
|
|
151
|
+
|
|
152
|
+
# Example usage of impute
|
|
153
|
+
df_imputed = impute(df, by='mean', columns=['Column1'])
|
|
154
|
+
|
|
155
|
+
# Example usage of spell_check_dataframe
|
|
156
|
+
misspelled = spell_check_dataframe(df, dictionary='en_US', columns=['TextColumn'])
|
|
157
|
+
|
|
158
|
+
# Example usage of detect_invalid_dates
|
|
159
|
+
invalid_dates = detect_invalid_dates(df['DateColumn'])
|
|
160
|
+
|
|
161
|
+
# Example usage of detect_data_entry_errors
|
|
162
|
+
errors = detect_data_entry_errors(df, spellcheck_dict='en_US', date_columns=['DateColumn'], numeric_columns=['NumericColumn'])
|
|
163
|
+
|
|
164
|
+
# Example usage of data_type_conversions
|
|
165
|
+
df_converted = data_type_conversions(df)
|
|
166
|
+
|
|
167
|
+
# Example usage of detect_outliers
|
|
168
|
+
outliers = detect_outliers(df, method='iqr', threshold=1.5)
|
|
169
|
+
|
|
170
|
+
# Example usage of display_logs
|
|
171
|
+
display_logs()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
# License
|
|
175
|
+
This package is licensed under the MIT License. See the LICENSE file for more details.
|
|
176
|
+
|
|
177
|
+
# Contact
|
|
178
|
+
For issues or questions, please contact [amynahreimoo@gmail.com].
|
|
179
|
+
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pyspan
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Python package for efficient data cleaning and preprocessing with Pandas.
|
|
5
|
+
Home-page: https://github.com/yourusername/my_simple_lib
|
|
6
|
+
Author: Noor Surani, Amynah Reimoo
|
|
7
|
+
Author-email: nsurani@hotmail.com, amynahreimoo@gmail.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.12
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy==2.0.1
|
|
15
|
+
Requires-Dist: pandas==2.2.2
|
|
16
|
+
Requires-Dist: pyspellchecker==0.8.1
|
|
17
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
18
|
+
Requires-Dist: pytz==2024.1
|
|
19
|
+
Requires-Dist: setuptools==72.2.0
|
|
20
|
+
Requires-Dist: six==1.16.0
|
|
21
|
+
Requires-Dist: tzdata==2024.1
|
|
22
|
+
|
|
23
|
+
# pyspan
|
|
24
|
+
|
|
25
|
+
## 'pyspan' is a Python package designed to facilitate data cleaning and preprocessing using Pandas. It provides various functions to handle missing values, detect outliers, spell check data, and more. Additionally, it includes a logging utility to keep track of function calls and their parameters.
|
|
26
|
+
|
|
27
|
+
Installation
|
|
28
|
+
|
|
29
|
+
To use pyspan, simply install the package using pip:
|
|
30
|
+
|
|
31
|
+
``bash
|
|
32
|
+
pip install pyspan
|
|
33
|
+
|
|
34
|
+
# Functions
|
|
35
|
+
|
|
36
|
+
1. handle_nulls(data: pd.DataFrame, columns: List[str], method: str, value: Optional[Union[str, float]] = None) -> pd.DataFrame
|
|
37
|
+
Handles missing values in the specified columns of a DataFrame.
|
|
38
|
+
Parameters:
|
|
39
|
+
data: DataFrame with missing values.
|
|
40
|
+
columns: List of column names to apply the fill operation.
|
|
41
|
+
method: Strategy to use for imputing missing values ('mean', 'median', 'mode', 'interpolate', 'forward_fill', 'backward_fill').
|
|
42
|
+
value: Custom value to fill NaNs with (optional).
|
|
43
|
+
|
|
44
|
+
2. remove_duplicates(df: pd.DataFrame) -> pd.DataFrame
|
|
45
|
+
Removes duplicate rows from a DataFrame.
|
|
46
|
+
Parameters:
|
|
47
|
+
df: DataFrame to remove duplicates from.
|
|
48
|
+
|
|
49
|
+
3. remove_columns(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame
|
|
50
|
+
Removes specified columns from a DataFrame.
|
|
51
|
+
Parameters:
|
|
52
|
+
df: DataFrame to remove columns from.
|
|
53
|
+
columns: List of column names to remove.
|
|
54
|
+
|
|
55
|
+
4. auto_rename_columns(df: pd.DataFrame) -> pd.DataFrame
|
|
56
|
+
Automatically renames columns to remove spaces and special characters.
|
|
57
|
+
Parameters:
|
|
58
|
+
df: DataFrame to rename columns in.
|
|
59
|
+
|
|
60
|
+
5. rename_dataframe_columns(df: pd.DataFrame, rename_dict: dict) -> pd.DataFrame
|
|
61
|
+
Renames columns in a DataFrame using a provided dictionary mapping.
|
|
62
|
+
Parameters:
|
|
63
|
+
df: DataFrame to rename columns in.
|
|
64
|
+
rename_dict: Dictionary mapping current column names to new column names.
|
|
65
|
+
|
|
66
|
+
6. change_dt(df: pd.DataFrame, columns: Union[str, List[str]], date_format: str = "%Y-%m-%d", time_format: str = "%H:%M:%S", from_timezone: Optional[str] = None, to_timezone: Optional[str] = None) -> pd.DataFrame
|
|
67
|
+
Changes the format of date and time columns and handles timezone conversion.
|
|
68
|
+
Parameters:
|
|
69
|
+
df: DataFrame containing date/time columns to reformat.
|
|
70
|
+
columns: Name(s) of the column(s) to be reformatted.
|
|
71
|
+
date_format: Desired date format.
|
|
72
|
+
time_format: Desired time format.
|
|
73
|
+
from_timezone: Original timezone of the datetime column(s).
|
|
74
|
+
to_timezone: Desired timezone for the datetime column(s).
|
|
75
|
+
|
|
76
|
+
7. detect_delimiter(series: pd.Series) -> str
|
|
77
|
+
Detects the most common delimiter in a Series of strings.
|
|
78
|
+
Parameters:
|
|
79
|
+
series: Series containing strings to analyze.
|
|
80
|
+
|
|
81
|
+
8. split_column(df: pd.DataFrame, column_name: str, delimiter: str = None) -> pd.DataFrame
|
|
82
|
+
Splits a single column into multiple columns based on a delimiter.
|
|
83
|
+
Parameters:
|
|
84
|
+
df: DataFrame containing the column to split.
|
|
85
|
+
column_name: Name of the column to be split.
|
|
86
|
+
delimiter: Delimiter to use for splitting (optional).
|
|
87
|
+
|
|
88
|
+
9. impute(data, by=None, value=None, columns=None) -> pd.DataFrame
|
|
89
|
+
Handles missing values using specified strategy or custom value.
|
|
90
|
+
Parameters:
|
|
91
|
+
data: DataFrame or Series with missing values.
|
|
92
|
+
by: Strategy for imputing missing values ('mean', 'median', 'mode', 'interpolate', 'forward_fill', 'backward_fill').
|
|
93
|
+
value: Custom value to fill NaNs with.
|
|
94
|
+
columns: List of column names to apply the fill operation.
|
|
95
|
+
|
|
96
|
+
10. spell_check_dataframe(data: pd.DataFrame, dictionary='en_US', columns=None) -> dict
|
|
97
|
+
Performs spell check on specified columns of a DataFrame.
|
|
98
|
+
Parameters:
|
|
99
|
+
data: DataFrame containing columns to spell check.
|
|
100
|
+
dictionary: Dictionary to use for spell checking ('en_US', 'en_GB', 'en_AU', 'en_IE').
|
|
101
|
+
columns: List of column names to perform spell check on.
|
|
102
|
+
|
|
103
|
+
11. detect_invalid_dates(series: pd.Series) -> pd.Series
|
|
104
|
+
Detects invalid date values in a Series.
|
|
105
|
+
Parameters:
|
|
106
|
+
series: Series to check for invalid dates.
|
|
107
|
+
|
|
108
|
+
12. detect_data_entry_errors(data, spellcheck_dict='en_US', date_columns=None, numeric_columns=None, text_columns=None) -> pd.DataFrame
|
|
109
|
+
Detects and flags data entry errors including invalid dates and misspelled words.
|
|
110
|
+
Parameters:
|
|
111
|
+
data: DataFrame to analyze.
|
|
112
|
+
spellcheck_dict: Dictionary to use for spell checking.
|
|
113
|
+
date_columns: List of columns to check for invalid dates.
|
|
114
|
+
numeric_columns: List of columns to check for numeric format errors.
|
|
115
|
+
text_columns: List of text columns to perform spell checking on.
|
|
116
|
+
|
|
117
|
+
13. data_type_conversions(data, column=None) -> pd.DataFrame or pd.Series
|
|
118
|
+
Recommends and applies data type conversions based on the analysis of each column's data.
|
|
119
|
+
Parameters:
|
|
120
|
+
data: DataFrame or Series to analyze.
|
|
121
|
+
column: Specific column to analyze (optional).
|
|
122
|
+
|
|
123
|
+
14. detect_outliers(data, method='iqr', threshold=1.5, columns=None, handle_missing=True) -> pd.DataFrame
|
|
124
|
+
Detects outliers in a dataset using specified method and threshold.
|
|
125
|
+
Parameters:
|
|
126
|
+
data: DataFrame or Series to analyze.
|
|
127
|
+
method: Outlier detection method ('z-score', 'iqr').
|
|
128
|
+
threshold: Threshold for outlier detection.
|
|
129
|
+
columns: List of columns to apply the outlier detection on (optional).
|
|
130
|
+
handle_missing: Whether to handle missing values by dropping them or not.
|
|
131
|
+
|
|
132
|
+
Logging Functions
|
|
133
|
+
1. display_logs()
|
|
134
|
+
Prints stored log entries.
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
## Example Usage
|
|
138
|
+
Here are some examples to illustrate the usage of the functions provided in pyspan:
|
|
139
|
+
|
|
140
|
+
import pandas as pd
|
|
141
|
+
from pyspan import handle_nulls, remove_duplicates, remove_columns, auto_rename_columns, rename_dataframe_columns
|
|
142
|
+
from pyspan import change_dt, detect_delimiter, split_column, impute, spell_check_dataframe
|
|
143
|
+
from pyspan import detect_invalid_dates, detect_data_entry_errors, data_type_conversions, detect_outliers
|
|
144
|
+
from pyspan import log_function_call, display_logs
|
|
145
|
+
|
|
146
|
+
# Load a dataset
|
|
147
|
+
df = pd.read_csv('/content/GlobalSharkAttacks.csv')
|
|
148
|
+
|
|
149
|
+
# Example usage of handle_nulls
|
|
150
|
+
df_filled = handle_nulls(df, columns=['Column1', 'Column2'], method='mean')
|
|
151
|
+
|
|
152
|
+
# Example usage of remove_duplicates
|
|
153
|
+
df_unique = remove_duplicates(df)
|
|
154
|
+
|
|
155
|
+
# Example usage of remove_columns
|
|
156
|
+
df_reduced = remove_columns(df, columns=['ColumnToRemove'])
|
|
157
|
+
|
|
158
|
+
# Example usage of auto_rename_columns
|
|
159
|
+
auto_rename_columns(df)
|
|
160
|
+
|
|
161
|
+
# Example usage of rename_dataframe_columns
|
|
162
|
+
rename_dict = {'OldName': 'NewName'}
|
|
163
|
+
df_renamed_dict = rename_dataframe_columns(df, rename_dict)
|
|
164
|
+
|
|
165
|
+
# Example usage of change_dt
|
|
166
|
+
df_formatted = change_dt(df, columns=['Date'], date_format='%d-%m-%Y', time_format='%I:%M %p')
|
|
167
|
+
|
|
168
|
+
# Example usage of detect_delimiter
|
|
169
|
+
delimiter = detect_delimiter(df['ColumnWithDelimiters'])
|
|
170
|
+
|
|
171
|
+
# Example usage of split_column
|
|
172
|
+
df_split = split_column(df, column_name='ColumnWithDelimiters')
|
|
173
|
+
|
|
174
|
+
# Example usage of impute
|
|
175
|
+
df_imputed = impute(df, by='mean', columns=['Column1'])
|
|
176
|
+
|
|
177
|
+
# Example usage of spell_check_dataframe
|
|
178
|
+
misspelled = spell_check_dataframe(df, dictionary='en_US', columns=['TextColumn'])
|
|
179
|
+
|
|
180
|
+
# Example usage of detect_invalid_dates
|
|
181
|
+
invalid_dates = detect_invalid_dates(df['DateColumn'])
|
|
182
|
+
|
|
183
|
+
# Example usage of detect_data_entry_errors
|
|
184
|
+
errors = detect_data_entry_errors(df, spellcheck_dict='en_US', date_columns=['DateColumn'], numeric_columns=['NumericColumn'])
|
|
185
|
+
|
|
186
|
+
# Example usage of data_type_conversions
|
|
187
|
+
df_converted = data_type_conversions(df)
|
|
188
|
+
|
|
189
|
+
# Example usage of detect_outliers
|
|
190
|
+
outliers = detect_outliers(df, method='iqr', threshold=1.5)
|
|
191
|
+
|
|
192
|
+
# Example usage of display_logs
|
|
193
|
+
display_logs()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# License
|
|
197
|
+
This package is licensed under the MIT License. See the LICENSE file for more details.
|
|
198
|
+
|
|
199
|
+
# Contact
|
|
200
|
+
For issues or questions, please contact [amynahreimoo@gmail.com].
|
|
201
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pyspan
|