AutoStatLib 0.2.1__tar.gz → 0.2.5__tar.gz

{autostatlib-0.2.1/src/AutoStatLib.egg-info → autostatlib-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: AutoStatLib
-Version: 0.2.1
+Version: 0.2.5
 Summary: AutoStatLib - a simple statistical analysis tool
 Author: Stemonitis, SciWare LLC
 Author-email: konung-yaropolk <yaropolk1995@gmail.com>
@@ -531,6 +531,7 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: scipy
 Requires-Dist: statsmodels
+Requires-Dist: scikit-posthocs
 Requires-Dist: pandas
 
 # AutoStatLib - python library for automated statistical analysis 
@@ -595,10 +596,11 @@ analysis.RunTtestPaired()
 analysis.RunWilcoxon()
 
 # 3 and more independed groups comparison:
-analysis.RunAnova()
+analysis.RunOnewayAnova()
 analysis.RunKruskalWallis()
 
 # 3 and more depended groups comparison:
+analysis.RunOnewayAnovaRM()
 analysis.RunFriedman()
 
 # single group tests"
@@ -648,20 +650,42 @@ If errors occured, *GetResult()* returns an empty dictionary
 
 
 ---
-## Pre-Alpha dev status.
-
-### TODO:
-
---Kruskal-Wallis test - add Dunn's multiple comparisons
---Anova: add 2-way anova and 3-way(?)
-
-check:
---Wilcoxon signed-rank test and Mann-whitney - check mechanism of one-tailed calc, looks like it works wrong
-
-
-checked tests:
---Wilcoxon 2 tail - ok
---Mann-whitney 2 tail - ok
+## Pre-Alpha dev status.  
+  
+### TODO:  
+  
+-- Anova: posthocs  
+-- Anova: add 2-way anova and 3-way anova  
+-- onevay Anova: add repeated measures (for normal dependent values) with and without Gaisser-Greenhouse correction  
+-- onevay Anova: add Brown-Forsithe and Welch (for normal independent values with unequal SDs between groups)  
+-- paired T-test: add ratio-paired t-test (ratios of paired values are consistent)  
+-- add Welch test (for norm data unequal variances)
+-- add Kolmogorov-smirnov test (unpaired nonparametric 2 sample, compare cumulative distributions)  
+-- add independent t-test with Welch correction (do not assume equal SDs in groups)  
+-- add correlation test, correlation diagram  
+-- add linear regression, regression diagram  
+-- add QQ plot
+-- n-sample tests: add onetail option
+  
+✅ done -- detailed normality test results  
+✅ done -- added posthoc: Kruskal-Wallis Dunn's multiple comparisons 
+
+  
+tests check:  
+1-sample:  
+--Wilcoxon 2,1 tails - ok  
+--t-tests 2,1 tails -ok  
+  
+2-sample:  
+--Wilcoxon 2,1 tails - ok  
+--Mann-whitney 2,1 tails - ok  
+--t-tests 2,1 tails -ok  
+  
+n-sample:  
+--Kruskal-Wallis 2 tail - ok  
+--Dunn's multiple comparisons - ??
+--Friedman 2 tail - ok  
+--one-way ANOWA 2 tail - ok  
 
 
 

{autostatlib-0.2.1 → autostatlib-0.2.5}/README.md

@@ -60,10 +60,11 @@ analysis.RunTtestPaired()
 analysis.RunWilcoxon()
 
 # 3 and more independed groups comparison:
-analysis.RunAnova()
+analysis.RunOnewayAnova()
 analysis.RunKruskalWallis()
 
 # 3 and more depended groups comparison:
+analysis.RunOnewayAnovaRM()
 analysis.RunFriedman()
 
 # single group tests"
@@ -113,20 +114,42 @@ If errors occured, *GetResult()* returns an empty dictionary
 
 
 ---
-## Pre-Alpha dev status.
-
-### TODO:
-
---Kruskal-Wallis test - add Dunn's multiple comparisons
---Anova: add 2-way anova and 3-way(?)
-
-check:
---Wilcoxon signed-rank test and Mann-whitney - check mechanism of one-tailed calc, looks like it works wrong
-
-
-checked tests:
---Wilcoxon 2 tail - ok
---Mann-whitney 2 tail - ok
+## Pre-Alpha dev status.  
+  
+### TODO:  
+  
+-- Anova: posthocs  
+-- Anova: add 2-way anova and 3-way anova  
+-- onevay Anova: add repeated measures (for normal dependent values) with and without Gaisser-Greenhouse correction  
+-- onevay Anova: add Brown-Forsithe and Welch (for normal independent values with unequal SDs between groups)  
+-- paired T-test: add ratio-paired t-test (ratios of paired values are consistent)  
+-- add Welch test (for norm data unequal variances)
+-- add Kolmogorov-smirnov test (unpaired nonparametric 2 sample, compare cumulative distributions)  
+-- add independent t-test with Welch correction (do not assume equal SDs in groups)  
+-- add correlation test, correlation diagram  
+-- add linear regression, regression diagram  
+-- add QQ plot
+-- n-sample tests: add onetail option
+  
+✅ done -- detailed normality test results  
+✅ done -- added posthoc: Kruskal-Wallis Dunn's multiple comparisons 
+
+  
+tests check:  
+1-sample:  
+--Wilcoxon 2,1 tails - ok  
+--t-tests 2,1 tails -ok  
+  
+2-sample:  
+--Wilcoxon 2,1 tails - ok  
+--Mann-whitney 2,1 tails - ok  
+--t-tests 2,1 tails -ok  
+  
+n-sample:  
+--Kruskal-Wallis 2 tail - ok  
+--Dunn's multiple comparisons - ??
+--Friedman 2 tail - ok  
+--one-way ANOWA 2 tail - ok  
 
 
 

{autostatlib-0.2.1 → autostatlib-0.2.5}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=62.6", "numpy", "scipy", "statsmodels"]
+requires = ["setuptools>=62.6", "numpy", "scipy", "statsmodels", "scikit-posthocs"]
 build-backend = "setuptools.build_meta"
 
 [project]

{autostatlib-0.2.1 → autostatlib-0.2.5}/requirements.txt

@@ -1,4 +1,5 @@
 numpy
 scipy
 statsmodels
-pandas
+scikit-posthocs
+pandas

autostatlib-0.2.5/src/AutoStatLib/AutoStatLib.py

@@ -0,0 +1,255 @@
+from AutoStatLib.statistical_tests import StatisticalTests
+from AutoStatLib.normality_tests import NormalityTests
+from AutoStatLib.helpers import Helpers
+from AutoStatLib.text_formatting import TextFormatting
+
+
+class StatisticalAnalysis(StatisticalTests, NormalityTests, TextFormatting, Helpers):
+    '''
+        The main class
+        *documentation placeholder*
+
+    '''
+
+    def __init__(self,
+                 groups_list,
+                 paired=False,
+                 tails=2,
+                 popmean=None,
+                 posthoc=False,
+                 verbose=True):
+        self.results = None
+        self.error = False
+        self.groups_list = groups_list
+        self.paired = paired
+        self.tails = tails
+        self.popmean = popmean
+        self.posthoc = posthoc
+        self.verbose = verbose
+        self.n_groups = len(self.groups_list)
+        self.warning_flag_non_numeric_data = False
+        self.summary = ''
+
+        # test IDs classification:
+        self.test_ids_all = [  # in aplhabetical order
+            'anova_1w_ordinary',
+            'anova_1w_rm',
+            'friedman',
+            'kruskal_wallis',
+            'mann_whitney',
+            't_test_independent',
+            't_test_paired',
+            't_test_single_sample',
+            'wilcoxon',
+            'wilcoxon_single_sample',
+        ]
+        self.test_ids_parametric = [
+            'anova_1w_ordinary',
+            'anova_1w_rm'
+            't_test_independent',
+            't_test_paired',
+            't_test_single_sample',
+        ]
+        self.test_ids_dependent = [
+            'anova_1w_rm',
+            'friedman',
+            't_test_paired',
+            'wilcoxon',
+        ]
+        self.test_ids_3sample = [
+            'anova_1w_ordinary',
+            'anova_1w_rm',
+            'friedman',
+            'kruskal_wallis',
+        ]
+        self.test_ids_2sample = [
+            'mann_whitney',
+            't_test_independent',
+            't_test_paired',
+            'wilcoxon',
+        ]
+        self.test_ids_1sample = [
+            't_test_single_sample',
+            'wilcoxon_single_sample',
+        ]
+        self.warning_ids_all = {
+            # 'not-numeric':                     '\nWarning: Non-numeric data was found in input and ignored.\n         Make sure the input data is correct to get the correct results\n',
+            'param_test_with_non-normal_data': '\nWarning: Parametric test was manualy chosen for Not-Normaly distributed data.\n         The results might be skewed. \n         Please, run non-parametric test or preform automatic test selection.\n',
+            'non-param_test_with_normal_data': '\nWarning: Non-Parametric test was manualy chosen for Normaly distributed data.\n         The results might be skewed. \n         Please, run parametric test or preform automatic test selection.\n',
+            'no_pop_mean_set':                 '\nWarning: No Population Mean was set up for single-sample test, used default 0 value.\n         The results might be skewed. \n         Please, set the Population Mean and run the test again.\n',
+        }
+
+    def run_test(self, test='auto'):
+
+        # reset values from previous tests
+        self.results = None
+        self.error = False
+        self.warnings = []
+        self.normals = []
+        self.test_name = None
+        self.test_id = None
+        self.test_stat = None
+        self.p_value = None
+        self.posthoc_matrix_df = None
+        self.posthoc_matrix = []
+
+        self.log('\n' + '-'*67)
+        self.log('Statistical analysis __init__iated for data in {} groups\n'.format(
+            len(self.groups_list)))
+
+        # adjusting input data type
+        self.data = self.floatify_recursive(self.groups_list)
+        if self.warning_flag_non_numeric_data:
+            self.log(
+                'Text or other non-numeric data in the input was ignored:')
+
+        # delete the empty cols from input
+        self.data = [col for col in self.data if any(
+            x is not None for x in col)]
+
+        # User input assertion block
+        try:
+            assert self.data, 'There is no input data'
+            assert self.tails in [1, 2], 'Tails parameter can be 1 or 2 only'
+            assert test in self.test_ids_all or test == 'auto', 'Wrong test id choosen, ensure you called correct function'
+            assert all(len(
+                group) >= 4 for group in self.data), 'Each group must contain at least four values'
+            assert not (self.paired == True
+                        and not all(len(lst) == len(self.data[0]) for lst in self.data)), 'Paired groups must have the same length'
+            assert not (test in self.test_ids_dependent
+                        and not all(len(lst) == len(self.data[0]) for lst in self.data)), 'Groups must have the same length for dependent groups test'
+            assert not (test in self.test_ids_2sample
+                        and self.n_groups != 2), f'Only two groups of data must be given for 2-groups tests, got {self.n_groups}'
+            assert not (test in self.test_ids_1sample
+                        and self.n_groups > 1), f'Only one group of data must be given for single-group tests, got {self.n_groups}'
+            assert not (test in self.test_ids_3sample
+                        and self.n_groups < 3), f'At least three groups of data must be given for multi-groups tests, got {self.n_groups}'
+        except AssertionError as error:
+            self.log('\nTest  :', test)
+            self.log('Error :', error)
+            self.log('-'*67 + '\n')
+            self.error = True
+            print(self.summary)
+            return
+
+        # Print the data
+        self.print_groups()
+
+        # Normality tests
+        self.log(
+            '\n\nThe group is assumed to be normally distributed if at least one')
+        self.log(
+            'normality test result is positive. Normality checked by tests:')
+        self.log('Shapiro-Wilk, Lilliefors, Anderson-Darling, D\'Agostino-Pearson')
+        self.log(
+            '[+] -positive, [-] -negative, [ ] -too small group for the test\n')
+        self.log('        Test   :   SW  LF  AD  AP  ')
+        for i, data in enumerate(self.data):
+            poll = self.check_normality(data)
+            isnormal = any(poll)
+            poll_print = tuple(
+                '+' if x is True else '-' if x is False else ' ' if x is None else 'e' for x in poll)
+            self.normals.append(isnormal)
+            self.log(
+                f'        Group {i+1}:    {poll_print[0]}   {poll_print[1]}   {poll_print[2]}   {poll_print[3]}   so disrtibution seems {"normal" if isnormal else "not normal"}')
+        self.parametric = all(self.normals)
+
+        # print test choosen
+        self.log('\n\nInput:\n')
+        self.log('Data Normaly Distributed:     ', self.parametric)
+        self.log('Paired Groups:                ', self.paired)
+        self.log('Groups:                       ', self.n_groups)
+        self.log('Test chosen by user:          ', test)
+
+        # Wrong test Warnings
+        if not test == 'auto' and not self.parametric and test in self.test_ids_parametric:
+            self.AddWarning('param_test_with_non-normal_data')
+        if not test == 'auto' and self.parametric and not test in self.test_ids_parametric:
+            self.AddWarning('non-param_test_with_normal_data')
+
+        # run the test
+
+        if test in self.test_ids_all:
+            self.run_test_by_id(test)
+        else:
+            self.run_test_auto()
+
+
+        # print the results
+        self.results = self.create_results_dict()
+        self.print_results()
+        self.log(
+            '\n\nResults above are accessible as a dictionary via GetResult() method')
+        self.log('-'*67 + '\n')
+
+        # print the results to console:
+        if self.verbose == True:
+            print(self.summary)
+
+
+
+    # public methods:
+    def RunAuto(self):
+        self.run_test(test='auto')
+
+    def RunManual(self, test):
+        self.run_test(test)
+
+    def RunOnewayAnova(self):
+        self.run_test(test='anova_1w_ordinary')
+
+    def RunOnewayAnovaRM(self):
+        self.run_test(test='anova_1w_rm')
+
+    def RunFriedman(self):
+        self.run_test(test='friedman')
+
+    def RunKruskalWallis(self):
+        self.run_test(test='kruskal_wallis')
+
+    def RunMannWhitney(self):
+        self.run_test(test='mann_whitney')
+
+    def RunTtest(self):
+        self.run_test(test='t_test_independent')
+
+    def RunTtestPaired(self):
+        self.run_test(test='t_test_paired')
+
+    def RunTtestSingleSample(self):
+        self.run_test(test='t_test_single_sample')
+
+    def RunWilcoxonSingleSample(self):
+        self.run_test(test='wilcoxon_single_sample')
+
+    def RunWilcoxon(self):
+        self.run_test(test='wilcoxon')
+
+    def GetResult(self):
+        if not self.results and not self.error:
+            print('No test chosen, no results to output')
+            # self.run_test(test='auto')
+            return self.results
+        if not self.results and self.error:
+            print('Error occured, no results to output')
+            return {}
+        else:
+            return self.results
+
+    def GetSummary(self):
+        if not self.results and not self.error:
+            print('No test chosen, no summary to output')
+            # self.run_test(test='auto')
+            return self.summary
+        else:
+            return self.summary
+
+    def GetTestIDs(self):
+        return self.test_ids_all
+
+    def PrintSummary(self):
+        print(self.summary)
+
+
+if __name__ == '__main__':
+    print('This package works as an imported module only.\nUse "import autostatlib" statement')

{autostatlib-0.2.1 → autostatlib-0.2.5}/src/AutoStatLib/_version.py

@@ -1,2 +1,2 @@
 # AutoStatLib package version:
-__version__ = "0.2.1"
+__version__ = "0.2.5"

autostatlib-0.2.5/src/AutoStatLib/helpers.py

@@ -0,0 +1,77 @@
+import numpy as np
+import pandas as pd
+
+class Helpers():
+
+    def matrix_to_dataframe(self, matrix):
+        data = []
+        cols = []
+        rows = []
+
+        order_number = 1
+        for i, row in enumerate(matrix):
+            for j, value in enumerate(row):
+                data.append(value)
+                cols.append(i)
+                rows.append(j)
+                order_number += 1
+
+        df = pd.DataFrame(
+            {'Row': rows, 'Col': cols, 'Value': data})
+        return df
+
+    def floatify_recursive(self, data):
+        if isinstance(data, list):
+            # Recursively process sublists and filter out None values
+            processed_list = [self.floatify_recursive(item) for item in data]
+            return [item for item in processed_list if item is not None]
+        else:
+            try:
+                # Try to convert the item to float
+                return np.float64(data)
+            except (ValueError, TypeError):
+                # If conversion fails, replace with None
+                self.warning_flag_non_numeric_data = True
+                return None
+
+    def create_results_dict(self) -> dict:
+
+        self.stars_int = self.make_stars(self.p_value.item())
+        self.stars_str = self.make_stars_printed(self.stars_int)
+
+        return {
+            'p-value': self.make_p_value_printed(self.p_value.item()),
+            'Significance(p<0.05)':  True if self.p_value.item() < 0.05 else False,
+            'Stars_Printed': self.stars_str,
+            'Test_Name': self.test_name,
+            'Groups_Compared': self.n_groups,
+            'Population_Mean': self.popmean if self.n_groups == 1 else 'N/A',
+            'Data_Normaly_Distributed': self.parametric,
+            'Parametric_Test_Applied': True if self.test_id in self.test_ids_parametric else False,
+            'Paired_Test_Applied': self.paired,
+            'Tails': self.tails,
+            'p-value_exact': self.p_value.item(),
+            'Stars':  self.stars_int,
+            # 'Stat_Value': self.test_stat.item(),
+            'Warnings': self.warnings,
+            'Groups_N': [len(self.data[i]) for i in range(len(self.data))],
+            'Groups_Median': [np.median(self.data[i]).item() for i in range(len(self.data))],
+            'Groups_Mean': [np.mean(self.data[i]).item() for i in range(len(self.data))],
+            'Groups_SD': [np.std(self.data[i]).item() for i in range(len(self.data))],
+            'Groups_SE': [np.std(self.data[i]).item() / np.sqrt(len(self.data)).item() for i in range(len(self.data))],
+            # actually returns list of lists of numpy dtypes of float64, next make it return regular floats:
+            'Samples': self.data,
+            'Posthoc_Matrix': self.posthoc_matrix if self.posthoc_matrix else 'N/A',
+            'Posthoc_Matrix_printed': [[self.make_p_value_printed(element) for element in row] for row in self.posthoc_matrix] if self.posthoc_matrix else 'N/A',
+            'Posthoc_Matrix_stars': [[self.make_stars_printed(self.make_stars(element)) for element in row] for row in self.posthoc_matrix] if self.posthoc_matrix else 'N/A',
+        }
+
+    def log(self, *args, **kwargs):
+        message = ' '.join(map(str, args))
+        # print(message, **kwargs)
+        self.summary += '\n' + message
+
+    def AddWarning(self, warning_id):
+        message = self.warning_ids_all[warning_id]
+        self.log(message)
+        self.warnings.append(message)

autostatlib-0.2.5/src/AutoStatLib/normality_tests.py

@@ -0,0 +1,85 @@
+from statsmodels.stats.diagnostic import lilliefors
+from scipy.stats import shapiro, normaltest, anderson
+
+
+class NormalityTests():
+    '''
+        Normality tests mixin
+
+        see the article about minimal sample size for tests:
+        Power comparisons of Shapiro-Wilk, Kolmogorov-Smirnov,
+        Lilliefors and Anderson-Darling tests, Nornadiah Mohd Razali1, Yap Bee Wah1
+    '''
+
+    def check_normality(self, data):
+        sw = None
+        lf = None
+        ad = None
+        ap = None
+        n = len(data)
+
+        # Shapiro-Wilk test
+        sw_stat, sw_p_value = shapiro(data)
+        if sw_p_value > 0.05:
+            sw = True
+        else:
+            sw = False
+
+        # Lilliefors test
+        lf_stat, lf_p_value = lilliefors(
+            data, dist='norm')
+        if lf_p_value > 0.05:
+            lf = True
+        else:
+            lf = False
+
+        # Anderson-Darling test
+        if n >= 20:
+            ad_stat, ad_p_value = self.anderson_get_p(
+                data, dist='norm')
+            if ad_p_value > 0.05:
+                ad = True
+            else:
+                ad = False
+
+        # D'Agostino-Pearson test
+        # test result is skewed if n<20
+        if n >= 20:
+            ap_stat, ap_p_value = normaltest(data)
+            if ap_p_value > 0.05:
+                ap = True
+            else:
+                ap = False
+
+        # print(ap_p_value, ad_p_value, sw_p_value, lf_p_value)
+
+        return (sw, lf, ad, ap)
+
+    def anderson_get_p(self, data, dist='norm'):
+        '''
+            calculating p-value for Anderson-Darling test using the method described here:
+            Computation of Probability Associated with Anderson-Darling Statistic
+            Lorentz Jantschi and Sorana D. Bolboaca, 2018 - Mathematics
+
+        '''
+        e = 2.718281828459045
+        n = len(data)
+
+        ad, critical_values, significance_levels = anderson(
+            data, dist=dist)
+
+        # adjust ad_stat for small sample sizes:
+        s = ad*(1 + 0.75/n + 2.25/(n**2))
+
+        if s >= 0.6:
+            p = e**(1.2937 - 5.709*s + 0.0186*s**2)
+        elif s > 0.34:
+            p = e**(0.9177 - 4.279*s - 1.38*s**2)
+        elif s > 0.2:
+            p = 1 - e**(-8.318 + 42.796*s - 59.938*s**2)
+        elif s <= 0.2:
+            p = 1 - e**(-13.436 + 101.14*s - 223.73*s**2)
+        else:
+            p = None
+
+        return ad, p