refs-mcc 1.0.0__py3-none-any.whl

aBioInf100.py

@@ -0,0 +1,58 @@
+from features import *
+from reduceData import *
+
+from joblib import Parallel, delayed
+import multiprocessing
+import time
+import argparse
+
+def mainRun(indexRun, numberOfFolds) :
+	run=indexRun
+	start_time = time.time()
+	globalAnt=0.0
+	globalIndex=0
+	globalAccuracy=0.0
+
+	X, y, biomarkerNames = loadDatasetOriginal(run)
+	
+	if (int(len(X[0]))>1000):
+		numberOfTopFeatures = 1000
+	else :
+		numberOfTopFeatures = int(len(X[0])*0.80)
+
+	variableSize=numberOfTopFeatures;
+	while True:
+		globalAnt=globalAccuracy
+		globalAccuracy=featureSelection(globalIndex,variableSize, run,numberOfFolds)
+		print(globalAccuracy)
+		print(globalIndex)
+		print(variableSize)
+		size,sizereduced=reduceDataset(globalIndex, run)
+		
+		if(variableSize==0):
+			break
+		variableSize=int(variableSize*0.80)
+		
+		globalIndex=globalIndex + 1
+
+	elapsed_time = time.time() - start_time
+	print("time")
+	print(elapsed_time)
+	return
+
+def main(threads, totalRuns, numberOfFolds) :
+	Parallel(n_jobs=threads, verbose=5, backend="multiprocessing")(delayed(mainRun)(i, numberOfFolds) for i in range(0,totalRuns))
+	return
+
+if __name__ == "__main__" :
+	parser = argparse.ArgumentParser(description="Run REFS-MCC - part 1")
+	parser.add_argument('--threads', type=int, default=10, help='Number of threads (default: 10)')
+	parser.add_argument('--totalRuns', type=int, default=10, help='Total number of runs (default: 10)')
+	parser.add_argument('--folds', type=int, default=10, help='Number of folds (default: 10)')
+	args = parser.parse_args()
+
+	threads=args.threads
+	totalRuns=args.totalRuns
+	numberOfFolds=args.folds
+
+	sys.exit( main(threads, totalRuns, numberOfFolds) )

classifiersMulti.py

@@ -0,0 +1,227 @@
+import copy
+import datetime
+import logging
+import numpy as np
+import os
+import sys
+import pandas as pd 
+import argparse
+
+from sklearn.ensemble import AdaBoostClassifier
+from sklearn.ensemble import BaggingClassifier
+from sklearn.ensemble import ExtraTreesClassifier
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.ensemble import RandomForestClassifier
+
+from sklearn.linear_model import ElasticNet
+from sklearn.linear_model import ElasticNetCV
+from sklearn.linear_model import Lasso
+from sklearn.linear_model import LassoCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegressionCV
+from sklearn.linear_model import PassiveAggressiveClassifier
+from sklearn.linear_model import RidgeClassifier
+from sklearn.linear_model import RidgeClassifierCV
+from sklearn.linear_model import SGDClassifier
+
+from sklearn.multiclass import OneVsOneClassifier 
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.multiclass import OutputCodeClassifier
+
+from sklearn.naive_bayes import BernoulliNB
+from sklearn.naive_bayes import GaussianNB
+from sklearn.naive_bayes import MultinomialNB 
+
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.neighbors import NearestCentroid
+from sklearn.neighbors import RadiusNeighborsClassifier
+
+from sklearn.svm import SVC
+
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import ExtraTreeClassifier
+from sklearn.neural_network import MLPClassifier
+
+# used for normalization
+from sklearn.preprocessing import  Normalizer
+from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler
+# used for cross-validation
+from sklearn.model_selection import StratifiedKFold
+from sklearn.metrics import roc_curve, auc
+from numpy import interp
+import matplotlib.pyplot as plt
+from pandas import read_csv
+
+from statsmodels.multivariate.manova import MANOVA
+import statsmodels.api as sm
+from scipy import stats
+from sklearn.metrics import f1_score
+def loadDataset() :
+	
+	# data used for the predictions
+	dfData = read_csv("./best/data_0.csv", header=None, sep=',')
+	dfLabels = read_csv("./best/labels.csv", header=None)
+		
+	return dfData.values, dfLabels.values.ravel() # to have it in the format that the classifiers like
+
+
+def runFeatureReduce(numberOfFolds) :
+	
+	# list of classifiers, selected on the basis of our previous paper "
+	classifierList = [
+		
+			[GradientBoostingClassifier(n_estimators=300), "GradientBoostingClassifier(n_estimators=300)"],
+			[RandomForestClassifier(n_estimators=300), "RandomForestClassifier(n_estimators=300)"],
+			[LogisticRegression(solver='lbfgs',), "LogisticRegression"],
+			[PassiveAggressiveClassifier(),"PassiveAggressiveClassifier"],
+			[SGDClassifier(), "SGDClassifier"],
+			[SVC(kernel='linear'), "SVC(linear)"],
+			[RidgeClassifier(), "RidgeClassifier"],
+			[BaggingClassifier(n_estimators=300), "BaggingClassifier(n_estimators=300)"],
+			# ensemble
+			#[AdaBoostClassifier(), "AdaBoostClassifier"],
+			#[AdaBoostClassifier(n_estimators=300), "AdaBoostClassifier(n_estimators=300)"],
+			#[AdaBoostClassifier(n_estimators=1500), "AdaBoostClassifier(n_estimators=1500)"],
+			#[BaggingClassifier(), "BaggingClassifier"],
+			
+			#[ExtraTreesClassifier(), "ExtraTreesClassifier"],
+			#[ExtraTreesClassifier(n_estimators=300), "ExtraTreesClassifier(n_estimators=300)"],
+			 # features_importances_
+			#[GradientBoostingClassifier(n_estimators=300), "GradientBoostingClassifier(n_estimators=300)"],
+			#[GradientBoostingClassifier(n_estimators=1000), "GradientBoostingClassifier(n_estimators=1000)"],
+			
+			#[RandomForestClassifier(n_estimators=300), "RandomForestClassifier(n_estimators=300)"],
+			#[RandomForestClassifier(n_estimators=1000), "RandomForestClassifier(n_estimators=1000)"], # features_importances_
+
+			# linear
+			#[ElasticNet(), "ElasticNet"],
+			#[ElasticNetCV(), "ElasticNetCV"],
+			#[Lasso(), "Lasso"],
+			#[LassoCV(), "LassoCV"],
+			 # coef_
+			#[LogisticRegressionCV(), "LogisticRegressionCV"],
+			  # coef_
+			 # coef_
+			#[RidgeClassifierCV(), "RidgeClassifierCV"],
+			 # coef_
+			 # coef_, but only if the kernel is linear...the default is 'rbf', which is NOT linear
+			
+			# naive Bayes
+			#[BernoulliNB(), "BernoulliNB"],
+			#[GaussianNB(), "GaussianNB"],
+			#[MultinomialNB(), "MultinomialNB"],
+			
+			# neighbors
+			#[KNeighborsClassifier(), "KNeighborsClassifier"], # no way to return feature importance
+			# TODO this one creates issues
+			#[NearestCentroid(), "NearestCentroid"], # it does not have some necessary methods, apparently
+			#[RadiusNeighborsClassifier(), "RadiusNeighborsClassifier"],
+			
+			# tree
+			#[DecisionTreeClassifier(), "DecisionTreeClassifier"],
+			#[ExtraTreeClassifier(), "ExtraTreeClassifier"],
+			[AdaBoostClassifier(n_estimators=300), "AdaBoostClassifier(n_estimators=300)"],
+            [ExtraTreesClassifier(n_estimators=300), "ExtraTreesClassifier(n_estimators=300)"],
+            [KNeighborsClassifier(), "KNeighborsClassifier"],
+            #[LassoCV(), "LassoCV"],
+			[MLPClassifier(), "MLPClassifier"],
+			[LassoCV(), "LassoCV"]
+			]
+	
+	# this is just a hack to check a few things
+	#classifierList = [
+	#		[RandomForestClassifier(), "RandomForestClassifier"]
+	#		]
+
+	print("Loading dataset...")
+	X, y = loadDataset()
+	
+	print(len(X))
+	print(len(X[0]))
+	print(len(y))
+
+	
+	labels=np.max(y)+1
+	# prepare folds
+	skf = StratifiedKFold(n_splits=numberOfFolds, shuffle=True)
+	indexes = [ (training, test) for training, test in skf.split(X, y) ]
+	
+	# this will be used for the top features
+	topFeatures = dict()
+	
+	# iterate over all classifiers
+	classifierIndex = 0
+	
+	
+	
+	
+	for originalClassifier, classifierName in classifierList :
+		
+		print("\nClassifier " + classifierName)
+		classifierPerformance = []
+		F1score= []
+
+		cMatrix=np.zeros((labels, labels))
+		# iterate over all folds
+		
+		indexFold = 0
+
+		yTest=[]
+		yNew=[]
+
+		for train_index, test_index in indexes :
+			
+			X_train, X_test = X[train_index], X[test_index]
+			y_train, y_test = y[train_index], y[test_index]
+			
+			# let's normalize, anyway
+			# MinMaxScaler StandardScaler Normalizer
+			scaler = StandardScaler()
+			X_train = scaler.fit_transform(X_train)
+			X_test = scaler.transform(X_test)
+
+		
+			
+			classifier = copy.deepcopy(originalClassifier)
+			classifier.fit(X_train, y_train)
+			scoreTraining = classifier.score(X_train, y_train)
+			scoreTest = classifier.score(X_test, y_test)
+			
+			y_new = classifier.predict(X_test)
+			
+			
+			
+			yNew.append(y_new)
+			yTest.append(y_test)
+			
+			
+			for i in range(0,len(y_new)):
+				cMatrix[y_test[i]][round(y_new[i])]+=1
+				y_new[i]=round(y_new[i])
+
+			print("\ttraining: %.4f, test: %.4f" % (scoreTraining, scoreTest))
+			classifierPerformance.append( scoreTest )
+			
+	
+			F1scoreTest = f1_score(y_test, y_new, average='weighted')
+			F1score.append(F1scoreTest)
+
+		pd.DataFrame(cMatrix).to_csv("./best/cMatrix"+str(classifierIndex)+".csv", header=None, index =None)
+		classifierIndex+=1
+		line ="%s \t %.4f \t %.4f \t %.4f \t %.4f\n" % (classifierName, np.mean(classifierPerformance), np.std(classifierPerformance), np.mean(F1score), np.std(F1score))
+		
+		
+		print(line)
+		fo = open("./best/results.txt", 'a')
+		fo.write( line )
+		fo.close()
+	
+	return
+
+if __name__ == "__main__" :
+	parser = argparse.ArgumentParser(description="Run multiple classifiers based on the best run")
+	parser.add_argument('--folds', type=int, default=10, help='Number of folds (default: 10)')
+	args = parser.parse_args()
+
+	sys.exit( runFeatureReduce(args.folds) )

features.py

@@ -0,0 +1,354 @@
+import copy
+import datetime
+import logging
+import numpy as np
+import os
+import sys
+
+from sklearn.ensemble import AdaBoostClassifier
+from sklearn.ensemble import BaggingClassifier
+from sklearn.ensemble import ExtraTreesClassifier
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.ensemble import RandomForestClassifier
+
+from sklearn.linear_model import ElasticNet
+from sklearn.linear_model import ElasticNetCV
+from sklearn.linear_model import Lasso
+from sklearn.linear_model import LassoCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegressionCV
+from sklearn.linear_model import PassiveAggressiveClassifier
+from sklearn.linear_model import RidgeClassifier
+from sklearn.linear_model import RidgeClassifierCV
+from sklearn.linear_model import SGDClassifier
+
+from sklearn.multiclass import OneVsOneClassifier 
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.multiclass import OutputCodeClassifier
+
+from sklearn.naive_bayes import BernoulliNB
+from sklearn.naive_bayes import GaussianNB
+from sklearn.naive_bayes import MultinomialNB 
+
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.neighbors import NearestCentroid
+from sklearn.neighbors import RadiusNeighborsClassifier
+
+from sklearn.svm import SVC
+
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import ExtraTreeClassifier
+
+# used for normalization
+from sklearn.preprocessing import StandardScaler
+
+# used for cross-validation
+from sklearn.model_selection import StratifiedKFold
+from sklearn.metrics import matthews_corrcoef
+
+from pandas import read_csv
+import pandas as pd 
+
+def loadDatasetOriginal(run) :
+	
+	# data used for the predictions
+	dfData = read_csv("../data/data_0.csv", header=None, sep=',')
+	dfLabels = read_csv("../data/labels.csv", header=None)
+	biomarkers = read_csv("../data/features_0.csv", header=None)
+
+	# create folder
+	folderName ="./run"+str(run)+"/"
+	if not os.path.exists(folderName) : os.makedirs(folderName)
+
+	pd.DataFrame(dfData.values).to_csv("./run"+str(run)+"/data_0.csv", header=None, index =None)
+	pd.DataFrame(biomarkers.values.ravel()).to_csv("./run"+str(run)+"/features_0.csv", header=None, index =None)
+	pd.DataFrame(dfLabels.values.ravel()).to_csv("./run"+str(run)+"/labels.csv", header=None, index =None)
+	return dfData.values, dfLabels.values.ravel(), biomarkers.values.ravel() # to have it in the format that the classifiers like
+
+
+def loadDataset(globalIndex, run) :
+	
+	# data used for the predictions
+	dfData = read_csv("./run"+str(run)+"/data_"+str(globalIndex)+".csv", header=None, sep=',')
+	dfLabels = read_csv("./run"+str(run)+"/labels.csv", header=None)
+	biomarkers = read_csv("./run"+str(run)+"/features_"+str(globalIndex)+".csv", header=None)
+
+	return dfData.values, dfLabels.values.ravel(), biomarkers.values.ravel() # to have it in the format that the classifiers like
+
+# this function returns a list of features, in relative order of importance
+def relativeFeatureImportance(classifier) :
+	
+	# this is the output; it will be a sorted list of tuples (importance, index)
+	# the index is going to be used to find the "true name" of the feature
+	orderedFeatures = []
+
+	# the simplest case: the classifier already has a method that returns relative importance of features
+	if hasattr(classifier, "feature_importances_") :
+
+		orderedFeatures = zip(classifier.feature_importances_ , range(0, len(classifier.feature_importances_)))
+		orderedFeatures = sorted(orderedFeatures, key = lambda x : x[0], reverse=True)
+	
+	# some classifiers are ensembles, and if each element in the ensemble is able to return a list of feature importances
+	# (that are going to be all scored following the same logic, so they could be easily aggregated, in theory)
+	elif hasattr(classifier, "estimators_") and hasattr(classifier.estimators_[0], "feature_importances_") :
+
+		# add up the scores given by each estimator to all features
+		global_score = np.zeros(classifier.estimators_[0].feature_importances_.shape[0])
+
+		for estimator in classifier.estimators_ :
+			for i in range(0, estimator.feature_importances_.shape[0]) :
+				global_score[i] += estimator.feature_importances_[i]
+
+		# "normalize", dividing by the number of estimators
+		for i in range(0, global_score.shape[0]) : global_score[i] /= len(classifier.estimators_)
+
+		# proceed as above to obtain the ranked list of features
+		orderedFeatures = zip(global_score, range(0, len(global_score)))
+		orderedFeatures = sorted(orderedFeatures, key = lambda x : x[0], reverse=True)
+	
+	# the classifier does not have "feature_importances_" but can return a list
+	# of all features used by a lot of estimators (typical of ensembles)
+	elif hasattr(classifier, "estimators_features_") :
+
+		numberOfFeaturesUsed = 0
+		featureFrequency = dict()
+		for listOfFeatures in classifier.estimators_features_ :
+			for feature in listOfFeatures :
+				if feature in featureFrequency :
+					featureFrequency[feature] += 1
+				else :
+					featureFrequency[feature] = 1
+			numberOfFeaturesUsed += len(listOfFeatures)
+		
+		for feature in featureFrequency : 
+			featureFrequency[feature] /= numberOfFeaturesUsed
+
+		# prepare a list of tuples (name, value), to be sorted
+		orderedFeatures = [ (featureFrequency[feature], feature) for feature in featureFrequency ]
+		orderedFeatures = sorted(orderedFeatures, key=lambda x : x[0], reverse=True)
+
+	# the classifier does not even have the "estimators_features_", but it's
+	# some sort of linear/hyperplane classifier, so it does have a list of
+	# coefficients; for the coefficients, the absolute value might be relevant
+	elif hasattr(classifier, "coef_") :
+	
+		# now, "coef_" is usually multi-dimensional, so we iterate on
+		# all dimensions, and take a look at the features whose coefficients
+		# more often appear close to the top; but it could be mono-dimensional,
+		# so we need two special cases
+		dimensions = len(classifier.coef_.shape)
+		#print("dimensions=", len(dimensions))
+		featureFrequency = None # to be initialized later
+		
+		# check on the dimensions
+		if dimensions == 1 :
+			featureFrequency = np.zeros(len(classifier.coef_))
+			
+			relativeFeatures = zip(classifier.coef_, range(0, len(classifier.coef_)))
+			relativeFeatures = sorted(relativeFeatures, key=lambda x : abs(x[0]), reverse=True)
+			
+			for index, values in enumerate(relativeFeatures) :
+				value, feature = values
+				featureFrequency[feature] += 1/(1+index)
+
+		elif dimensions > 1 :
+			featureFrequency = np.zeros(len(classifier.coef_[0]))
+			
+			# so, for each dimension (corresponding to a class, I guess)
+			for i in range(0, len(classifier.coef_)) :
+				# we give a bonus to the feature proportional to
+				# its relative order, good ol' 1/(1+index)
+				relativeFeatures = zip(classifier.coef_[i], range(0, len(classifier.coef_[i])))
+				relativeFeatures = sorted(relativeFeatures, key=lambda x : abs(x[0]), reverse=True)
+				
+				for index, values in enumerate(relativeFeatures) :
+					value, feature = values
+					featureFrequency[feature] += 1/(1+index)
+			
+		# finally, let's sort
+		orderedFeatures = [ (featureFrequency[feature], feature) for feature in range(0, len(featureFrequency)) ]
+		orderedFeatures = sorted(orderedFeatures, key=lambda x : x[0], reverse=True)
+
+	else :
+		print("The classifier does not have any way to return a list with the relative importance of the features")
+
+	return np.array(orderedFeatures)
+
+def featureSelection(globalIndex, variableSize,run, numberOfFolds) :
+	
+	classifierList = [
+			# ensemble
+			#[AdaBoostClassifier(), "AdaBoostClassifier"],
+			#[AdaBoostClassifier(n_estimators=300), "AdaBoostClassifier(n_estimators=300)"],
+			#[AdaBoostClassifier(n_estimators=1500), "AdaBoostClassifier(n_estimators=1500)"],
+			#[BaggingClassifier(), "BaggingClassifier"],
+			
+			[GradientBoostingClassifier(n_estimators=300), "GradientBoostingClassifier(n_estimators=300)"],
+			[RandomForestClassifier(n_estimators=300), "RandomForestClassifier(n_estimators=300)"],
+			[LogisticRegression(), "LogisticRegression"],
+			[PassiveAggressiveClassifier(),"PassiveAggressiveClassifier"],
+			[SGDClassifier(), "SGDClassifier"],
+			[SVC(kernel='linear'), "SVC(linear)"],
+			[RidgeClassifier(), "RidgeClassifier"],
+			[BaggingClassifier(n_estimators=300), "BaggingClassifier(n_estimators=300)"],
+			#[ExtraTreesClassifier(), "ExtraTreesClassifier"],
+			#[ExtraTreesClassifier(n_estimators=300), "ExtraTreesClassifier(n_estimators=300)"],
+			#[GradientBoostingClassifier(), "GradientBoostingClassifier"], # features_importances_
+			#[GradientBoostingClassifier(n_estimators=300), "GradientBoostingClassifier(n_estimators=300)"],
+			#[GradientBoostingClassifier(n_estimators=1000), "GradientBoostingClassifier(n_estimators=1000)"],
+			#[RandomForestClassifier(), "RandomForestClassifier"],
+			#[RandomForestClassifier(n_estimators=300), "RandomForestClassifier(n_estimators=300)"],
+			#[RandomForestClassifier(n_estimators=1000), "RandomForestClassifier(n_estimators=1000)"], # features_importances_
+
+			# linear
+			#[ElasticNet(), "ElasticNet"],
+			#[ElasticNetCV(), "ElasticNetCV"],
+			#[Lasso(), "Lasso"],
+			#[LassoCV(), "LassoCV"],
+			#[LogisticRegression(), "LogisticRegression"], # coef_
+			#[LogisticRegressionCV(), "LogisticRegressionCV"],
+			#[PassiveAggressiveClassifier(), "PassiveAggressiveClassifier"], # coef_
+			#[RidgeClassifier(), "RidgeClassifier"], # coef_
+			#[RidgeClassifierCV(), "RidgeClassifierCV"],
+			#[SGDClassifier(), "SGDClassifier"], # coef_
+			#[SVC(kernel='linear'), "SVC(linear)"], # coef_, but only if the kernel is linear...the default is 'rbf', which is NOT linear
+			
+			# naive Bayes
+			#[BernoulliNB(), "BernoulliNB"],
+			#[GaussianNB(), "GaussianNB"],
+			#[MultinomialNB(), "MultinomialNB"],
+			
+			# neighbors
+			#[KNeighborsClassifier(), "KNeighborsClassifier"], # no way to return feature importance
+			# TODO this one creates issues
+			#[NearestCentroid(), "NearestCentroid"], # it does not have some necessary methods, apparently
+			#[RadiusNeighborsClassifier(), "RadiusNeighborsClassifier"],
+			
+			# tree
+			#[DecisionTreeClassifier(), "DecisionTreeClassifier"],
+			#[ExtraTreeClassifier(), "ExtraTreeClassifier"],
+
+			]
+	
+	# this is just a hack to check a few things
+	#classifierList = [
+	#	   [RandomForestClassifier(), "RandomForestClassifier"]
+	#	   ]
+
+	print("Loading dataset...")
+	if (globalIndex==0):
+		X, y, biomarkerNames = loadDatasetOriginal(run)
+	else:
+		X, y, biomarkerNames = loadDataset(globalIndex, run)
+	
+	
+	
+	numberOfTopFeatures=int(variableSize)
+	# create folder
+	folderName ="./run"+str(run)+"/"
+	if not os.path.exists(folderName) : os.makedirs(folderName)
+	
+	# prepare folds
+	skf = StratifiedKFold(n_splits=numberOfFolds, shuffle=True)
+	indexes = [ (training, test) for training, test in skf.split(X, y) ]
+	
+	# this will be used for the top features
+	topFeatures = dict()
+	
+	# iterate over all classifiers
+	classifierIndex = 0
+	
+	globalAccuracy=0;
+	
+	for originalClassifier, classifierName in classifierList :
+		
+		print("\nClassifier " + classifierName)
+		classifierPerformance = []
+		classifierTopFeatures = dict()
+
+		# iterate over all folds
+		for train_index, test_index in indexes :
+			
+			X_train, X_test = X[train_index], X[test_index]
+			y_train, y_test = y[train_index], y[test_index]
+			
+			# let's normalize, anyway
+			scaler = StandardScaler()
+			X_train = scaler.fit_transform(X_train)
+			X_test = scaler.transform(X_test)
+
+			classifier = copy.deepcopy(originalClassifier)
+			classifier.fit(X_train, y_train)
+			#scoreTraining = classifier.score(X_train, y_train)
+			#scoreTest = classifier.score(X_test, y_test)
+			
+			y_new_test = classifier.predict(X_test)
+			for i in range(0, len(y_new_test)):
+				y_new_test[i]=round(y_new_test[i])
+			y_new_train = classifier.predict(X_train)
+			for i in range(0, len(y_new_train)):
+				y_new_train[i]=round(y_new_train[i])
+				
+			scoreTest = matthews_corrcoef(y_test, y_new_test)
+			scoreTraining = matthews_corrcoef(y_train, y_new_train)
+				
+			
+			
+			print("\ttraining: %.4f, test: %.4f" % (scoreTraining, scoreTest))
+			classifierPerformance.append( scoreTest )
+			
+			# now, let's get a list of the most important features, then mark the ones in the top X
+			orderedFeatures = relativeFeatureImportance(classifier) 
+			for i in range(0, numberOfTopFeatures) :
+				
+				feature = int(orderedFeatures[i][1])
+
+				if feature in topFeatures :
+					topFeatures[ feature ] += 1
+				else :
+					topFeatures[ feature ] = 1
+				
+				if feature in classifierTopFeatures :
+					classifierTopFeatures[ feature ] += 1
+				else :
+					classifierTopFeatures[ feature ] = 1
+			
+		line ="%s\t%.4f\t%.4f\n" % (classifierName, np.mean(classifierPerformance), np.std(classifierPerformance))
+		
+		globalAccuracy=globalAccuracy+np.mean(classifierPerformance)
+		
+		print(line)
+		fileName=folderName+"results.txt"
+		fo = open(fileName, 'a')
+		fo.write( line )
+		fo.close()
+		# save most important features for the classifier
+		with open( os.path.join(folderName, classifierName + ".csv"), "w" ) as fp :
+	
+			fp.write("feature,frequencyInTop" + str(numberOfTopFeatures) + "\n")
+			
+			# transform dictionary into list
+			listOfClassifierTopFeatures = [ (key, classifierTopFeatures[key]) for key in classifierTopFeatures ]
+			listOfClassifierTopFeatures = sorted( listOfClassifierTopFeatures, key = lambda x : x[1], reverse=True )
+			
+			for feature, frequency in listOfClassifierTopFeatures :
+				fp.write( str(biomarkerNames[feature]) + "," + str(float(frequency/numberOfFolds)) + "\n")
+	
+	# save most important features overall
+	with open( os.path.join(folderName, "global_"+str(int(globalIndex))+".csv"), "w" ) as fp :
+		
+		fp.write("feature,frequencyInTop" + str(numberOfTopFeatures) + "\n")
+		
+		# transform dictionary into list
+		listOfTopFeatures = [ (key, topFeatures[key]) for key in topFeatures ]
+		listOfTopFeatures = sorted( listOfTopFeatures, key = lambda x : x[1], reverse=True )
+		
+		tempIndex=0
+		for feature, frequency in listOfTopFeatures :
+			if tempIndex<numberOfTopFeatures:
+				fp.write( str(biomarkerNames[feature]) + "," + str(float(frequency/numberOfFolds)) + "\n")
+			tempIndex=tempIndex+1
+
+	globalAccuracy=globalAccuracy/8
+	return globalAccuracy
+

reduceData.py

@@ -0,0 +1,49 @@
+from pandas import read_csv
+import numpy as np
+
+import pandas as pd 
+
+def reduceDataset(globalIndex, run) :
+	
+	# data used for the predictions
+	if(globalIndex==0):
+		dfData = read_csv("../data/data_"+str(globalIndex)+".csv", header=None, sep=',')
+		ids=read_csv("../data/features_"+str(globalIndex)+".csv", header=None, sep=',')
+	else:
+		dfData = read_csv("./run"+str(run)+"/data_"+str(globalIndex)+".csv", header=None, sep=',')
+		ids=read_csv("./run"+str(run)+"/features_"+str(globalIndex)+".csv", header=None, sep=',')
+	
+	idsRed=read_csv("./run"+str(run)+"/global_"+str(globalIndex)+".csv", sep=',')
+	
+	data=dfData.values
+	idsRed=idsRed.values
+	
+	ids=ids.values
+
+	print("data Y %d" %(len(data)))
+	print("data X %d" %(len(data[0])))
+	print(len(ids))
+	print(len(idsRed))
+	
+	tempIds=[]
+	for i in range(0,len(idsRed)):
+		if (idsRed[i,1]>=1):
+			tempIds.append(idsRed[i,0])
+	print(len(tempIds))
+	count=0
+	
+	dataRed=np.zeros((len(data),len(tempIds)))
+
+	for i in range(0,len(tempIds)):
+		for j in range (0,len(ids)):
+			if (ids[j]== tempIds[i]):
+				count=count+1
+				for k in range(0,len(data)):
+					dataRed[k,i]=data[k,j]
+	
+	
+	pd.DataFrame(tempIds).to_csv("./run"+str(run)+"/features_"+str(globalIndex+1)+".csv", header=None, index =None)
+	pd.DataFrame(dataRed).to_csv("./run"+str(run)+"/data_"+str(globalIndex+1)+".csv", header=None, index =None)			
+	print(count)
+	
+	return len(ids),len(tempIds)

refs_mcc-1.0.0.dist-info/METADATA

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: refs-mcc
+Version: 1.0.0
+Summary: Recursive Ensemble Feature Selection using Matthews Correlation Coefficient
+Author: Alejandro Lopez-Rincon, Alberto Tonda, Brigitta Varga
+Author-email: Alejandro Lopez-Rincon <alejandrolopezrn@gmail.com>
+License: MIT
+Project-URL: source, https://github.com/steppenwolf0/REFS-MCC/
+Project-URL: tracker, https://github.com/steppenwolf0/REFS-MCC/issues
+Keywords: feature-selection,matthews-correlation-coefficient,bioinformatics,machine-learning,ensemble-learning
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: joblib>=1.5.3
+Requires-Dist: matplotlib>=3.10.8
+Requires-Dist: numpy>=2.4.1
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: scikit-learn>=1.8.0
+Requires-Dist: scipy>=1.17.0
+Requires-Dist: statsmodels>=0.14.6
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+
+# REFS-MCC
+Recursive Ensemble Feauture Selection using Matthews Correlation Coefficient
+
+--------------------------------------------------------------------
+Installation
+
+```bash
+   pip install refs-mcc
+```
+
+After installation it can be used as the following (using the default parameters):
+```python
+    from refs_mcc import REFS_MCC
+    REFS_MCC().run()
+```
+
+Or from CLI. For more information, run:
+```bash
+   refs-mcc --help
+```
+ 
+--------------------------------------------------------------------
+Input
+
+Next to the folder where the code is executed from, a `data` folder needs to be present with the following files:
+- `data_0.csv`
+- `features_0.csv`
+- `ids.csv`
+- `labels.csv`
+
+--------------------------------------------------------------------
+Output
+
+The following folders and files will be created:
+- run folders (`run0`, `run1`, ..., `run{n-1}`, where n is the selected number of total runs, 10 by default)
+- `best` folder 
+- `sumFig.pdf` & `sumFig.png`
+
+--------------------------------------------------------------------

refs_mcc-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+aBioInf100.py,sha256=saaD9_GwyWCd--Wc6EFaif6lcn_tMwdUbZyUyyxwmhA,1679
+classifiersMulti.py,sha256=uLSySF0v_cdcYQj6LWDHjwTYowxjju2c3sW35PW9YuU,7737
+features.py,sha256=UHl6NCinqlqjU286nxfTfEG9UMrCQwrcN8hkMRXcHUY,14511
+reduceData.py,sha256=sCzoheZn8clva2RQKWwH3B-4y3wilIi5DQPXyenSIcM,1444
+refs_mcc.py,sha256=IHzbaeSfgAIA3FtfGrtqAOXBIP5r8EdPfYwELtdn4L0,1118
+sumFig.py,sha256=sjljlTIeOhO7pzOxuFZHTqZzw9ikVYxIQWq47xrDnLY,2819
+summaryMulti.py,sha256=Wo44vWXY3z8OOeqFpW_tJKNHytr0lVewyFc855CVJak,3697
+refs_mcc-1.0.0.dist-info/METADATA,sha256=ZmZYO3QcYhNobci9wr2U-gRunWITqTnCCa4KTl0-hss,2615
+refs_mcc-1.0.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+refs_mcc-1.0.0.dist-info/entry_points.txt,sha256=Wdp8mPnH0-qa4cbpPd_xKeOJpAwZ19IsN7758QhyLlU,43
+refs_mcc-1.0.0.dist-info/top_level.txt,sha256=cihDROHg3c_S0-46mDNd9c59gG2s9tzrDySfZYnhpc8,77
+refs_mcc-1.0.0.dist-info/RECORD,,

refs_mcc-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

refs_mcc-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+refs-mcc = refs_mcc:main

refs_mcc-1.0.0.dist-info/top_level.txt

@@ -0,0 +1,7 @@
+aBioInf100
+classifiersMulti
+features
+reduceData
+refs_mcc
+sumFig
+summaryMulti

refs_mcc.py

@@ -0,0 +1,30 @@
+import argparse
+import sys
+
+from aBioInf100 import main as part1
+from summaryMulti import fakeBootStrapper as part2
+from sumFig import create_summary_figure as part3
+
+class REFS_MCC:
+    def __init__(self, threads: int = 10, totalRuns: int = 10, numberOfFolds: int = 10):
+        self.threads = threads
+        self.totalRuns = totalRuns
+        self.numberOfFolds = numberOfFolds
+
+    def run(self):
+        part1(self.threads, self.totalRuns, self.numberOfFolds)
+        part2(self.totalRuns, self.numberOfFolds)
+        part3(self.totalRuns)
+
+def main():
+    parser = argparse.ArgumentParser(description="REFS-MCC")
+    parser.add_argument('--threads', type=int, default=10, help='Number of threads (default: 10)')
+    parser.add_argument('--totalRuns', type=int, default=10, help='Total number of runs (default: 10)')
+    parser.add_argument('--folds', type=int, default=10, help='Number of folds (default: 10)')
+    args = parser.parse_args()
+
+    refs_mcc = REFS_MCC(args.threads, args.totalRuns, args.folds)
+    return refs_mcc.run()
+
+if __name__ == "__main__":
+    sys.exit(main())

sumFig.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jan  7 12:15:27 2021
+
+@author: alber
+"""
+# convert float to percentage string
+def convert_to_percentage(f) :
+    if f != "" :
+        print("Converting \"%.4f\"..." % f)
+        percentage = f 
+        return "%.1f" % percentage
+    else :
+        return ""
+
+# script to create a figure
+import sys
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.cm
+import argparse
+
+def create_summary_figure(totalRuns):
+
+    file2 = open('./best/sumA.csv', 'w')
+
+    #file2.write("features, run0, run1, run2, run3, run4, run5, run6, run7, run8, run9\n")
+
+    k = totalRuns
+
+    header = "features, " + ", ".join(f"run{i}" for i in range(k)) + "\n"
+    file2.write(header)
+
+    filepath = './best/sum.csv'
+    with open(filepath) as file1:
+        line = file1.readline()
+        cnt = 1
+        while line:
+            #print("Line {}: {}".format(cnt, line.strip()))
+            file2.write(line)
+            line = file1.readline()
+            cnt += 1
+    file1.close()
+    file2.close()
+
+
+
+    df = pd.read_csv("./best/sumA.csv")
+
+    x = df['features'].values
+
+    features=pd.read_csv("./best/features_0.csv", header=None)
+    print("len features:"+str(len(features.values)))
+
+    maxValue=len(features.values)
+
+
+    runs = [r for r in list(df) if r != 'features']
+
+    #We declare 15 because numbers 0-5 are almost white.
+    n_lines=15
+    c = np.arange(1, n_lines + 1)
+
+    norm = mpl.colors.Normalize(vmin=c.min(), vmax=c.max())
+    cmap = mpl.cm.ScalarMappable(norm=norm, cmap=mpl.cm.Blues)
+    cmap.set_array([])
+
+    import matplotlib.pyplot as plt
+    fig = plt.figure(figsize=(8,6))
+    ax = fig.add_subplot(111)
+    ax.set_xscale('log')
+
+    #starts in 5 because numbers 0-5 are almost white.
+    i=5
+    for r in runs :
+        y = df[r].values
+        ax.plot(x, y, label=r, c=cmap.to_rgba(i))
+        i=i+1
+
+    plt.xticks(x, [str(a) for a in list(x)], rotation=90, fontsize=6)
+    y_locs = ax.get_yticks()
+    print(y_locs)
+    plt.yticks(y_locs, [convert_to_percentage(p) for p in y_locs])
+
+    ax.axvline(linewidth=2, color='r', x=maxValue)
+    ax.grid(linestyle='--')    
+    ax.legend(loc='best')
+    ax.set_xlabel("Number of features (log scale)")
+    ax.set_ylabel("Ensemble MCC")
+    ax.set_title("MCC vs number of features in REFS runs")
+    plt.savefig("sumFig.pdf")
+    plt.savefig("sumFig.png", dpi=300)
+
+if __name__ == "__main__" :
+    parser = argparse.ArgumentParser(description="Create summary figure")
+    parser.add_argument('--totalRuns', type=int, default=10, help='Total number of runs (default: 10)')
+    args = parser.parse_args()
+
+    totalRuns = args.totalRuns
+
+    sys.exit( create_summary_figure(totalRuns) )

summaryMulti.py

@@ -0,0 +1,128 @@
+import numpy as np
+import os
+import sys
+import pandas as pd 
+import argparse
+
+from classifiersMulti import *
+
+
+from pandas import read_csv
+
+directory="data"
+
+def fakeBootStrapper(runs, numberOfFolds):
+	# create folder
+	folderName ="./best/"
+	if not os.path.exists(folderName) : os.makedirs(folderName)
+
+	orig_stdout = sys.stdout
+	f = open('./best/out.txt', 'w')
+	sys.stdout = f
+	
+	directory="run"+str(0)
+	f=open("./"+directory+"/results.txt", "r")
+	fl =f.readlines()
+	blocks=int(len(fl)/8)
+	print(blocks)
+	results=np.zeros((blocks,runs+1))
+	for j in range(0,runs):
+		directory="run"+str(j)
+		f=open("./"+directory+"/results.txt", "r")
+		fl =f.readlines()
+		blocks=int(len(fl)/8)
+		count=0
+		value=0
+		variables=np.zeros(blocks)
+		accuracy=np.zeros(blocks)
+		indexResults=0
+		for x in fl:
+			a=x.split("\t")
+			value=value+float(a[1])/8.0
+			count=count+1
+			if (count==8):
+				accuracy[indexResults]=value
+				value=0
+				count=0
+				indexResults=indexResults+1
+		indexResults=0
+		
+		for i in range (0,blocks):
+			dfFeats = (read_csv("./"+directory+"/features_"+str(i)+".csv", header=None)).values.ravel() 
+			variables[i]=len(dfFeats)
+			results[i,j+1]=accuracy[i]
+			results[i,0]=variables[i]
+		
+		
+	
+	pd.DataFrame(results).to_csv("./best/sum.csv", header=None, index =None)
+	
+	bestVal=np.zeros(runs)
+	bestSize=np.zeros(runs)
+	bestPos=np.zeros(runs)
+	for j in range(0,runs):
+		bestVal[j]=np.max(results[:,j+1])
+	
+	for j in range(0,runs):	
+		for i in range (0,blocks):
+			if ( bestVal[j]==results[i,j+1]):
+				bestSize[j]=int(results[i,0])
+				bestPos[j]=i
+	print(bestVal)
+	print(bestSize)
+	print(bestPos)
+	
+	bestFeatures=[]
+	signatures=[]
+	for j in range(0,runs):
+		dfFeats = (read_csv("./run"+str(j)+"/features_"+str(int(bestPos[j]))+".csv", header=None))
+		bestFeatures.extend(dfFeats.values.ravel())
+		signatures.append(dfFeats.values.ravel())
+	#print(bestFeatures)
+	signatures.append(bestVal)
+	signatures.append(bestSize)
+	pd.DataFrame(signatures).to_csv("./best/signatures.csv", header=None, index =None)
+	
+	unique, counts = np.unique(bestFeatures, return_counts=True)
+	
+	resultsFeatures=np.zeros((len(unique),2), 'U16')
+	for j in range(0,len(unique)):
+		resultsFeatures[j,0]=unique[j]
+		resultsFeatures[j,1]=counts[j]
+	
+	
+	pd.DataFrame(resultsFeatures).to_csv("./best/resultsFeatures.csv", header=None, index =None)
+	
+	print(np.max(bestVal))
+	print(np.argmax(bestVal))
+	print(int(bestPos[np.argmax(bestVal)]))
+	
+	runBest=int(np.argmax(bestVal))
+	indexBest=int(bestPos[np.argmax(bestVal)])
+	
+	
+	
+	# data used for the predictions
+	dfData = read_csv("./run"+str(runBest)+"/data_"+str(indexBest)+".csv", header=None, sep=',')
+	dfLabels = read_csv("./run"+str(runBest)+"/labels.csv", header=None)
+	biomarkers = read_csv("./run"+str(runBest)+"/features_"+str(indexBest)+".csv", header=None)
+	
+	pd.DataFrame(dfData.values).to_csv("./best/data_0.csv", header=None, index =None)
+	pd.DataFrame(biomarkers.values.ravel()).to_csv("./best/features_0.csv", header=None, index =None)
+	pd.DataFrame(dfLabels.values.ravel()).to_csv("./best/labels.csv", header=None, index =None)
+	
+	runFeatureReduce(numberOfFolds)
+	sys.stdout = orig_stdout
+	f.close()
+	return
+
+if __name__ == "__main__" :
+	parser = argparse.ArgumentParser(description="Run REFS-MCC - part 2")
+	parser.add_argument('--totalRuns', type=int, default=10, help='Total number of runs (default: 10)')
+	parser.add_argument('--folds', type=int, default=10, help='Number of folds (default: 10)')
+	args = parser.parse_args()
+
+	runs=args.totalRuns
+	numberOfFolds=args.folds
+
+	sys.exit( fakeBootStrapper(runs, numberOfFolds) )