molSimplify 1.7.4__py3-none-any.whl

molSimplify/Scripts/findcorrelations.py

@@ -0,0 +1,287 @@
+# @file findcorrelations.py
+#  Automated correlation analysis module
+#
+#   Written by JP Janet for HJK Group
+#
+#  Dpt of Chemical Engineering, MIT
+
+import os
+import sys
+import numpy as np
+from molSimplify.Classes.dft_obs import (dft_observation)
+from sklearn import linear_model, preprocessing, metrics, feature_selection, model_selection
+
+# def test_skl():
+#     valid = 'ok'
+#     try:
+#         from sklearn import linear_model, preprocessing, metrics, feature_selection, model_selection
+#     except:
+#         valid = False
+#     return valid
+
+
+def analysis_supervisor(args, rootdir):
+    status = True
+    print('looking for scikit-learn')
+    # if test_skl():
+    #     pass
+    # else:
+    #     print("Error, scikit-learn not loadable")
+    #     status = False
+    if not args.correlate:
+        print("Error, correlation path not given")
+        status = False
+    print(('looking for file at '+str(args.correlate)))
+    if not args.correlate:
+        print("Error, correlation path not given")
+        status = False
+    if not os.path.exists(args.correlate) and status:
+        print(("Error, correlation file not found at" + str(args.correlate)))
+        status = False
+    if not status:
+        print('correlation cannot begin! Exiting...')
+        sys.exit()
+    if args.lig_only:
+        print('using ligand-only descriptors (assuming all metals are the same)')
+    if args.simple:
+        print('using simple autocorrelation descriptors only')
+    if args.max_descriptors:
+        print(('using a max of '+str(args.max_descriptors)+' only'))
+    correlation_supervisor(
+        args.correlate, rootdir, args.simple, args.lig_only, args.max_descriptors)
+
+
+def accquire_file(path):
+    # set display options
+    np.set_printoptions(precision=3)
+    # this function reads in values from a correctly formated path
+    # [name],[y],[folder_name]
+    file_dict = dict()
+    fail_dict = dict()
+    counter = 0  # number of added paths
+    ncounter = 0  # number of skipped paths
+    if os.path.exists(path):
+        print('found file, opening...')
+        with open(path, 'r') as f:
+            # expects csv fomart,
+            ### value | path
+            for i, lines in enumerate(f):
+                print(('read line: ' + str(i)))
+                ll = lines.strip('\n\r').split(",")
+                name = ll[0]
+                if i == 0:
+                    # this is the first line
+                    # y_value_name = ll[1]
+                    # check if path exists:
+                    # paths_name = ll[2].strip('/') + '/'+name+'.xyz'
+                    if len(ll) > 3:
+                        print('custom descriptors found!')
+                        custom_names = [ll[i] for i in range(4, len(ll))]
+
+                else:
+                    y_value = ll[1]
+                    # check if path exists:
+                    this_path = '/'+ll[2].strip('/') + '/'+name+'.xyz'
+                    # print('name = '+str(name))
+                    this_obs = dft_observation(name, this_path)
+                    this_obs.sety(y_value)
+                    if os.path.isfile(this_path):
+                        # print('path exists')
+                        this_obs.obtain_mol3d()
+                        if this_obs.health:
+                            counter += 1
+                            file_dict.update({counter: this_obs})
+                            if len(ll) > 3:
+                                print(
+                                    ('custom descriptors found for job ' + str(name)))
+                                custom_descriptors = float(
+                                    [ll[i] for i in range(4, len(ll))])
+                                this_obs.append_descriptors(
+                                    custom_names, custom_descriptors, '', '')
+                        else:  # bad geo
+                            this_obs.comments.append(
+                                ' geo is not healthy, culling ' + str(this_path))
+                            ncounter += 1
+                            fail_dict.update({counter: this_obs})
+                    else:  # no geo file found
+                        this_obs.comments.append(
+                            ' geo could not be found: ' + str(this_path))
+                        ncounter += 1
+                        fail_dict.update({counter: this_obs})
+    if counter > 0:
+        print(('file import successful, ' + str(counter) + ' geos loaded'))
+    len_fail = len(list(fail_dict.keys()))
+    if len_fail > 0:
+        print((str(len_fail) + ' unsuccessful imports :'))
+        for keys in list(fail_dict.keys()):
+            print(('failed at line ' + str(keys) +
+                   ' for job ' + str(fail_dict[keys].name)))
+    return(file_dict, fail_dict)
+
+
+def correlation_supervisor(path, rootdir, simple=False, lig_only=False, max_descriptors=False):
+    # load the files from the given input file
+    file_dict, fail_dict = accquire_file(path)
+    # loop over sucessful imports to get descriptors:
+    big_mat = list()
+    col_names = list()
+    for i, keyv in enumerate(file_dict.keys()):
+        file_dict[keyv].get_descriptor_vector(
+            lig_only, simple, name=False, loud=False)
+        # print('i = ',str(i))
+        if i == 0:
+            col_names = file_dict[keyv].descriptor_names
+        # reorganize the data
+        this_row = list()
+        this_row.append(float(file_dict[keyv].yvalue))
+        this_row.extend(file_dict[keyv].descriptors)
+        big_mat.append(this_row)
+    big_mat = np.array(big_mat)
+    # let's do some regression
+    # standardize model:
+    col_array = np.array(col_names)
+    print(('length of col array is  ' + str(len(col_array))))
+    n_tot = len(col_array)
+    X = big_mat[:, 1:]
+    print(('dimension of data matrix is ' + str(big_mat.shape)))
+    n_obs = len(X[:, 1])
+    Scaler = preprocessing.StandardScaler().fit(X)
+    Xs = Scaler.transform(X)
+    Y = big_mat[:, 0]
+    # find baseline model (all descriptors)
+    Reg = linear_model.LinearRegression()
+    Reg.fit(Xs, Y)
+    Ypred_all_all = Reg.predict(Xs)
+    rs_all_all = metrics.r2_score(Y, Ypred_all_all)
+    loo = model_selection.LeaveOneOut()
+    r_reduce = list()
+    mse_reduce = list()
+    # stepwise reduce the feature set until only one is left
+    for n in range(0, n_tot):
+        reductor = feature_selection.RFE(Reg, n_features_to_select=n_tot-n,
+                                         step=1, verbose=0)
+        reductor.fit(Xs, Y)
+        Ypred_all = reductor.predict(Xs)
+        rs_all = metrics.r2_score(Y, Ypred_all)
+        mse_all = metrics.mean_squared_error(Y, Ypred_all)
+    r_reduce.append(rs_all)
+    mse_reduce.append(mse_all)
+    # reduce to one feature
+
+    reductor_features = list()
+    for i, ranks in enumerate(reductor.ranking_):
+        reductor_features.append([col_array[i], ranks])
+    reductor_features = sorted(reductor_features, key=lambda x: x[1])
+    # print(reductor_features)
+    print('****************************************')
+    # select best number using cv
+    selector = feature_selection.RFECV(
+        Reg, step=1, cv=loo, verbose=0, scoring='neg_mean_squared_error')
+    selector.fit(Xs, Y)
+    select_mse = selector.grid_scores_
+    Ypred = selector.predict(Xs)
+    rs = metrics.r2_score(Y, Ypred)
+    n_opt = selector.n_features_
+    opt_features = col_array[selector.support_]
+    ranked_features = list()
+    for i, ranks in enumerate(selector.ranking_):
+        ranked_features.append([col_array[i], ranks])
+    ranked_features = sorted(ranked_features, key=lambda x: x[1])
+    print(ranked_features)
+    if max_descriptors:  # check if we need to reduce further
+        print(('a max of ' + str(max_descriptors) + ' were requested'))
+        n_max = int(max_descriptors)
+        if n_opt > n_max:
+            print(('the RFE process selected ' +
+                   str(n_opt) + ' varibles as optimal'))
+            print(('discarding an additional ' + str(n_max-n_opt) + ' variables'))
+            new_variables = list()
+            for i in range(0, n_max):
+                new_variables.append(ranked_features[i])
+    # report results to user
+    print(('analzyed ' + str(n_obs) + ' molecules'))
+    print(('the full-space R2 is  '+str("%0.2f" %
+                                        rs_all_all) + ' with ' + str(n_tot) + ' features'))
+    print(('optimal number of features is ' +
+           str(n_opt) + ' of total ' + str(n_tot)))
+    print(('the opt R2 is  '+str("%0.2f" % rs)))
+
+    # print(ranked_features)
+    X_r = selector.transform(Xs)
+    reg_red = linear_model.LinearRegression()
+    reg_red.fit(X_r, Y)
+    Ypred_r = reg_red.predict(X_r)
+    coefs = reg_red.coef_
+    intercept = reg_red.intercept_
+    mse_all = metrics.mean_squared_error(Y, Ypred_all_all)
+    mse_r = metrics.mean_squared_error(Y, Ypred_r)
+    if n_opt < 30:
+        print(('the optimal variables are: ' + str(opt_features)))
+        print(('the coefficients are' + str(coefs)))
+    else:
+        print(('the (first 30) optimal variables are: ' +
+               str(opt_features[0:29])))
+        print(('the (first 30) coefficients are' + str(coefs[0:29])))
+    print(('the intercept is ' + str("%0.2f" % intercept)))
+    print(('the  training MSE with the best feature set is ' + str("%0.2f" % mse_r)))
+    print(('the MSE  with all features  is ' + str("%0.2f" % mse_all)))
+    print(('by eliminating ' + str(n_tot - n_opt) + ' features,' +
+           ' CV-prediction MSE decreased from ' + str("%0.0f" % abs(select_mse[0])) + ' to ' + str("%00f" % abs(select_mse[n_tot - n_opt]))))
+    with open(rootdir+'RFECV_rankings.csv', 'w') as f:
+        f.write('RFE_rank,RFE_col,RFECV_rank,RFECV_col, \n')
+        for i, items in enumerate(reductor_features):
+            f.write(str(items[0]) + ',' + str(items[1]) + ',' +
+                    str(ranked_features[i][0]) + ',' + str(ranked_features[i][1]) + '\n')
+    with open(rootdir + 'y_data.csv', 'w') as f:
+        for items in Y:
+            f.write(str(items) + '\n')
+    with open(rootdir + 'y_pred_r.csv', 'w') as f:
+        for items in Ypred_r:
+            f.write(str(items) + '\n')
+    with open(rootdir+'optimal_decriptor_space.csv', 'w') as f:
+        for i in range(0, n_obs):
+            for j in range(0, n_opt):
+                if j == (n_opt-1):
+                    f.write(str(X_r[i][j])+'\n')
+                else:
+                    f.write(str(X_r[i][j])+',')
+    with open(rootdir+'full_descriptor_space.csv', 'w') as f:
+        for names in col_names:
+            f.write(names+',')
+        f.write('\n')
+        for i in range(0, n_obs):
+            for j in range(0, n_tot):
+                if j == (n_tot-1):
+                    f.write(str(Xs[i][j])+'\n')
+                else:
+                    f.write(str(Xs[i][j])+',')
+    with open(rootdir+'scaling.csv', 'w') as f:
+        means = Scaler.mean_
+        var = Scaler.var_
+        f.write('name, mean,variance \n')
+        for i in range(0, n_tot):
+            f.write(str(col_names[i])+','+str(means[i]) + ',' +
+                    str(var[i])+','+str(selector.ranking_[i])+'\n')
+    with open(rootdir+'coeficients.csv', 'w') as f:
+        f.write('intercept,'+str(intercept) + '\n')
+        for i in range(0, n_opt):
+            f.write(str(opt_features[i])+','+str(coefs[i])+'\n')
+    with open(rootdir + 'rfe_mse.csv', 'w') as f:
+        f.write('features removed,mean CV error,'+str(intercept) + '\n')
+        count = 0
+        for items in mse_reduce:
+            f.write(str(count)+','+str(items) + '\n')
+            count += 1
+# with open('y_full_all.csv','w') as f:
+    # for items in Ypred_all_all:
+        # f.write(str(items) + '\n')
+# with open('rfe_r.csv','w') as f:
+    # for items in r_reduce:
+        # f.write(str(items) + '\n')
+
+# with open('select_mse.csv','w') as f:
+    # for items in select_mse:
+        # f.write(str(items) + '\n')
+# with open('errors.csv','w') as f:
+    # for items in errors:
+        # f.write(str(items) + '\n')

molSimplify/Scripts/generator.py

@@ -0,0 +1,267 @@
+# @file generator.py
+#  Main script that coordinates all parts of the program.
+#
+#  Written by Kulik Group
+#
+#  Department of Chemical Engineering, MIT
+
+import os
+import sys
+import glob
+import argparse
+import copy
+from molSimplify.Classes.globalvars import (globalvars)
+from molSimplify.Scripts.addtodb import addtoldb
+from molSimplify.Scripts.io import loadcdxml
+from molSimplify.Scripts.cellbuilder import (slab_module_supervisor)
+from molSimplify.Scripts.chains import (chain_builder_supervisor)
+from molSimplify.Scripts.dbinteract import (dbsearch)
+from molSimplify.Scripts.findcorrelations import (analysis_supervisor)
+from molSimplify.Scripts.inparse import (checkinput,
+                                         cleaninput,
+                                         parseall,
+                                         parseinputfile)
+from molSimplify.Scripts.postproc import (postproc)
+from molSimplify.Scripts.rungen import (constrgen,
+                                        multigenruns,
+                                        draw_supervisor)
+
+
+def startgen_pythonic(input_dict={'-core': 'fe', '-lig': 'cl,cl,cl,cl,cl,cl'},
+                      argv=['main.py', '-i', 'asdfasdfasdfasdf'],
+                      flag=True,
+                      gui=False,
+                      write=False):
+    """This is the main way to generate structures completely within Python.
+
+        Parameters
+        ----------
+            input_dict : dict
+                Argument list in the form of a dictionary.
+            argv : list
+                Default argument list used to "fool" startgen into accepting input_dict.
+            flag : bool, optional
+                Flag for printing information. Default is True.
+            gui : bool, optional
+                Flag for GUI. Default is False.
+            write : bool, optional
+                Flag to generate outputfile from python
+
+        Returns
+        -------
+            strfiles : str
+                Folder containing the runs.
+            emsg : bool
+                Flag for error. If error, returns a string with error.
+            this_diag : rundiag
+                Rundiag class instance that contains ANN attributes (this_diag.ANN_attributes)
+                and a mol3D class instance (this_diag.mol).
+
+    """
+    # from molSimplify.Scripts.generator import startgen_pythonic
+    inputfile_str = '\n'.join([k + ' ' + v for k, v in list(input_dict.items())])
+    if write:
+        startgen(argv, flag, gui, inputfile_str, write_files=write)
+    else:
+        strfiles, emsg, this_diag = startgen(argv, flag, gui, inputfile_str, write_files=write)
+        return (strfiles, emsg, this_diag)
+
+
+# Coordinates subroutines
+#  @param argv Argument list
+#  @param flag Flag for printing information
+#  @param gui Flag for GUI
+#  @return Error messages
+def startgen(argv, flag, gui, inputfile_str=None, write_files=True):
+    """Coordinates subroutines.
+
+        Parameters
+        ----------
+            argv : list
+                Argument list.
+            flag : bool
+                Flag for printing information.
+            gui : bool
+                Flag for GUI.
+            inputfile_str : str, optional
+                Optional input passed in as a string. Default is None.
+            write_files : bool, optional
+                Flag for whether or not files should be written. Should set to false for pythonic generation.
+
+        Returns
+        -------
+            emsg : bool
+                Flag for error. If error, returns a string with error.
+
+    """
+    emsg = False
+    # check for configuration file
+    # homedir = os.path.expanduser("~")
+    # configfile = False if not glob.glob(homedir+'/.molSimplify') else True
+    # if not configfile:
+    #    print("It looks like the configuration file '~/.molSimplify' does not exist!"
+    #          "Please follow the next steps to configure the file.")
+    #    instdir = raw_input("Please select the full path of the top installation directory for the program: ")
+    #    cdbdir = raw_input("Please specify the full path of the directory containing chemical databases:")
+    #    mwfn = raw_input("Specify the full path to the Multiwfn executable (for post-processing):")
+    #    with open(homedir+'/.molSimplify','w') as f:
+    #        if len(instdir) > 1:
+    #            f.write("INSTALLDIR="+instdir+'\n')
+    #        if len(cdbdir) > 1:
+    #            f.write("CHEMDBDIR="+cdbdir+'\n')
+    #        if len(mwfn) > 1 :
+    #            f.write("MULTIWFN="+mwfn[0]+'\n')
+    # ## end set-up configuration file ###
+    # ########### GLOBALS DEFINITION ############
+    globs = globalvars()
+    # installdir = globs.installdir
+    rundir = globs.rundir
+    PROGRAM = globs.PROGRAM
+    # ##### END GLOBALS DEFINITION ##############
+    # correct installdir
+    # if installdir[-1]!='/':
+    #    installdir+='/'
+    # print welcome message
+    ss = "\n************************************************************"
+    ss += "\n******** Welcome to "+PROGRAM+"! Let's get started. ********\n"
+    ss += "************************************************************\n\n"
+    if not flag:
+        print(ss)
+    sys.argv = argv
+    parser = argparse.ArgumentParser()
+    args = parseall(parser)
+    # check if input file exists
+    if not glob.glob(args.i) and not inputfile_str:
+        emsg = 'Input file '+args.i+' does not exist. Please specify a valid input file.\n'
+        print(emsg)
+        return emsg
+    args.gui = gui  # add gui flag
+    # parse input file
+    if args.i or inputfile_str:
+        parseinputfile(args, inputfile_str=inputfile_str)
+    if args.cdxml:
+        print('converting cdxml file into xyz')
+        cdxml = args.cdxml[0]
+        fname, msg = loadcdxml(cdxml)
+        print(msg)
+        if 'two' in msg:
+            core = fname + '_cat.xyz'
+            sub = fname + '_sub.xyz'
+            args.core = [core]
+            args.substrate = [sub]
+            args.tsgen = True
+    if args.custom_data_dir is not None:
+        globs.custom_path = args.custom_data_dir
+
+    # if not args.postp and not args.dbsearch and not args.dbfinger and not args.drawmode
+    # and not (args.slab_gen or args.place_on_slab) and not (args.chain) and not (args.correlate):
+    # check input arguments
+    if (not args.postp and not args.dbsearch and not args.dbfinger
+            and not (args.slab_gen or args.place_on_slab)
+            and not (args.chain) and not (args.correlate)):
+
+        # check input arguments
+        print('Checking input...')
+        if args.tsgen:
+            emsg = checkinput(args, calctype="tsgen")
+        elif args.ligadd:
+            emsg = checkinput(args, calctype="dbadd")
+        else:
+            emsg = checkinput(args)
+        # check before cleaning input arguments and clean only if checked
+        cleaninput(args)
+    args.gui = False  # deepcopy will give error
+    if emsg:
+        del args
+        return emsg
+    # check for jobs directory
+    rundir = args.rundir+'/' if (args.rundir) else rundir
+    if not os.path.isdir(rundir):
+        if write_files:
+            os.mkdir(rundir)
+    # ################## START MAIN ####################
+    args0 = copy.deepcopy(args)  # save initial arguments
+    # add gui flag
+    args.gui = gui
+    # postprocessing run?
+
+    if (args.postp):
+        postproc(rundir, args, globs)
+    # database search?
+    elif (args.dbsearch or args.dbfinger):
+        emsg = dbsearch(rundir, args, globs)
+        if emsg:
+            del args
+            return emsg
+        else:
+            print('Successful database search!\n')
+    # random generation?
+    elif (args.rgen):  # check if random generation was requested
+        if args.charge:
+            args.charge = args.charge[0]
+        if args.spin:
+            args.spin = args.spin[0]
+        corests = args.core
+        for cc in corests:
+            args = copy.deepcopy(args0)
+            # add gui flag
+            args.gui = gui
+            args.core = cc
+            if (args.lig or args.coord or args.lignum or args.ligocc):  # constraints given?
+                args, emsg = constrgen(rundir, args)
+                if emsg:
+                    del args
+                    return emsg
+            else:
+                emsg = 'For random generation specify at least a ligand, coordination or ligand types.\n'
+                print(emsg)
+                del args
+                return emsg
+    elif args.drawmode:
+        draw_supervisor(args, rundir)
+    # slab/place on slab?
+    elif (args.slab_gen or args.place_on_slab):
+        emsg = slab_module_supervisor(args, rundir)
+    # chain builder
+    elif (args.chain):
+        print('chain on')
+        emsg = chain_builder_supervisor(args, rundir)
+    # correlation analysis
+    elif (args.correlate):
+
+        print('analysis is looking for correlations')
+        analysis_supervisor(args, rundir)
+    # add ligand to list
+    elif (args.ligadd):
+        print(('adding ' + str(args.ligadd) + ' to ligand database  with name ' +
+               args.ligname + ' and connection atom(s) ' + str(args.ligcon)))
+        addtoldb(smimol=args.ligadd, sminame=args.ligname, smident=len(args.ligcon),
+                 smicat=str(args.ligcon).strip('[]'), smigrps="custom", smictg="custom", ffopt=args.ligffopt)
+
+    # normal structure generation or transition state building
+    else:
+        args = copy.deepcopy(args0)
+        # add gui flag
+        args.gui = gui
+        corests = args.core
+        # if args.tsgen: # goes through multigenruns for maximum interoperability
+        #     print('building a transition state')
+        if args.tsgen:  # goes through multigenruns for maximum interoperability
+            print('building a transition state')
+        else:
+            print('building an equilibrium complex')
+        for cc in corests:
+            args.core = cc
+            emsg = multigenruns(rundir, args, write_files=write_files)
+            if emsg:
+                print(emsg)
+                del args
+                return emsg
+    ss = "\n**************************************************************"
+    ss += "\n***** Thank you for using "+PROGRAM+". Have a nice day! ******\n"
+    ss += "**************************************************************"
+    ss += globs.about
+    if not flag:
+        print(ss)
+    del args
+    return emsg