testsuit 0.1.1__py3-none-any.whl

testsuit/__init__.py

@@ -0,0 +1,15 @@
+"""Toolbox for preparing and analysing test data.
+
+Subpackages:
+    datatools     load data files into pandas, convert to HDF5, push to a DB, datapacks
+    exploit       mexploit YAML scenario checks and the exploit runner
+    jupytertools  ipywidgets GUIs for notebooks
+    misc          logging, progress reporting, file helpers
+    cli           command-line entry points (see [project.scripts] in pyproject.toml)
+"""
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("testsuit")
+except PackageNotFoundError:  # running from a source tree without install
+    __version__ = "0+unknown"

testsuit/cli/apply_clock_correction.py

@@ -0,0 +1,167 @@
+
+import os
+import argparse
+import concurrent.futures
+import sys
+
+
+from testsuit.datatools.datatoolbox import loadDataframeFromFile
+from testsuit.datatools.DataFileMgrs.AFileMgr import enable_pandas_display_helpers
+from testsuit.datatools.DataFileMgrs.FolderParamMgr import FolderParamMgr
+from testsuit.datatools.DataframeToHdf5 import DataframeToHdf5
+
+from testsuit.misc.logger import get_logger
+from testsuit.misc.logger import create_logger
+from testsuit.misc.MonitorProgress import MonitorProgress,consoleRichProgressCb
+
+CORRECTED_CLOCK_FILE_SUFFIX="clock_corrected"
+
+NB_WORKERS=8
+
+def _process_file_with_clock_correction(fpath, clock_correction_file, shiftDateSec, shiftDateRegex, monitorProgress, deleteIfExists=True):
+    """Worker function for multithreaded clock correction processing."""
+    monitorProgress.msg(msg=f"applying dates correction on file: {fpath}")
+
+    if clock_correction_file:
+        mgr = FolderParamMgr([fpath, clock_correction_file], shiftDateSec=shiftDateSec, shiftDateRegex=shiftDateRegex, shiftDateInverted=True,continueOnError=True)
+    else:
+        mgr = FolderParamMgr([fpath], shiftDateSec=shiftDateSec, shiftDateRegex=shiftDateRegex, shiftDateInverted=True,continueOnError=True)
+    try:        
+        dfs = mgr.findParams(os.path.basename(fpath) + "::", monitorProgress=monitorProgress)#,silent=True)        
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        monitorProgress.msg(msg=f"could not apply clock correction to {fpath} : " + str(e), msgSeverity="error")
+        return
+
+    comment = {}
+    comment["Comments"] = "Extracted from original data"
+
+    if len(dfs) == 0:
+        monitorProgress.msg(msg=f"No param in {os.path.basename(fpath)}", msgSeverity="warning")
+        return
+
+    targetFile = dfs[0].origin.replace(".h5", f".{CORRECTED_CLOCK_FILE_SUFFIX}.h5")
+    if deleteIfExists and os.path.exists(targetFile):
+        os.remove(targetFile)
+        
+    h5GroupName = os.path.basename(dfs[0].origin)
+    for df in dfs:
+        if "clock corrected" in df.attrs:
+            DataframeToHdf5(targetFile, h5GroupName, df, comment=comment, chunk_size=True)
+
+    monitorProgress.msg(msg=f"written {targetFile}")
+    
+def apply_clock_correction(target_folder, 
+                           shiftDateSec, shiftDateRegex, shiftDateSrcFile, shiftDateInverted=False, 
+                           monitorProgress=None,deleteIfExist=True):
+    """
+    Apply given clock correction (either constant litteral or a param name to be loaded.
+    Actual processing is planned to be applicable on-the-fly, and thus is implemented in
+    FolderParamMgr::findParams() and AFileMgr::finalizeParam() methods."""
+     
+    monitorProgress.msg(msg=[f"correcting clock for {shiftDateRegex}",
+                    f"clock correction from {shiftDateSec} in {shiftDateSrcFile} "])
+    
+    
+    # try to interpret provided shitDateSec as a float litteral, if not, it is considered as a param name
+    try: shiftDateSec=float(shiftDateSec)
+    except (TypeError, ValueError): pass
+            
+    clock_correction_file=None
+    
+    totalSteps=3 if isinstance(shiftDateSec,str) else 2
+    
+    monitorProgress.set_total_items(totalSteps)
+    subFindMatchingDfmP = monitorProgress.child("find matching params")
+    
+    # early check of availability of provided shitDateSec param name
+    # if not str, it is a float litteral constant correction value
+    if isinstance(shiftDateSec,str):
+        
+        clock_correction_file = shiftDateSrcFile
+        if not os.path.exists(clock_correction_file) and not clock_correction_file.startswith(os.sep):
+            clock_correction_file = os.path.join(target_folder, shiftDateSrcFile)
+            monitorProgress.msg(msg=f"Using file {clock_correction_file} to get clock correction info '{shiftDateSec}'.",
+                    msgSeverity="info")
+        if not os.path.exists(clock_correction_file):
+            monitorProgress.msg(msg=f"File {clock_correction_file} not reachable. Cannot apply clock correction '{shiftDateSec}'.",
+                        msgSeverity="error")
+            return False
+        
+        shiftdatefilemgr = FolderParamMgr(clock_correction_file)
+        shiftDateMatchingParams = shiftdatefilemgr.findParams(shiftDateSec, dryRun=True, silent=True, monitorProgress=subFindMatchingDfmP)
+        shiftDateParamNames = [df.name for df in shiftDateMatchingParams]
+        if len(shiftDateParamNames) == 0:
+            monitorProgress.msg(msg=f"No param matching shiftDateSec='{shiftDateSec}' not found in {clock_correction_file}.",
+                        msgSeverity="error")
+            return False        
+        if len(shiftDateParamNames) > 1:
+            monitorProgress.msg(msg=f"more than one param matching shiftDateSec='{shiftDateSec}' in {clock_correction_file}: {shiftDateParamNames}",
+                    msgSeverity="error")
+            return False 
+   
+    subLoadMatchingDfmP = monitorProgress.child("load matching params")
+    
+    # list all params matching given regex, to identify corresponding files
+    matching_dfs = loadDataframeFromFile(sourceFolderOrFile=target_folder, paramRegexes=shiftDateRegex, 
+                                         indices=None, dryRun=True, silent=True, monitorProgress=subLoadMatchingDfmP)
+    if len(matching_dfs) == 0:
+        monitorProgress.msg(msg=["found no data needing clock correction", f"pattern: {shiftDateRegex}"], msgSeverity="warning")
+        return False
+        
+    files_to_process = list(set(df.origin for df in matching_dfs if (df.origin and CORRECTED_CLOCK_FILE_SUFFIX not in df.origin)))
+    subApplyClockmP = monitorProgress.child("apply clock correction",len(files_to_process))
+    
+    # Parallelize file processing with multithreading
+    with concurrent.futures.ThreadPoolExecutor(max_workers=NB_WORKERS) as executor:
+        futures = {
+            executor.submit(
+                _process_file_with_clock_correction,
+                fpath,
+                clock_correction_file,
+                shiftDateSec,
+                shiftDateRegex,
+                subApplyClockmP.child(fpath),
+                deleteIfExist
+            ): fpath
+            for fpath in files_to_process
+        }
+        for future in concurrent.futures.as_completed(futures):
+            fpath = futures[future]
+            try:
+                future.result()
+            except Exception as e:
+                monitorProgress.msg(msg=f"unexpected error processing {fpath}: {e}", msgSeverity="error")
+    return True
+
+
+ 
+def main():
+    enable_pandas_display_helpers()
+    parser = argparse.ArgumentParser(
+        description="Apply clock correction to extracted data files."
+    )
+    parser.add_argument('target_folder', help="Path to the folder containing the extracted data files")
+    parser.add_argument('--shiftDateSec', help="Float value, or name of param containing clock diff to apply at each timestamp.", default="/ParamClockRef$")
+    parser.add_argument('--shiftDateRegex', help="Filter which params/files to apply date shift", default="ref_.*::")
+    parser.add_argument('--shiftDateInverted', action='store_true', default=False,  help="Inverse logic if your clock diff is based on the other clock")
+    parser.add_argument('--update', action='store_true', default=False,  help="Update existing file if any, rather than deleting it. (useful to add manually a new param in a file)")
+    parser.add_argument('--shiftDateSrcFile', help="Name of file containing the shiftDateSec if it is a param name", default="ref_data.h5")
+    
+    args = parser.parse_args()
+    
+    
+    create_logger("apply_clock_correction")
+    
+    monitorProgress=MonitorProgress(name="apply_clock_correction",progressCb=consoleRichProgressCb)
+        
+    rst = apply_clock_correction(args.target_folder, args.shiftDateSec, args.shiftDateRegex, args.shiftDateSrcFile, 
+                        shiftDateInverted=args.shiftDateInverted,monitorProgress=monitorProgress, deleteIfExist=not args.update)
+    if not rst:
+        get_logger().error(f"failed to fix clock based on {args.shiftDateSec}.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

testsuit/cli/data2db.py

@@ -0,0 +1,82 @@
+
+import argparse,os,sys,json,logging
+
+
+from testsuit.datatools.datatoolbox import SUPPORTED_DATAFILE_EXTENSIONS
+from testsuit.plugins import load_plugins
+from testsuit.datatools.DataFileMgrs.AFileMgr import enable_pandas_display_helpers
+from testsuit.misc.MonitorProgress import MonitorProgress,consoleRichProgressCb
+from testsuit.datatools.data2db import create_data2db
+from testsuit.misc.logger import create_logger,get_logger
+
+
+create_logger("data2db")
+
+## check if the given file is accessible
+def isInputReadable(f):
+    if not os.access(f,os.R_OK):
+        get_logger().error(f"{f} does not exist or is not reachable")
+        sys.exit(1)
+
+    return f
+
+# override the parsing error message using logger
+class HelpParser(argparse.ArgumentParser):
+    def error(self, message):
+        get_logger().error("Input Arguments Error : "+message)
+        sys.exit(1)
+
+## the main function
+def main():
+    # external formats (entry points, TESTSUIT_PLUGINS) show up in --extensions help
+    load_plugins()
+    enable_pandas_display_helpers()
+    parser = HelpParser(description=
+    """Extract given parameters from data files or DB.
+    
+    To extract into a db, you must provide a json file as follow:
+        { 
+            "params" : ["No2"],
+            "indices" : [],
+            "db" : {
+                "dbtype" : "influxdb",
+                "url" : "http://influxdb:8086", 
+                "org" : "_sandbox_",
+                "bucket" : "testBucket", 
+                "measurement" :"testMeas",
+                "rename_params_re" : ".*([^/]+)$",
+                "dateUnit" : "s"
+            }
+        }
+
+    Return 1 if something went wrong, 0 otherwise
+    """,
+    formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('sourceFolderOrFile', metavar="FolderOrFile", help="source folder or file  where to scan data files",type=isInputReadable)
+    parser.add_argument('confJson', help="Conf for DB parameters. See up there of details about expected contents",type=isInputReadable)
+    parser.add_argument('-t','--token',help="DB password or token")
+    parser.add_argument("--test",action='store_true', default=False, help="Dry-run: does not actually inject data")
+    parser.add_argument('-d',"--debug",action='store_true', default=False, help="Show debug messages")
+    parser.add_argument('--extensions',type=lambda s: s.split(","),metavar='ext1,ext2,...',
+                        help="File extensions to use as input (default: "+",".join(SUPPORTED_DATAFILE_EXTENSIONS)+")")
+    args = parser.parse_args()
+
+    if args.debug:
+        get_logger().setLevel(logging.DEBUG)
+
+    conf=None
+    with open(args.confJson) as f:
+        conf=json.load(f)           
+        
+    monitorProgress=MonitorProgress(progressCb=consoleRichProgressCb,name="data2db")
+    dbHandler = create_data2db(conf["db"]["dbtype"])
+    rst = dbHandler.data2db(args.sourceFolderOrFile,conf,args.extensions,token=args.token,dryRun=args.test,monitorProgress=monitorProgress,silent=True)
+
+    if rst == True:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

testsuit/cli/data2h5.py

@@ -0,0 +1,168 @@
+
+import argparse
+import sys,os,logging
+
+
+from testsuit.misc.logger import create_logger
+from testsuit.datatools.DataFileMgrs.AFileMgr import enable_pandas_display_helpers, pretty_str
+from testsuit.misc.logger import get_logger
+from testsuit.misc.MonitorProgress import MonitorProgress,consoleRichProgressCb,consoleSilentProgressCb
+
+from testsuit.datatools.datatoolbox import SUPPORTED_DATAFILE_EXTENSIONS
+from testsuit.plugins import load_plugins
+from testsuit.datatools.datatoolbox import loadDataframeFromFile,getDateParser
+from testsuit.datatools.DataframeToHdf5 import DataframeToHdf5
+
+sys.stdout.reconfigure(encoding='utf-8')
+sys.stderr.reconfigure(encoding='utf-8')
+
+create_logger("data2h5")
+
+## check if the given file is accessible
+def isInputReadable(f):
+    if not os.access(f,os.R_OK):
+        get_logger().error(f"{f} does not exist or is not reachable")
+        sys.exit(1)
+
+    return f
+
+# override the parsing error message using logger
+class HelpParser(argparse.ArgumentParser):
+    def error(self, message):
+        get_logger().error("Input Arguments Error : "+message)
+        sys.exit(1)
+
+
+def extract_to_hdf5(h5fileName,paramsDfList):
+
+    for df in paramsDfList:
+        get_logger().info("extracting params "+str(df.columns)+"")
+        DataframeToHdf5(h5fileName,os.path.basename(df.origin),df,comment="",chunk_size=True)
+
+    return True
+        
+def data2h5(sourceFolderOrFile,paramRegexes,indices=None,targetFile=None,extensions=None,
+                                        excludeParamsRegex=None, minDate=None, maxDate=None, mergeParams=True,
+                                        shiftDateSec=None,shiftDateRegex=None, shiftDateInverted=None,
+                                        listOnly=False, silent=False, 
+                                        monitorProgress=None):
+    """Extract data into H5 file.
+    See __main__ args list for parameters list and usage."""    
+    
+    minDateSec=None
+    maxDateSec=None
+    if minDate:
+        convFunc=getDateParser(minDate)
+        minDateSec=convFunc(minDate).timestamp() if convFunc else minDateSec        
+    if maxDate:
+        convFunc=getDateParser(maxDate)
+        maxDateSec=convFunc(maxDate).timestamp() if convFunc else maxDateSec    
+        
+
+    def cbPrintName(df):
+            print(df.name)
+            
+    def cbSaveDfAsH5File(df,monitorProgress=None):
+
+        if df is None:
+            get_logger().warning(f"skipping H5 writing of None param from {targetFile}")    
+            return True
+        
+        monitorProgress.set_total_items(1)
+        monitorProgress.msg("writing H5 data for "+str(df.name)+"")
+
+        h5GroupName=os.path.basename(df.origin)
+        if len(h5GroupName)==0:
+            h5GroupName=os.path.basename(os.path.dirname(df.origin))
+        comment={}
+        comment["Comments"]="Generated by data2h5"
+        if minDate: comment["Min Date"]=minDate
+        if maxDate: comment["Max Date"]=maxDate
+        try:
+            DataframeToHdf5(targetFile,h5GroupName,df,comment=comment,chunk_size=True)
+        except Exception as e:
+            monitorProgress.msg(msg=f"while writing file '{targetFile}' for {df.name} : {str(e)}",msgSeverity="error")
+            return False
+
+        monitorProgress.msg(msg=f"generated/updated file '{targetFile}' for {df.name}")
+        
+        monitorProgress.complete_n(1)
+        return True
+
+    def cbprintDf(df,name=None,origin=None):
+        if df is not None: 
+            print(pretty_str(df))
+        return True
+
+
+    cbToUse = None
+    if listOnly:
+        cbToUse=cbPrintName
+    elif targetFile is None:
+        cbToUse=cbprintDf
+    else:
+        cbToUse=cbSaveDfAsH5File
+        
+    dfListRst=loadDataframeFromFile(sourceFolderOrFile,paramRegexes,indices,extensions,
+                                    minDateSec=minDateSec,maxDateSec=maxDateSec, 
+                                    excludeParamsRegex=excludeParamsRegex,mergeParams=mergeParams,
+                                    shiftDateSec=shiftDateSec,shiftDateRegex=shiftDateRegex,
+                                    shiftDateInverted=shiftDateInverted,
+                                    dryRun=False,monitorProgress=monitorProgress,
+                                    silent=silent,callback=cbToUse)
+
+    if len(dfListRst)==0:
+        return False, dfListRst
+    
+    return True, dfListRst
+
+
+## the main function
+def main():
+    # external formats (entry points, TESTSUIT_PLUGINS) show up in --extensions help
+    load_plugins()
+    enable_pandas_display_helpers()
+    
+
+    parser = HelpParser(description=
+    """Extract given parameters into an HDF5 file. 
+        If target file is not provided, simply list matching fields found.
+        If params list is empty, list all param names found.
+
+    Return 1 if something went wrong, 0 otherwise
+    """,
+    formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('sourceFolderOrFile', metavar="FolderOrFile", help="source folder or file  where to scan data files",type=isInputReadable)
+    parser.add_argument('target', nargs='?',metavar="Target", help="Target of params contents: xxx.h5")
+    parser.add_argument('-l','--list',action='store_true', default=False, help="list all params found in given datasources" )
+    parser.add_argument('-p','--params',metavar='param1,param2,...', help="list of params (regexes) to extract into and HDF5 file. You can use python regex. Ex: 'param_02,my_.*4$'",default=".*")
+    parser.add_argument('--indices',metavar='paramIdx1,paramIdx2,...', help="use provided params as indices, 'auto' for trying to automatically identify one. Ex: 'param_02_timestamp,auto'")
+    parser.add_argument('-d',"--debug",action='store_true', default=False, help="Show debug messages")
+    parser.add_argument('-s',"--silent",action='store_true', default=False, help="Only show relevant output messages, no progress.")
+    parser.add_argument('--extensions',type=lambda s: s.split(","),metavar='ext1,ext2,...',
+                        help="File extensions to use as input (default: "+",".join(SUPPORTED_DATAFILE_EXTENSIONS)+")")
+    parser.add_argument('--minDate', help="Minimal date of data to record in file")
+    parser.add_argument('--maxDate', help="Maximal date of data to record in file")
+    
+    args = parser.parse_args()
+    
+    if args.debug:
+        get_logger().setLevel(logging.DEBUG)
+
+    get_logger().debug(args.target)
+    
+    monitorProgress=MonitorProgress(progressCb=consoleSilentProgressCb if args.silent else consoleRichProgressCb,
+                                       name="data2h5")
+        
+    rst, dfList = data2h5(args.sourceFolderOrFile,args.params,args.indices,args.target,args.extensions,
+                                            args.minDate,args.maxDate,listOnly=args.list,silent=args.silent,
+                                            monitorProgress=monitorProgress)
+
+    if rst == True:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

testsuit/cli/datapack.py

@@ -0,0 +1,46 @@
+
+import argparse
+import os
+import sys
+
+from colorama import Fore, Style
+
+from testsuit import misc
+from testsuit.datatools.datapack.datapack_tools import datapack
+
+## check if the given file is accessible
+def _isInputReadable(f):
+    if not os.access(misc.files.expandPath(f),os.R_OK):
+        raise argparse.ArgumentTypeError(f"{misc.files.expandPath(f)} does not exist or is not reachable")
+    return f
+
+# override the parsing error message using logger
+class _HelpParser(argparse.ArgumentParser):
+    def error(self, message):
+        print(Fore.RED+"ERROR: Input Arguments Error : "+message+Style.RESET_ALL)
+        sys.exit(1)
+    
+## the main function
+def main():
+    parser = _HelpParser(description=
+    """Generate datapack from given test definition file.
+
+       Dictionaries listed in test definition are applied to the test definition file itself and also to pointed datapack config.
+       Dictionaries listed in datapack config are applied to datapack config itself, but *not* to test definition.
+    
+    Return 1 if something went wrong, 0 otherwise
+    """,
+    formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('testdef_file',nargs='?',default="datapack.xml",help="the YAML file containing datapack definition",type=_isInputReadable)
+    parser.add_argument('--nocheck',action='store_true',help="Ignore warnings and consistency checks (git tags etc.) for test procedure and Setup folders.")
+    args = parser.parse_args()
+
+    success = datapack(misc.files.expandPath(args.testdef_file),args.nocheck)
+    if success:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

testsuit/cli/evalfile.py

@@ -0,0 +1,88 @@
+
+
+##  @package evalfile
+# Evaluate given file both for includes and keys replacement, based on given dictionaries.
+# This script is based on 'evalkeys.py' and 'evalincludes.py' algorithms.
+#
+# Performs several replacement and includes as long as some replacements are done, which means that key value can be a reference to another key,
+# and include can contain a key value.
+#
+# Result lines are displayed on STDOUT.
+#
+#
+# USAGE: evalfile.py [-h] targetfile dico [dico ...]
+#
+#
+import sys,os
+
+
+
+import argparse
+import os.path
+
+from testsuit.datatools.datapack.evalfiles.evalfile import evalfile,finalizeLines
+from testsuit.datatools.datapack.evalfiles import evalkeys
+from testsuit.datatools.datapack.evalfiles import evalincludes
+
+
+## check if the given file is accessible
+def _isInputReadable(f):
+    if not os.access(f,os.R_OK):
+        raise argparse.ArgumentTypeError(f"{f} does not exist or is not readable")
+    return f
+
+# override the parsing error message using logger
+class _HelpParser(argparse.ArgumentParser):
+    def error(self, message):
+        print("Input Arguments Error : "+message)
+        sys.exit(1)
+
+## the main function
+def main():
+	parser = _HelpParser(description=
+"""Evaluate given file for includes and keys replacement, based on given dictionnaries.
+This script is based on 'evalkeys' and 'evalincludes' algorithms.
+
+Performs several processing cycles as long as some replacements are done.
+This means that key value can be a reference to another key, and an include can contain a key value.
+
+When option '-o' is used, generates also an HTML view of performed key replacements and a '.<file>.keys' file containing the list of used keys and their provenance.
+Otherwise result lines are displayed on STDOUT by default.
+
+Result is displayed in <stdout>.
+
+Return:
+	3 a problem occured during processing
+	2 if circular inclusion or key reference detected, 
+	1 if unreacheable included file or key referene (and no circular include or key ref. detected)
+	0 if okay""",
+	formatter_class=argparse.RawTextHelpFormatter)
+	parser.add_argument('targetfile',help="the text file to be processed",type=_isInputReadable,metavar="targetfile")
+	parser.add_argument('dico',nargs='+', help="dictionary file(s) to be used for keys replacement, most important one first",type=_isInputReadable)	
+	parser.add_argument('--output', '-o',help="output in the given file, generating also the '.<file>.html' and '.<file>.keys' associated files")
+	parser.add_argument('--partial','-p',action='store_true',help="Partial replace : ignore unknown keys, process only defined ones")
+
+
+	args = parser.parse_args()
+
+	lines, usedkeys=evalfile(args.targetfile, args.dico, args.partial)
+	if not lines :
+		sys.exit(3)
+
+	finalizeLines(lines, usedkeys, args.partial, args.output)
+
+	if args.output is not None:
+		print("generated "+str(args.output)+" and associated 'keys' and 'html' files.")
+
+	if evalkeys.nbInfinateRecursion>0 or evalincludes.nbCircularRecursions>0:
+		sys.exit(2)
+	if evalincludes.nbNotFoundIncludes>0:
+		sys.exit(1)
+	if not args.partial and evalkeys.nbUndefined>0:
+		sys.exit(1)
+
+	sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()