scilens 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

scilens/components/compare_floats.py

@@ -5,7 +5,7 @@ _A=None
 import logging
 from scilens.components.compare_models import SEVERITY_ERROR,SEVERITY_WARNING,CompareGroup,CompareFloatsErr,Compare2ValuesResults
 from scilens.components.compare_errors import CompareErrors
-from scilens.config.models import CompareFloatThresholdsConfig
+from scilens.config.models import CompareFloatThresholdsConfig,CompareFloatVectorsConfig
 from scilens.components.num import vectors as CheckVectors
 def vector_get_amplitude(vector):A=vector;B=min(A);C=max(A);return{'min':B,'max':C,_D:abs(C-B)}
 class CompareFloats:
@@ -30,43 +30,43 @@ class CompareFloats:
 			if H:B=D.compare_errors.add(F,H,info={'key':C});G+=1;F.incr(_C)
 			if B:break
 		return B,G
-	def compare_vectors(A,test_vector,reference_vector,group,info_vector=_A):
-		R='ignore';M=info_vector;L='RIAE_trapezoid';H=group;F=reference_vector;B=test_vector
-		if len(B)!=len(F):raise Exception('Vectors have different lengths')
-		N=0;G=_B;E=A.thresholds.vectors.ponderation_method if A.thresholds.vectors else _A
-		if E=='RIAE':E=L
-		if E:logging.debug(f"Using ponderation method: {E} with reduction_method {A.thresholds.vectors.reduction_method}")
-		I=_A
-		if A.thresholds.vectors and E=='amplitude_moderation':S=vector_get_amplitude(B)[_D];I=S*A.thresholds.vectors.amplitude_moderation_multiplier;O=A.thresholds.vectors.reduction_method
+	def compare_vectors(G,vectors_config,test_vector,reference_vector,group,info_vector=_A):
+		S='ignore';N=info_vector;M='RIAE_trapezoid';I=group;F=reference_vector;B=vectors_config;A=test_vector
+		if len(A)!=len(F):raise Exception('Vectors have different lengths')
+		O=0;H=_B;E=B.ponderation_method if B else _A
+		if E=='RIAE':E=M
+		if E:logging.debug(f"Using ponderation method: {E} with reduction_method {B.reduction_method}")
 		J=_A
-		if A.thresholds.vectors and E in[L,'RIAE_midpoint']:
-			K=CheckVectors.relative_integral_absolute_error_trapezoid(F,B,range(len(B)))if E==L else CheckVectors.relative_integral_absolute_error_midpoint(F,B,range(len(B)))
-			if K is _A:logging.warning('RIAE calculation returned None. This may indicate an issue with the vectors.')
+		if B and E=='amplitude_moderation':T=vector_get_amplitude(A)[_D];J=T*B.amplitude_moderation_multiplier;P=B.reduction_method
+		K=_A
+		if B and E in[M,'RIAE_midpoint']:
+			L=CheckVectors.relative_integral_absolute_error_trapezoid(F,A,range(len(A)))if E==M else CheckVectors.relative_integral_absolute_error_midpoint(F,A,range(len(A)))
+			if L is _A:logging.warning('RIAE calculation returned None. This may indicate an issue with the vectors.')
 			else:
-				J=A.thresholds.vectors.reduction_method
-				if K>A.thresholds.vectors.riae_threshold:T=CompareFloatsErr(is_relative=_B,value=K);D=Compare2ValuesResults(SEVERITY_ERROR,f"RIAE ({E}) > {A.thresholds.vectors.riae_threshold}",T);G=A.compare_errors.add(H,D)
-		U=len(B)
-		for C in range(U):
-			if B[C]is _A and F[C]is _A:continue
-			P=B[C]-F[C]
-			if P==0:continue
-			else:N+=1;H.incr(_C)
-			if G:continue
-			if J==R:continue
-			if I is not _A and abs(P)<I:
-				if O==R:continue
-				elif O=='soften':
-					D=A.compare_2_values(B[C],F[C])
+				K=B.reduction_method
+				if L>B.riae_threshold:U=CompareFloatsErr(is_relative=_B,value=L);D=Compare2ValuesResults(SEVERITY_ERROR,f"RIAE ({E}) > {B.riae_threshold}",U);H=G.compare_errors.add(I,D)
+		V=len(A)
+		for C in range(V):
+			if A[C]is _A and F[C]is _A:continue
+			Q=A[C]-F[C]
+			if Q==0:continue
+			else:O+=1;I.incr(_C)
+			if H:continue
+			if K==S:continue
+			if J is not _A and abs(Q)<J:
+				if P==S:continue
+				elif P=='soften':
+					D=G.compare_2_values(A[C],F[C])
 					if D:D.severity=SEVERITY_WARNING
 			else:
-				D=A.compare_2_values(B[C],F[C])
-				if D and J:D.severity=SEVERITY_WARNING
+				D=G.compare_2_values(A[C],F[C])
+				if D and K:D.severity=SEVERITY_WARNING
 			if D:
-				Q={'index':C}
-				if M:Q['info']=M[C]
-				G=A.compare_errors.add(H,D,info=Q)
-		return G,N
-	def add_group_and_compare_vectors(A,group_name,parent_group,group_data,test_vector,reference_vector,info_vector=_A):C,B=A.compare_errors.add_group('vectors',group_name,parent=parent_group,data=group_data);return(B,)+A.compare_vectors(test_vector,reference_vector,B,info_vector=info_vector)
+				R={'index':C}
+				if N:R['info']=N[C]
+				H=G.compare_errors.add(I,D,info=R)
+		return H,O
+	def add_group_and_compare_vectors(A,group_name,parent_group,group_data,vectors_config,test_vector,reference_vector,info_vector=_A):C,B=A.compare_errors.add_group('vectors',group_name,parent=parent_group,data=group_data);return(B,)+A.compare_vectors(vectors_config,test_vector,reference_vector,B,info_vector=info_vector)
 	def compare_matrices(H,test_mat,ref_mat,group,x_vector=_A,y_vector=_A):
 		K=y_vector;J=x_vector;I=group;D=ref_mat;C=test_mat;L=0;E=_B;F=len(C);M=len(C[0])if F>0 else 0;N=len(D);P=len(D[0])if N>0 else 0
 		if F!=N or M!=P:raise Exception('Matrices have different dimensions')

scilens/config/models/__init__.py

@@ -1,6 +1,6 @@
 from.app import AppConfig
 from.compare import CompareConfig
-from.compare_float_thresholds import CompareFloatThresholdsConfig
+from.compare_float_thresholds import CompareFloatThresholdsConfig,CompareFloatVectorsConfig
 from.execute_and_compare import ExecuteAndCompareConfig
 from.execute import ExecuteConfig
 from.reader_format_txt import ReaderTxtIgnoreConfig,ReaderTxtConfig

scilens/config/models/compare.py

@@ -3,4 +3,4 @@ _A=None
 from pydantic import BaseModel,Field
 from scilens.config.models.compare_float_thresholds import CompareFloatThresholdsConfig
 class CompareSourceFoldersConfig(BaseModel,extra=_B):not_matching_source_ignore_pattern:str|_A=Field(default=_A,description="Ignore les fichiers de test et de référence qui ne correspondent pas selon l'expression régulière défini (si = `*` ignore tout).");recursive:bool=Field(default=False,description='Recherche récursive dans les répertoires de test et de référence.');additional_path_suffixes:list[str]|_A=Field(default=_A,description='Additionals paths to add after test and reference folders.');test_folder_relative_path:str=Field(default='test',description='Relative path to the working directory for the test folder.');reference_folder_relative_path:str=Field(default='reference',description='Relative path to the working directory for the reference folder.');test_filename_match_ignore:str|_A=Field(default=_A,description='Chaîne spécifique dans le nom de fichier test à ignorer pour matcher les noms. Ex : `.test` or `test_`');reference_filename_match_ignore:str|_A=Field(default=_A,description='Chaîne spécifique dans le nom de fichier référence à ignorer pour matcher les noms. Ex : `.ref` or `ref_`')
-class CompareConfig(BaseModel,extra=_B):sources:CompareSourceFoldersConfig=Field(default=CompareSourceFoldersConfig(),description='Configuration pour les sources des données de test et de référence.');float_thresholds:CompareFloatThresholdsConfig=Field(default=CompareFloatThresholdsConfig(),description='Seuils de comparaison pour les valeurs flottantes.');ignore_warnings:bool=Field(default=False,description='Si `true`, ne lève pas les erreurs de sévérité `warning`.');errors_limit:int=Field(default=1000,description="Limite d'erreurs à atteindre avant de s'arrêter (Sévérité `warning` et `error`).");metrics_compare:bool=Field(default=True,description='Si `true`, compare les métriques entre les données de test et de référence.');metrics_thresholds:CompareFloatThresholdsConfig|_A=Field(default=_A,description='Seuils de comparaison pour les métriques.');name_patterns_thresholds:dict[str,CompareFloatThresholdsConfig]|_A=Field(default=_A,description='Mapping de patterns des datasets (Unix shell-style wildcards) (ex: data*.csv) et seuils de comparaison pour les comparaison de valeurs flottantes ')
+class CompareConfig(BaseModel,extra=_B):sources:CompareSourceFoldersConfig=Field(default=CompareSourceFoldersConfig(),description='Configuration pour les sources des données de test et de référence.');float_thresholds:CompareFloatThresholdsConfig=Field(default=CompareFloatThresholdsConfig(),description='Seuils de comparaison pour les valeurs flottantes.');ignore_warnings:bool=Field(default=False,description='Si `true`, ne lève pas les erreurs de sévérité `warning`.');errors_limit:int=Field(default=1000,description="Limite d'erreurs à atteindre avant de s'arrêter (Sévérité `warning` et `error`).");metrics_compare:bool=Field(default=True,description='Si `true`, compare les métriques entre les données de test et de référence.');metrics_thresholds:CompareFloatThresholdsConfig|_A=Field(default=_A,description='Seuils de comparaison pour les métriques.');name_patterns_thresholds:dict[str,CompareFloatThresholdsConfig]|_A=Field(default=_A,description='Mapping de patterns sur les noms des datasets (Unix shell-style wildcards) (ex: data*.csv) et seuils de comparaison pour les comparaison de valeurs flottantes ')

scilens/config/models/compare_float_thresholds.py

@@ -8,4 +8,4 @@ class CompareFloatVectorsConfig(BaseModel,extra=_B):
 		A=self
 		if A.ponderation_method.startswith('RIAE')and not A.riae_threshold:raise ValueError('riae_threshold is required when ponderation_method is "RIAE"')
 		return A
-class CompareFloatThresholdsConfig(BaseModel,extra=_B):relative_vs_absolute_min:float=Field(default=1e-12,description="Si la valeur de test est inférieure à ce seuil, calcul de l'erreur absolue.");relative_error_min:float=Field(default=.001,description="Si l'erreur relative est supérieure à ce seuil, génère une erreur de sévérité `warning`.");relative_error_max:float=Field(default=.01,description="Si l'erreur relative est supérieure à ce seuil, génère une erreur de sévérité `error`.");absolute_error_min:float=Field(default=1e-07,description="Si l'erreur absolue est supérieure à ce seuil, génère une erreur de sévérité `warning`.");absolute_error_max:float=Field(default=1e-06,description="Si l'erreur absolue est supérieure à ce seuil, génère une erreur de sévérité `error`.");vectors:CompareFloatVectorsConfig|_A=Field(default=_A,description='Paramètres pour la comparaison de vecteurs de flottants (csv, nc, ...).')
+class CompareFloatThresholdsConfig(BaseModel,extra=_B):relative_vs_absolute_min:float=Field(default=1e-12,description="Si la valeur de test est inférieure à ce seuil, calcul de l'erreur absolue.");relative_error_min:float=Field(default=.001,description="Si l'erreur relative est supérieure à ce seuil, génère une erreur de sévérité `warning`.");relative_error_max:float=Field(default=.01,description="Si l'erreur relative est supérieure à ce seuil, génère une erreur de sévérité `error`.");absolute_error_min:float=Field(default=1e-07,description="Si l'erreur absolue est supérieure à ce seuil, génère une erreur de sévérité `warning`.");absolute_error_max:float=Field(default=1e-06,description="Si l'erreur absolue est supérieure à ce seuil, génère une erreur de sévérité `error`.");vectors:CompareFloatVectorsConfig|_A=Field(default=_A,description='Paramètres pour la comparaison de vecteurs de flottants (csv, nc, ...).');vectors_columns:dict[str,CompareFloatVectorsConfig|_A]|_A=Field(default=_A,description='Mapping de Colonne => Paramètres (ou `null`) pour la comparaison de vecteurs de flottants.')

scilens/readers/cols_dataset.py

@@ -69,21 +69,24 @@ class ColsDataset:
 @dataclass
 class ColsCurves:type:str;info:dict;curves:dict
 def compare(group,compare_floats,reader_test,reader_ref,cols_curve):
-	O='Errors limit reached';F=reader_ref;D=group;C=cols_curve;A=reader_test;logging.debug(f"compare cols: {D.name}")
-	if len(A.numeric_col_indexes)!=len(F.numeric_col_indexes):D.error=f"Number Float columns indexes are different: {len(A.numeric_col_indexes)} != {len(F.numeric_col_indexes)}";return
-	E=[''for A in range(A.cols_count)];I=_A;G=_A
-	if C and C.type==ReaderCurveParserNameConfig.COL_X:J=C.info[_D];I=A.data[J];G=A.names[J]
-	K=False
+	R='Errors limit reached';F=reader_ref;E=compare_floats;D=cols_curve;C=group;A=reader_test;logging.debug(f"compare cols: {C.name}")
+	if F.names!=A.names:C.error=f"Columns of datasets are different : {F.names} {A.names}";return
+	if len(A.numeric_col_indexes)!=len(F.numeric_col_indexes):C.error=f"Number Float columns indexes are different: {len(A.numeric_col_indexes)} != {len(F.numeric_col_indexes)}";return
+	G=[''for A in range(A.cols_count)];K=_A;H=_A
+	if D and D.type==ReaderCurveParserNameConfig.COL_X:L=D.info[_D];K=A.data[L];H=A.names[L]
+	M=False
 	for B in range(A.cols_count):
 		if B not in A.numeric_col_indexes:continue
-		P=A.data[B];Q=F.data[B];logging.debug(f"compare cols: {A.names[B]}");L,R,T=compare_floats.add_group_and_compare_vectors(A.names[B],D,{'info_prefix':G}if G else _A,P,Q,info_vector=I)
-		if R:K=True;E[B]=O;continue
-		if L.total_errors>0:E[B]=f"{L.total_errors} comparison errors"
-	if C:
-		for M in C.curves[_E]:
-			N=0
-			for S in M[_B]:
-				H=C.curves[_B][S]
-				if E[H[_C]]:H['comparison_error']=E[H[_C]];N+=1
-			M['comparison']={'curves_nb_with_error':N}
-	D.error=O if K else _A;D.info={'cols_has_error':E}
+		S=A.data[B];T=F.data[B];I=A.names[B];logging.debug(f"compare vector colum: {I}");N=E.thresholds.vectors
+		if E.thresholds.vectors_columns and I in E.thresholds.vectors_columns:N=E.thresholds.vectors_columns[I]
+		O,U,W=E.add_group_and_compare_vectors(A.names[B],C,{'info_prefix':H}if H else _A,N,S,T,info_vector=K)
+		if U:M=True;G[B]=R;continue
+		if O.total_errors>0:G[B]=f"{O.total_errors} comparison errors"
+	if D:
+		for P in D.curves[_E]:
+			Q=0
+			for V in P[_B]:
+				J=D.curves[_B][V]
+				if G[J[_C]]:J['comparison_error']=G[J[_C]];Q+=1
+			P['comparison']={'curves_nb_with_error':Q}
+	C.error=R if M else _A;C.info={'cols_has_error':G}

{scilens-0.5.2.dist-info → scilens-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scilens
-Version: 0.5.2
+Version: 0.5.4
 Summary: A CesGensLaB framework for data collecting and deep analysis
 Home-page: https://scilens.dev
 License: Proprietary

{scilens-0.5.2.dist-info → scilens-0.5.4.dist-info}/RECORD

@@ -10,7 +10,7 @@ scilens/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 scilens/components/analyse_folder.py,sha256=yqc-dscKaHLZJCYeXGak2v0c3F2aeX0E11AFPfya6r0,208
 scilens/components/compare_2_files.py,sha256=hpLvlsDeifeZ0SbYBLSFpelLc9nQ02LMaYjxkg6dDPw,2417
 scilens/components/compare_errors.py,sha256=vGb4DWP89HMIeBm0dZU2nt-ksppAs_37xtCHaPd0w5Y,1640
-scilens/components/compare_floats.py,sha256=g6tD8QodXdz3kcgMW1-ccZp6A4bFia3xll8mPQ0rlg8,4685
+scilens/components/compare_floats.py,sha256=w8zY8BJIxtg9zbPwNx4MqEB2jYrN9q_--r8AObA9zNA,4583
 scilens/components/compare_folders.py,sha256=s7cgo2JhZi8F7tt3lGbPhkaO2726Lz3c4LYGP2HbWtA,2773
 scilens/components/compare_models.py,sha256=EvxBZ1hPhqViCiTx7pfVIcwllkHjCjuc0mNEVa0Yqw0,941
 scilens/components/executor.py,sha256=PLeKolzPd4wPX8e6DdfOb0uHGky3DxTcKN9QtB6oe3Q,3712
@@ -20,11 +20,11 @@ scilens/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scilens/config/cli_run_options.py,sha256=Ls7yK5QDUPFbk73nbjGuPvuRbBRYw4Miag5ISpu3prg,281
 scilens/config/env_var.py,sha256=NqNBoIfngJEXaGEm7jGqre5pmkJ9eUjiWzbDrTVfi2c,292
 scilens/config/load.py,sha256=ltcv90GlsMJR2FE2ZL_jDscL7k5aGoHoMatWw61lrTw,1672
-scilens/config/models/__init__.py,sha256=lTpDxeSVAGOJtH9MFg25yh0vwwf3VdMFlhdbrhgkk6o,776
+scilens/config/models/__init__.py,sha256=JZ8fVq6A0EaPz24gKYdtaNZMD2OmhsgblRHvB2tcwiA,802
 scilens/config/models/app.py,sha256=UCC7Be6gOwvxQpM5Wl1HIf2Xb61CmslFzXvj_50eqsQ,1558
 scilens/config/models/base.py,sha256=k92CR8TA5L8dZJtg28c8albk16AK9-3umdosA7aYsxw,499
-scilens/config/models/compare.py,sha256=AvGJEsmYLRlvM766Z9IxfmBqXiHxVXFYnIJoU39xQgI,2398
-scilens/config/models/compare_float_thresholds.py,sha256=4l-AMfzpOUnBYU9hsSFAmIL-B6OyfQeVnsiAhRZSYsQ,2153
+scilens/config/models/compare.py,sha256=HmEmw9b0F_xo4D_Y3nC0t2oeKh9wSLJX8bxg23aFX58,2411
+scilens/config/models/compare_float_thresholds.py,sha256=x7UHmwnSXpq1c4IBbMB7HQdEAzrJ_NmtNRSiMxNAD5c,2335
 scilens/config/models/execute.py,sha256=yF_q8swd4-s2KyT_RwtSzOW41AY5GiEZaLhGUGJBU0Y,2340
 scilens/config/models/execute_and_compare.py,sha256=aIWF1NIgdq5USS9ZVWOKgpEyMagCvqYYWYSXWvBBgjg,597
 scilens/config/models/file_reader.py,sha256=uQrrwJZLlhgrEvEcYBAZijWtS01ob66fhtP_ipknAMo,1925
@@ -51,7 +51,7 @@ scilens/processors/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 scilens/processors/models/results.py,sha256=KoWxh13Zgi7PuPql8hkf4VjCis42ZxAuzIgJxBWVaX8,119
 scilens/processors/processor_interface.py,sha256=jzMp1529JXnMGTJijVy6b_1zmARAMNv70f2lgys7vn4,452
 scilens/readers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scilens/readers/cols_dataset.py,sha256=Gqb_uE3bUOJKBs3LpEWghhT6bBB0I0BIKT0mUQCDpf4,4084
+scilens/readers/cols_dataset.py,sha256=KC0EEJaVlMjXH7H7irgWa80tcKcGXsmZSrQVdjtNcGE,4328
 scilens/readers/exceptions.py,sha256=JzmxcjnR5sH-IOWVeCC5A1bSwxv-jCAtIJvDjzx1CTI,32
 scilens/readers/mat_dataset.py,sha256=Z9TYDWaH2aqdniLNDjlpR6VVNHMSARjh52clhdMyOn4,1496
 scilens/readers/reader_csv.py,sha256=ISEjLjMFcRziryZTYyH-Jg4vOtgPVwAH1kgfEQBHVAA,5619
@@ -122,7 +122,7 @@ scilens/utils/time_tracker.py,sha256=DdVBoMpVLXrX0qZZXyLm4g38EwDVLlRcBqcpNex1mYY
 scilens/utils/vectors.py,sha256=4N2BZSC5n3HgZqPujDGF5NdjVmSL1rOHb_qw4OoABQY,103
 scilens/utils/web.py,sha256=MAFWpIFOKz7QhqDoFh-Qwstvc76KpcxstSgHFT8FOL4,901
 scilens/utils/xml.py,sha256=HcB-ymJy8o4lsczHpXznGrbYahq_3cnnkPOdRfWdHfg,461
-scilens-0.5.2.dist-info/METADATA,sha256=LivGLk121VECrhRXLow1b07GIeE1yHgySBIj4lyYhZU,1405
-scilens-0.5.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-scilens-0.5.2.dist-info/entry_points.txt,sha256=DaKGgxUEUv34GJAoXtta6ecL37ercejep9sCSSRQK2s,48
-scilens-0.5.2.dist-info/RECORD,,
+scilens-0.5.4.dist-info/METADATA,sha256=fEncH81h03c7n54j7NjsLy8ujvqsEWFxXrZBXyeuNz4,1405
+scilens-0.5.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+scilens-0.5.4.dist-info/entry_points.txt,sha256=DaKGgxUEUv34GJAoXtta6ecL37ercejep9sCSSRQK2s,48
+scilens-0.5.4.dist-info/RECORD,,