medpython 1.1.1__tar.gz → 1.1.2__tar.gz

{medpython-1.1.1/src/medpython.egg-info → medpython-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: medpython
-Version: 1.1.1
+Version: 1.1.2
 Summary: Medial EarlySign Python
 Author: Alon Lanyado
 License-Expression: MIT

{medpython-1.1.1 → medpython-1.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "medpython"
-version = "1.1.1"
+version = "1.1.2"
 description = "Medial EarlySign Python"
 readme = "README.md"
 license = "MIT"

{medpython-1.1.1/src/med/cpp_code/Internal/MedPyExport/generate_binding → medpython-1.1.2}/setup.py

@@ -48,6 +48,33 @@ def get_version_from_pyproject() -> str:
     return version
 
 
+def get_safe_parallel_jobs() -> int:
+    """Calculates a safe number of build jobs based on CPU and RAM."""
+    GB_PER_CPU = 2
+    cores = os.cpu_count() or 1
+    # If not Linux, just use all cores (assume Mac/Windows have decent RAM)
+    if not sys.platform.startswith("linux"):
+        return cores
+    try:
+        # Read total memory safely
+        with open("/proc/meminfo", "r") as f:
+            for line in f:
+                if line.startswith("MemTotal:"):
+                    # Extract KB and convert to GB
+                    mem_kb = int(line.split()[1])
+                    mem_gb = mem_kb / (1024 * 1024)
+
+                    # Rule: Allow 1 job per ~2GB of RAM.
+                    # Max out at the number of available CPU cores.
+                    safe_jobs = max(1, int(mem_gb) // GB_PER_CPU)
+                    return min(cores, safe_jobs)
+
+    except Exception:
+        pass  # Fallback if /proc/meminfo is missing or unreadable
+
+    return cores
+
+
 class CustomSdist(_sdist):
     """Custom sdist command to copy external C++ source into the package tree."""
 
@@ -143,7 +170,9 @@ class CMakeBuild(build_ext):
 
         # Parallel build
         if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
-            build_args += [f"-j{os.cpu_count()}"]
+            safe_jobs = get_safe_parallel_jobs()
+
+            build_args += [f"-j{safe_jobs}"]
 
         # Version Info
         if "GIT_HEAD_VERSION" not in os.environ:
@@ -162,7 +191,7 @@ class CMakeBuild(build_ext):
         subprocess.check_call(
             ["cmake", "--build", "."] + build_args, cwd=self.build_temp
         )
-        print("########## Done CMAKE BUILD ############3")
+        print("########## Done CMAKE BUILD #############")
 
         # 3. MANUAL COPY (If CMake doesn't put the .py file exactly where we want)
         # CMake will put _medpython.so in destination_dir (because of CMAKE_LIBRARY_OUTPUT_DIRECTORY)

{medpython-1.1.1 → medpython-1.1.2}/src/AlgoMarker/AlgoMarker.py

@@ -349,22 +349,22 @@ class AlgoMarker:
                     f"Error in add_data_simple - all values must have the same size, but got {len(elem.values)} != {values_size}"
                 )
 
-            # Convert to ctypes arrays
-            c_times = (ctypes.c_long * len(flat_times))(*flat_times)
-            c_values = (ctypes.c_float * len(flat_values))(*flat_values)
-            res = self.__lib.AM_API_AddData(
-                self.__obj,
-                patient_id,
-                ctypes.create_string_buffer(signal_name.encode("ascii")),
-                len(flat_times),
-                c_times,
-                len(flat_values),
-                c_values,
-            )
-            if res != 0:
-                msg = f"Error in add_data_simple - error code {res} for patient_id {patient_id}, signal_name {signal_name}, data: {elem}"
-                print(f"Error in add_data_simple - error code {res} for a patient more details in response message")
-                messages.append(msg)
+        # Convert to ctypes arrays
+        c_times = (ctypes.c_long * len(flat_times))(*flat_times)
+        c_values = (ctypes.c_float * len(flat_values))(*flat_values)
+        res = self.__lib.AM_API_AddData(
+            self.__obj,
+            patient_id,
+            ctypes.create_string_buffer(signal_name.encode("ascii")),
+            len(flat_times),
+            c_times,
+            len(flat_values),
+            c_values,
+        )
+        if res != 0:
+            msg = f"Error in add_data_simple - error code {res} for patient_id {patient_id}, signal_name {signal_name}, data: {elem}"
+            print(f"Error in add_data_simple - error code {res} for a patient more details in response message")
+            messages.append(msg)
         # No return value, errors are handled by the library
         return messages
 

{medpython-1.1.1 → medpython-1.1.2}/src/med/cpp_code/External/LightGBM_2.2.3/LightGBM-2.2.3/src/io/json11.cpp

@@ -25,6 +25,7 @@
 #include <cstdlib>
 #include <cstdio>
 #include <limits>
+#include <cstdint>
 
 namespace json11 {
 

medpython-1.1.2/src/med/cpp_code/External/README

@@ -0,0 +1,8 @@
+## Those Are used:
+1. Eigen - https://gitlab.com/libeigen/eigen/tree/master - 3.2.92 (src/Core/util/Macros.h EIGEN_WORLD_VERSION Macros.h) was updated to 3.4.1
+2. nlohmann/json (https://github.com/nlohmann/json) - JSON for Modern C++ - 3.11.2
+3. XGBoost - v2.0.3 (2023.12.19)
+4. LightGBM - 2.2.3 (2019.02.05)
+
+## Additional External
+1. Boost Headers - (Tested with 1.67-1.89) - you will bring  yours/will be downloaded in CMakeLists.txt

{medpython-1.1.1 → medpython-1.1.2}/src/med/cpp_code/Internal/AlgoMarker/AlgoMarker/AlgoMarker.h

@@ -362,9 +362,9 @@ private:
 	int AddJsonData(int patient_id, json &j_data, vector<string> &messages, map<pair<int, int>, pair<int, vector<char>>> *data = NULL);
 	int rec_AddDataByType(int DataType, const char *data, vector<string> &messages);
 	void clear_patients_data(const vector<int> &pids);
-	int AddDataStr_data(int patient_id, const char *signalName, int TimeStamps_len, long long* TimeStamps, int Values_len, char** Values, 
+	vector<pair<int, string>> AddDataStr_data(int patient_id, const char *signalName, int TimeStamps_len, long long* TimeStamps, int Values_len, char** Values, 
 	map<pair<int, int>, pair<int, vector<char>>> *data);
-	int AddData_data(int patient_id, const char *signalName, int TimeStamps_len, long long* TimeStamps, int Values_len, float* Values, 
+	vector<pair<int, string>> AddData_data(int patient_id, const char *signalName, int TimeStamps_len, long long* TimeStamps, int Values_len, float* Values, 
 	map<pair<int, int>, pair<int, vector<char>>> *data);
 public:
 	MedialInfraAlgoMarker() { set_type((int)AM_TYPE_MEDIAL_INFRA); add_supported_stype("Raw"); }

{medpython-1.1.1 → medpython-1.1.2}/src/med/cpp_code/Internal/AlgoMarker/AlgoMarker/MedialInfra_AM.cpp

@@ -2,7 +2,6 @@
 
 #include <Logger/Logger/Logger.h>
 #include <MedTime/MedTime/MedTime.h>
-#include <MedUtils/MedUtils/MedUtils.h>
 #include <json/json.hpp>
 #include "AlgoMarkerErr.h"
 
@@ -187,53 +186,116 @@ int MedialInfraAlgoMarker::ClearData()
 //-----------------------------------------------------------------------------------
 // AddData() - adding data for a signal with values and timestamps
 //-----------------------------------------------------------------------------------
-int MedialInfraAlgoMarker::AddData_data(int patient_id, const char *signalName, int TimeStamps_len, long long *TimeStamps, int Values_len, float *Values,
-										map<pair<int, int>, pair<int, vector<char>>> *data)
+vector<pair<int, string>> MedialInfraAlgoMarker::AddData_data(int patient_id, const char *signalName, int TimeStamps_len, long long *TimeStamps, int Values_len, float *Values,
+															  map<pair<int, int>, pair<int, vector<char>>> *data)
 {
 	// At the moment MedialInfraAlgoMarker only loads timestamps given as ints.
 	// This may change in the future as needed.
+	vector<pair<int, string>> ret;
 	int *i_times = NULL;
 	vector<int> times_int;
+	vector<float> values_filter;
+	vector<int> skip_e_index;
+	int time_ch_count, val_ch_count;
+	int *is_categ;
+	string sig = string(signalName);
 
 	int tu = get_time_unit();
+
 	if (TimeStamps_len > 0)
 	{
 		times_int.resize(TimeStamps_len);
+		int write_index = 0;
 
 		// currently assuming we only work with dates ... will have to change this when we'll move to other units
-		for (int i = 0; i < TimeStamps_len; i++)
+		int i = 0;
+		while (i < TimeStamps_len)
 		{
-			times_int[i] = AMPoint::auto_time_convert(TimeStamps[i], tu);
-			if (times_int[i] < 0)
+			times_int[write_index] = AMPoint::auto_time_convert(TimeStamps[i], tu);
+			if (times_int[write_index] < 0)
 			{
-				MERR("Error in AddData :: patient %d, signals %s, timestamp %lld is ilegal\n",
-					 patient_id, signalName, TimeStamps[i]);
-				return AM_ERROR_ADD_DATA_FAILED;
+				char buff[5000];
+				snprintf(buff, sizeof(buff), "Error in AddData :: patient %d, signals %s, timestamp %lld is ilegal",
+						 patient_id, signalName, TimeStamps[i]);
+				MERR("%s\n", buff);
+				ret.push_back(pair<int, string>(AM_DATA_BAD_FORMAT_NON_FATAL, string(buff)));
+				if (skip_e_index.empty())
+				{
+					get_sig_structure(sig, time_ch_count, val_ch_count, is_categ);
+					values_filter.reserve(Values_len);
+				}
+
+				// Need to skip this element index and create a complete new signal:
+				int current_element_index = int(i / time_ch_count);
+				skip_e_index.push_back(current_element_index);
+				int time_pos = i % time_ch_count;
+				write_index = current_element_index * time_ch_count;
+
+				// skip current element
+				i += time_ch_count - time_pos;
+				continue;
 			}
+			++write_index;
+			++i;
 			// MLOG("time convert %ld to %d\n", TimeStamps[i], times_int[i]);
 		}
 		i_times = &times_int[0];
 	}
+	if (!skip_e_index.empty())
+	{
+		// Rewrite values_filter from Values based on skipping skip_e_index elements. Nujm of channels is: val_ch_count
+		int i = 0, j = 0;
+		while (i < Values_len)
+		{
+			int current_element_index = int(i / val_ch_count);
+			if (j < (int)skip_e_index.size() && current_element_index == skip_e_index[j])
+			{
+				++j;
+				i += val_ch_count;
+				continue;
+			}
+			else
+			{
+				values_filter.push_back(Values[i]);
+			}
+			++i;
+		}
+		Values = &values_filter[0];
 
-	if (ma.data_load_pid_sig(patient_id, signalName, i_times, TimeStamps_len, Values, Values_len, data) < 0)
-		return AM_ERROR_ADD_DATA_FAILED;
+		TimeStamps_len -= (int)skip_e_index.size() * time_ch_count;
+		Values_len = (int)values_filter.size();
+	}
 
-	return AM_OK_RC;
+	if (TimeStamps_len > 0 || Values_len > 0)
+	{
+		if (ma.data_load_pid_sig(patient_id, signalName, i_times, TimeStamps_len, Values, Values_len, data) < 0)
+		{
+			ret.push_back(pair<int, string>(AM_GENERAL_FATAL, "General error in data_load_pid_sig"));
+			return ret;
+		}
+	}
+	if (ret.empty())
+		ret.push_back(pair<int, string>(AM_OK_RC, ""));
+	return ret;
 }
 
 int MedialInfraAlgoMarker::AddData(int patient_id, const char *signalName, int TimeStamps_len, long long *TimeStamps, int Values_len, float *Values)
 {
 	MedRepository &rep = ma.get_rep();
 	map<pair<int, int>, pair<int, vector<char>>> *data = &rep.in_mem_rep.data;
-	return AddData_data(patient_id, signalName, TimeStamps_len, TimeStamps, Values_len, Values, data);
+	vector<pair<int, string>> ret = AddData_data(patient_id, signalName, TimeStamps_len, TimeStamps, Values_len, Values, data);
+	if (ret.empty())
+		return AM_OK_RC;
+	return ret[0].first;
 }
 
 //-----------------------------------------------------------------------------------
 // AddDatStr() - adding data for a signal with values and timestamps
 //-----------------------------------------------------------------------------------
-int MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalName, int TimeStamps_len, long long *TimeStamps, int Values_len, char **Values,
-										   map<pair<int, int>, pair<int, vector<char>>> *data)
+vector<pair<int, string>> MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalName, int TimeStamps_len, long long *TimeStamps, int Values_len, char **Values,
+																 map<pair<int, int>, pair<int, vector<char>>> *data)
 {
+	vector<pair<int, string>> ret;
 	vector<float> converted_Values;
 	vector<long long> final_tm;
 	final_tm.reserve(TimeStamps_len);
@@ -254,10 +316,11 @@ int MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalNam
 			n_elem = (int)(Values_len / rep.sigs.Sid2Info[sid].n_val_channels);
 		else
 			n_elem = (int)(TimeStamps_len / rep.sigs.Sid2Info[sid].n_time_channels);
+
+		long long last_time = 0;
 		for (int i = 0; i < n_elem; i++)
 		{
 			bool skip_val = false;
-			int val_start = Values_i;
 			for (int j = 0; j < rep.sigs.Sid2Info[sid].n_val_channels; j++)
 			{
 				if (rep.sigs.is_categorical_channel(sid, j))
@@ -266,11 +329,28 @@ int MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalNam
 					{
 						// MWARN("Found undefined code for signal \"%s\" and value \"%s\"\n",
 						//	sig.c_str(), Values[Values_i]);
+						// TODO: Use same simple error collection inside "ret" variable. No need to collecte inside "get_unknown_codes"
 						(*ma.get_unknown_codes(patient_id))[sig].insert(Values[Values_i]);
 						skip_val = true;
 					}
-					++Values_i;
 				}
+				else
+				{
+					// Check if valid value:
+					try
+					{
+						stof(Values[Values_i]);
+					}
+					catch (...)
+					{
+						char buff[5000];
+						snprintf(buff, sizeof(buff), "Error in AddDataStr_data :: patient %d, signal %s, index %d, channel %d, value %s is non-numeric",
+								 patient_id, signalName, i, j, Values[Values_i]);
+						ret.push_back(pair<int, string>(AM_DATA_BAD_FORMAT_NON_FATAL, string(buff)));
+						skip_val = true;
+					}
+				}
+				++Values_i;
 			}
 			if (skip_val)
 			{
@@ -281,6 +361,21 @@ int MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalNam
 			else
 			{
 				// All done
+				if (rep.sigs.Sid2Info[sid].n_time_channels > 0)
+				{
+					if (TimeStamps[Time_i] < last_time)
+					{
+						char buff[5000];
+						snprintf(buff, sizeof(buff), "Error in AddDataStr_data :: patient %d, signal %s, data is not sorted by time. All signal data is skipped",
+								 patient_id, signalName);
+						// Mark that all signal is not loaded
+						ret.push_back(pair<int, string>(AM_DATA_GENERAL_ERROR, string(buff)));
+						TimeStamps_len = 0;
+						Values_len = 0;
+						break;
+					}
+					last_time = TimeStamps[Time_i];
+				}
 				for (int j = 0; j < rep.sigs.Sid2Info[sid].n_time_channels; j++)
 				{
 					final_tm.push_back(TimeStamps[Time_i]);
@@ -289,10 +384,25 @@ int MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalNam
 				for (int j = 0; j < rep.sigs.Sid2Info[sid].n_val_channels; j++)
 				{
 					float val;
+					int current_index = i * rep.sigs.Sid2Info[sid].n_val_channels + j;
 					if (!rep.sigs.is_categorical_channel(sid, j))
-						val = stof(Values[Values_i++]);
+					{
+						try
+						{
+							val = stof(Values[current_index]);
+						}
+						catch (...)
+						{
+							char buff[5000];
+							snprintf(buff, sizeof(buff), "Error in AddDataStr_data :: patient %d, signal %s, value %s is non-numeric",
+									 patient_id, signalName, Values[current_index]);
+							ret.push_back(pair<int, string>(AM_DATA_BAD_FORMAT_FATAL, string(buff)));
+							// Not Suppose to happen - already tested before that this is valid float.
+							throw;
+						}
+					}
 					else
-						val = category_map.at(Values[val_start + j]);
+						val = category_map.at(Values[current_index]);
 					converted_Values.push_back(val);
 				}
 			}
@@ -301,19 +411,31 @@ int MedialInfraAlgoMarker::AddDataStr_data(int patient_id, const char *signalNam
 	catch (...)
 	{
 		MERR("Catched Error MedialInfraAlgoMarker::AddDataStr!!\n");
-		return AM_FAIL_RC;
+		ret.push_back(pair<int, string>(AM_GENERAL_FATAL, "Catched Internal Error MedialInfraAlgoMarker::AddDataStr - skipped signal"));
+		return ret;
 	}
 
 	if (TimeStamps_len > 0 || Values_len > 0)
-		return AddData_data(patient_id, signalName, TimeStamps_len, final_tm.data(), Values_len, converted_Values.data(), data);
-	return AM_OK_RC;
+	{
+		vector<pair<int, string>> ret_i = AddData_data(patient_id, signalName, TimeStamps_len, final_tm.data(), Values_len, converted_Values.data(), data);
+		// Can append AM_OK_RC, even when we have previous errors:
+		if (!ret_i.empty() && (ret.empty() || (ret_i.size() > 1 || ret_i[0].first != AM_OK_RC)))
+			ret.insert(ret.end(), ret_i.begin(), ret_i.end());
+		return ret;
+	}
+	if (ret.empty())
+		ret.push_back(pair<int, string>(AM_OK_RC, ""));
+	return ret;
 }
 
 int MedialInfraAlgoMarker::AddDataStr(int patient_id, const char *signalName, int TimeStamps_len, long long *TimeStamps, int Values_len, char **Values)
 {
 	MedRepository &rep = ma.get_rep();
 	map<pair<int, int>, pair<int, vector<char>>> *data = &rep.in_mem_rep.data;
-	return AddDataStr_data(patient_id, signalName, TimeStamps_len, TimeStamps, Values_len, Values, data);
+	vector<pair<int, string>> ret = AddDataStr_data(patient_id, signalName, TimeStamps_len, TimeStamps, Values_len, Values, data);
+	if (ret.empty())
+		return AM_OK_RC;
+	return ret[0].first;
 }
 
 //-----------------------------------------------------------------------------------
@@ -1847,11 +1969,14 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 					}
 					if (good_sig)
 					{
+						// good - controls if to trigger warning in the end, any error/global in either signal
+						// good_sig - if to completly skip signal
+						// good_record - just to skip current record in signal
 						for (auto &d : s["data"])
 						{
 							int nt = 0;
 							bool good_record = true;
-							if (d.find("timestamp") != d.end() && !d["timestamp"].is_array())
+							if (d.find("timestamp") == d.end() || !d["timestamp"].is_array())
 							{
 								char buf[5000];
 								if (patient_id != 1)
@@ -1863,10 +1988,12 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 								messages.push_back(string(buf));
 								get_current_time(current_time);
 								MLOG("%s::%s\n", current_time.c_str(), buf);
+								good_record = false;
+								// good_sig = false;
 								good = false;
-								break;
+								continue;
 							}
-							if (d.find("value") != d.end() && !d["value"].is_array())
+							if (d.find("value") == d.end() || !d["value"].is_array())
 							{
 								char buf[5000];
 								if (patient_id != 1)
@@ -1878,8 +2005,10 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 								messages.push_back(string(buf));
 								get_current_time(current_time);
 								MLOG("%s::%s\n", current_time.c_str(), buf);
+								good_record = false;
+								// good_sig = false;
 								good = false;
-								break;
+								continue;
 							}
 							for (auto &t : d["timestamp"])
 							{
@@ -1904,7 +2033,7 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 										get_current_time(current_time);
 										MLOG("%s::%s\n", current_time.c_str(), buf);
 										good = false;
-										good_sig = false;
+										// good_sig = false;
 										good_record = false;
 										break;
 									}
@@ -1922,7 +2051,7 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 									get_current_time(current_time);
 									MLOG("%s::%s\n", current_time.c_str(), buf);
 									good = false;
-									good_sig = false;
+									// good_sig = false;
 									good_record = false;
 									break;
 								}
@@ -1931,7 +2060,11 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 								++nt;
 							}
 							if (!good_record)
-								break;
+							{
+								if (nt > 0)
+									times.resize(times.size() - nt);
+								continue;
+							}
 							// Check size of timestamps:
 							if (nt != n_time_channels)
 							{
@@ -1946,13 +2079,18 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 								get_current_time(current_time);
 								MLOG("%s::%s\n", current_time.c_str(), buf);
 								good = false;
-								good_sig = false;
+								// good_sig = false;
 								good_record = false;
 								// return AM_FAIL_RC;
 							}
 							if (!good_record)
-								break;
+							{
+								if (nt > 0)
+									times.resize(times.size() - nt);
+								continue;
+							}
 							int nv = 0;
+							int total_nv = 0;
 							for (auto &v : d["value"])
 							{
 								string sv;
@@ -1975,7 +2113,7 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 										get_current_time(current_time);
 										MLOG("%s::%s\n", current_time.c_str(), buf);
 										good = false;
-										good_sig = false;
+										// good_sig = false;
 										good_record = false;
 										break;
 									}
@@ -2005,7 +2143,7 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 											get_current_time(current_time);
 											MLOG("%s::%s\n", current_time.c_str(), buf);
 											good = false;
-											good_sig = false;
+											// good_sig = false;
 											good_record = false;
 											break;
 										}
@@ -2023,7 +2161,7 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 										get_current_time(current_time);
 										MLOG("%s::%s\n", current_time.c_str(), buf);
 										good = false;
-										good_sig = false;
+										// good_sig = false;
 										good_record = false;
 										break;
 									}
@@ -2041,12 +2179,23 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 								sinds.push_back(curr_s);
 								curr_s += slen + 1;
 								++nv;
+								total_nv += slen + 1;
 								// char *sp = &sdata[sinds.back()];
 								// MLOG("val %d %d %s : %s len: %d curr_s %d s_data_size %d %d\n", sinds.size(), sinds.back(), sp, sv.c_str(), slen, curr_s, s_data_size, sdata.size());
 								// MLOG("%s ", v.get<string>().c_str());
 							}
 							if (!good_record)
-								break;
+							{
+								if (nt > 0)
+									times.resize(times.size() - nt);
+								if (nv > 0)
+								{
+									// Update: sinds, curr_s to remove what we have added
+									sinds.resize(sinds.size() - nv);
+									curr_s -= total_nv;
+								}
+								continue;
+							}
 							// Check size of value:
 							if (nv != n_val_channels)
 							{
@@ -2061,14 +2210,24 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 								get_current_time(current_time);
 								MLOG("%s::%s\n", current_time.c_str(), buf);
 								good = false;
-								good_sig = false;
+								// good_sig = false;
 								good_record = false;
 								// return AM_FAIL_RC;
 							}
 							// MLOG("\n");
 							if (!good_record)
-								break;
-							n_data++;
+							{
+								if (nt > 0)
+									times.resize(times.size() - nt);
+								if (nv > 0)
+								{
+									// Update: sinds, curr_s to remove what we have added
+									sinds.resize(sinds.size() - nv);
+									curr_s -= total_nv;
+								}
+								continue;
+							}
+							++n_data;
 						}
 					}
 				}
@@ -2086,19 +2245,20 @@ int MedialInfraAlgoMarker::AddJsonData(int patient_id, json &j_data, vector<stri
 
 				if (good_sig)
 				{
-					if (AddDataStr_data(patient_id, sig.c_str(), n_times, p_times, n_vals, str_values, data) != AM_OK_RC)
+					vector<pair<int, string>> ret = AddDataStr_data(patient_id, sig.c_str(), n_times, p_times, n_vals, str_values, data);
+					if ((!ret.empty() && ret[0].first != AM_OK_RC) || ret.size() > 1)
 					{
 						char buf[5000];
-						if (patient_id != 1)
-							snprintf(buf, sizeof(buf), "(%d)General error in signal: %s for patient %d",
-									 AM_DATA_GENERAL_ERROR, sig.c_str(), patient_id);
-						else
-							snprintf(buf, sizeof(buf), "(%d)General error in signal: %s",
-									 AM_DATA_GENERAL_ERROR, sig.c_str());
-						messages.push_back(string(buf));
-						get_current_time(current_time);
-						MLOG("%s::%s\n", current_time.c_str(), buf);
-						good_sig = false;
+						for (const pair<int, string> &msg : ret)
+						{
+							snprintf(buf, sizeof(buf), "(%d)%s",
+									 msg.first, msg.second.c_str());
+
+							get_current_time(current_time);
+							messages.push_back(string(buf));
+							MLOG("%s::%s\n", current_time.c_str(), buf);
+						}
+						// good_sig = false;
 						good = false;
 						// return AM_FAIL_RC;
 					}