tranfi 0.1.2 → 0.2.0

package/csrc/op_anomaly.c

@@ -11,6 +11,18 @@
 #include <string.h>
 #include <stdio.h>
 #include <math.h>
+#include <stdint.h>
+
+typedef enum {
+    ANOMALY_MISSING_ERROR,
+    ANOMALY_MISSING_NULL,
+    ANOMALY_MISSING_IGNORE,
+} anomaly_missing_policy;
+
+typedef enum {
+    ANOMALY_TYPE_FAIL,
+    ANOMALY_TYPE_NULL,
+} anomaly_type_policy;
 
 typedef struct {
     char   *column;
@@ -20,12 +32,79 @@ typedef struct {
     size_t  count;
     double  mean;
     double  m2;
+    anomaly_missing_policy missing;
+    anomaly_type_policy on_type_error;
 } anomaly_state;
 
-static double get_numeric(const tf_batch *b, size_t r, int ci) {
+static int anomaly_is_numeric_type(tf_type type) {
+    return type == TF_TYPE_INT64 || type == TF_TYPE_FLOAT64;
+}
+
+static double anomaly_get_numeric(const tf_batch *b, size_t r, int ci) {
     if (b->col_types[ci] == TF_TYPE_INT64) return (double)tf_batch_get_int64(b, r, ci);
-    if (b->col_types[ci] == TF_TYPE_FLOAT64) return tf_batch_get_float64(b, r, ci);
-    return 0;
+    return tf_batch_get_float64(b, r, ci);
+}
+
+static void anomaly_set_col_error(const char *column, const char *suffix) {
+    char msg[512];
+    snprintf(msg, sizeof(msg), "anomaly: column '%s' %s", column ? column : "", suffix);
+    tf_set_last_error(msg);
+}
+
+static int anomaly_parse_missing_policy(const cJSON *args, anomaly_missing_policy *out) {
+    *out = ANOMALY_MISSING_ERROR;
+    const cJSON *j = cJSON_GetObjectItemCaseSensitive(args, "missing");
+    if (!j) return TF_OK;
+    if (!cJSON_IsString(j)) {
+        tf_set_last_error("anomaly: missing must be error, null, or ignore");
+        return TF_ERROR;
+    }
+    if (strcmp(j->valuestring, "error") == 0) *out = ANOMALY_MISSING_ERROR;
+    else if (strcmp(j->valuestring, "null") == 0) *out = ANOMALY_MISSING_NULL;
+    else if (strcmp(j->valuestring, "ignore") == 0) *out = ANOMALY_MISSING_IGNORE;
+    else {
+        tf_set_last_error("anomaly: missing must be error, null, or ignore");
+        return TF_ERROR;
+    }
+    return TF_OK;
+}
+
+static int anomaly_parse_type_policy(const cJSON *args, anomaly_type_policy *out) {
+    *out = ANOMALY_TYPE_FAIL;
+    const cJSON *j = cJSON_GetObjectItemCaseSensitive(args, "on_type_error");
+    if (!j) return TF_OK;
+    if (!cJSON_IsString(j)) {
+        tf_set_last_error("anomaly: on_type_error must be fail or null");
+        return TF_ERROR;
+    }
+    if (strcmp(j->valuestring, "fail") == 0) *out = ANOMALY_TYPE_FAIL;
+    else if (strcmp(j->valuestring, "null") == 0) *out = ANOMALY_TYPE_NULL;
+    else {
+        tf_set_last_error("anomaly: on_type_error must be fail or null");
+        return TF_ERROR;
+    }
+    return TF_OK;
+}
+
+static int anomaly_passthrough(tf_batch *in, tf_batch **out) {
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+    *out = ob;
+    return TF_OK;
 }
 
 static int anomaly_process(tf_step *self, tf_batch *in, tf_batch **out,
@@ -34,45 +113,86 @@ static int anomaly_process(tf_step *self, tf_batch *in, tf_batch **out,
     anomaly_state *st = self->state;
     *out = NULL;
 
+    int ci = tf_batch_col_index(in, st->column);
+    int force_null = 0;
+    if (ci < 0) {
+        if (st->missing == ANOMALY_MISSING_ERROR) {
+            anomaly_set_col_error(st->column, "not found");
+            return TF_ERROR;
+        }
+        if (st->missing == ANOMALY_MISSING_IGNORE) return anomaly_passthrough(in, out);
+        force_null = 1;
+    } else if (!anomaly_is_numeric_type(in->col_types[ci])) {
+        if (st->on_type_error == ANOMALY_TYPE_FAIL) {
+            anomaly_set_col_error(st->column, "must be numeric");
+            return TF_ERROR;
+        }
+        force_null = 1;
+    }
+
+    const char *extra_names[1] = {st->result};
+    tf_type extra_types[1] = {TF_TYPE_INT64};
     tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
     if (!ob) return TF_ERROR;
-    for (size_t c = 0; c < in->n_cols; c++)
-        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
-    tf_batch_set_schema(ob, in->n_cols, st->result, TF_TYPE_INT64);
-
-    int ci = tf_batch_col_index(in, st->column);
+    if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
 
     for (size_t r = 0; r < in->n_rows; r++) {
-        tf_batch_copy_row(ob, r, in, r);
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+
+        if (force_null) {
+            if (tf_batch_set_null(ob, r, in->n_cols) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
+            if (tf_batch_expose_row(ob, r) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
+            continue;
+        }
 
-        if (ci < 0 || tf_batch_is_null(in, r, ci)) {
-            tf_batch_set_int64(ob, r, in->n_cols, 0);
-            ob->n_rows = r + 1;
+        if (tf_batch_is_null(in, r, ci)) {
+            if (tf_batch_set_int64(ob, r, in->n_cols, 0) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
+            if (tf_batch_expose_row(ob, r) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
             continue;
         }
 
-        double val = get_numeric(in, r, ci);
+        double val = anomaly_get_numeric(in, r, ci);
 
-        /* Update Welford's */
-        st->count++;
-        double delta = val - st->mean;
-        st->mean += delta / (double)st->count;
-        double delta2 = val - st->mean;
-        st->m2 += delta * delta2;
+        if (st->count == SIZE_MAX) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        size_t next_count = st->count + 1;
+        double next_mean = st->mean;
+        double next_m2 = st->m2;
+        double delta = val - next_mean;
+        next_mean += delta / (double)next_count;
+        double delta2 = val - next_mean;
+        next_m2 += delta * delta2;
 
         /* Compute z-score (need at least 2 data points for variance) */
         int is_anomaly = 0;
-        if (st->count >= 2) {
-            double var = st->m2 / (double)(st->count - 1);
+        if (next_count >= 2) {
+            double var = next_m2 / (double)(next_count - 1);
             double std = sqrt(var);
             if (std > 0) {
-                double z = fabs((val - st->mean) / std);
+                double z = fabs((val - next_mean) / std);
                 is_anomaly = z > st->threshold ? 1 : 0;
             }
         }
 
-        tf_batch_set_int64(ob, r, in->n_cols, is_anomaly);
-        ob->n_rows = r + 1;
+        if (tf_batch_set_int64(ob, r, in->n_cols, is_anomaly) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        st->count = next_count;
+        st->mean = next_mean;
+        st->m2 = next_m2;
     }
 
     *out = ob;
@@ -92,25 +212,40 @@ static void anomaly_destroy(tf_step *self) {
 tf_step *tf_anomaly_create(const cJSON *args) {
     if (!args) return NULL;
     cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
-    if (!cJSON_IsString(col_j)) return NULL;
+    if (!cJSON_IsString(col_j) || !col_j->valuestring[0]) {
+        tf_set_last_error("anomaly: column is required");
+        return NULL;
+    }
 
-    anomaly_state *st = calloc(1, sizeof(anomaly_state));
+    anomaly_state *st = tf_callocarray_checked(1, sizeof(anomaly_state));
     if (!st) return NULL;
-    st->column = strdup(col_j->valuestring);
+    st->column = tf_strdup_checked(col_j->valuestring);
+    if (!st->column) { free(st); return NULL; }
 
     cJSON *thresh_j = cJSON_GetObjectItemCaseSensitive(args, "threshold");
     st->threshold = cJSON_IsNumber(thresh_j) ? thresh_j->valuedouble : 3.0;
+    if (!isfinite(st->threshold) || st->threshold < 0.0) {
+        tf_set_last_error("anomaly: threshold must be a non-negative finite number");
+        free(st->column);
+        free(st);
+        return NULL;
+    }
+    if (anomaly_parse_missing_policy(args, &st->missing) != TF_OK ||
+        anomaly_parse_type_policy(args, &st->on_type_error) != TF_OK) {
+        free(st->column);
+        free(st);
+        return NULL;
+    }
 
     cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
     if (cJSON_IsString(res_j)) {
-        st->result = strdup(res_j->valuestring);
+        st->result = tf_strdup_checked(res_j->valuestring);
     } else {
-        char buf[256];
-        snprintf(buf, sizeof(buf), "%s_anomaly", st->column);
-        st->result = strdup(buf);
+        st->result = tf_string_append_suffix_checked(st->column, "_anomaly");
     }
+    if (!st->result) { free(st->column); free(st); return NULL; }
 
-    tf_step *step = malloc(sizeof(tf_step));
+    tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
     if (!step) { free(st->column); free(st->result); free(st); return NULL; }
     step->process = anomaly_process;
     step->flush = anomaly_flush;