tranfi 0.1.2 → 0.2.1

package/csrc/op_stats.c

@@ -1,10 +1,14 @@
 /*
  * op_stats.c — Streaming aggregates with online algorithms.
  *
- * Config: {"stats": ["count","sum","avg","min","max","var","stddev",
- *                     "median","p25","p75","skewness","kurtosis",
- *                     "distinct","hist","sample"]}
+ * Config: {"stats": ["count","missing","complete_rate","sum","avg",
+ *                     "min","max","var","stddev","median","p25",
+ *                     "p75","skewness","kurtosis","distinct",
+ *                     "hist","sample"]}
  *   or {} for default stats (count, sum, avg, min, max, var, stddev, median).
+ * The scan op uses this engine with profile defaults: count, missing,
+ * complete_rate, distinct, min, max, avg, stddev, p25, median, p75,
+ * hist, and sample.
  *
  * Algorithms ported from OnlineStats.jl (MIT):
  *   - Welford's online variance (var, stddev)
@@ -182,8 +186,9 @@ static void hll_update(hll_state *h, const char *val) {
     uint32_t x = hll_hash(val);
     uint32_t idx = x & (HLL_M - 1);
     uint32_t w = x >> HLL_P;
-    int rho = hll_clz32(w) + 1;
-    if (rho > 32 - HLL_P) rho = 32 - HLL_P;
+    int rho = w == 0 ? (32 - HLL_P + 1) : (hll_clz32(w) - HLL_P + 1);
+    if (rho < 1) rho = 1;
+    if (rho > 32 - HLL_P + 1) rho = 32 - HLL_P + 1;
     if ((uint8_t)rho > h->M[idx]) h->M[idx] = (uint8_t)rho;
 }
 
@@ -345,6 +350,7 @@ static void reservoir_update(reservoir_state *r, double y) {
 
 typedef struct {
     /* Basic stats */
+    size_t total;
     size_t count;
     double sum;
     double min;
@@ -381,7 +387,8 @@ typedef struct {
     size_t      n_cols;
     int         initialized;
     /* Which stats to output */
-    int want_count, want_sum, want_avg, want_min, want_max;
+    int want_count, want_missing, want_complete_rate;
+    int want_sum, want_avg, want_min, want_max;
     int want_var, want_stddev;
     int want_median, want_p25, want_p75;
     int want_skewness, want_kurtosis;
@@ -390,7 +397,18 @@ typedef struct {
     int want_sample;
 } stats_state;
 
-static void accum_init(col_accum *a, const stats_state *st) {
+static char *stats_strdup(const char *s) {
+    const char *src = s ? s : "";
+    size_t len = strlen(src);
+    size_t n;
+    if (tf_size_add(len, 1, &n) != TF_OK) return NULL;
+    char *out = tf_mallocarray_checked(n, sizeof(char));
+    if (!out) return NULL;
+    memcpy(out, src, len + 1);
+    return out;
+}
+
+static int accum_init(col_accum *a, const stats_state *st) {
     a->min = DBL_MAX;
     a->max = -DBL_MAX;
     p2_init(&a->p2_median, 0.5);
@@ -398,17 +416,21 @@ static void accum_init(col_accum *a, const stats_state *st) {
     p2_init(&a->p2_p75, 0.75);
 
     if (st->want_distinct) {
-        a->hll = calloc(1, sizeof(hll_state));
-        if (a->hll) hll_init(a->hll);
+        a->hll = tf_callocarray_checked(1, sizeof(hll_state));
+        if (!a->hll) return TF_ERROR;
+        hll_init(a->hll);
     }
     if (st->want_hist) {
-        a->hist = calloc(1, sizeof(hist_state));
-        if (a->hist) hist_init(a->hist);
+        a->hist = tf_callocarray_checked(1, sizeof(hist_state));
+        if (!a->hist) return TF_ERROR;
+        hist_init(a->hist);
     }
     if (st->want_sample) {
-        a->reservoir = calloc(1, sizeof(reservoir_state));
-        if (a->reservoir) reservoir_init(a->reservoir);
+        a->reservoir = tf_callocarray_checked(1, sizeof(reservoir_state));
+        if (!a->reservoir) return TF_ERROR;
+        reservoir_init(a->reservoir);
     }
+    return TF_OK;
 }
 
 static void accum_update(col_accum *a, double val, const char *str_val,
@@ -455,6 +477,23 @@ static void accum_free(col_accum *a) {
     free(a->reservoir);
 }
 
+static void stats_clear_accumulation(stats_state *st) {
+    if (!st) return;
+    if (st->accums) {
+        for (size_t i = 0; i < st->n_cols; i++)
+            accum_free(&st->accums[i]);
+        free(st->accums);
+    }
+    if (st->col_names) {
+        for (size_t i = 0; i < st->n_cols; i++) free(st->col_names[i]);
+        free(st->col_names);
+    }
+    st->accums = NULL;
+    st->col_names = NULL;
+    st->n_cols = 0;
+    st->initialized = 0;
+}
+
 /* ---- Process / Flush / Destroy ---- */
 
 static int stats_process(tf_step *self, tf_batch *in, tf_batch **out,
@@ -465,18 +504,26 @@ static int stats_process(tf_step *self, tf_batch *in, tf_batch **out,
 
     if (!st->initialized) {
         st->n_cols = in->n_cols;
-        st->accums = calloc(in->n_cols, sizeof(col_accum));
-        st->col_names = calloc(in->n_cols, sizeof(char *));
-        if (!st->accums || !st->col_names) return TF_ERROR;
+        st->accums = tf_callocarray_checked(in->n_cols ? in->n_cols : 1, sizeof(col_accum));
+        st->col_names = tf_callocarray_checked(in->n_cols ? in->n_cols : 1, sizeof(char *));
+        if (!st->accums || !st->col_names) {
+            stats_clear_accumulation(st);
+            return TF_ERROR;
+        }
         for (size_t c = 0; c < in->n_cols; c++) {
-            st->col_names[c] = strdup(in->col_names[c]);
-            accum_init(&st->accums[c], st);
+            st->col_names[c] = stats_strdup(in->col_names[c]);
+            if (!st->col_names[c] || accum_init(&st->accums[c], st) != TF_OK) {
+                stats_clear_accumulation(st);
+                return TF_ERROR;
+            }
         }
         st->initialized = 1;
     }
 
     for (size_t r = 0; r < in->n_rows; r++) {
         for (size_t c = 0; c < st->n_cols && c < in->n_cols; c++) {
+            col_accum *a = &st->accums[c];
+            a->total++;
             if (tf_batch_is_null(in, r, c)) continue;
 
             double val = 0;
@@ -524,7 +571,7 @@ static int stats_process(tf_step *self, tf_batch *in, tf_batch **out,
                 default:
                     break;
             }
-            accum_update(&st->accums[c], val, str_val, is_num);
+            accum_update(a, val, str_val, is_num);
         }
     }
 
@@ -540,8 +587,10 @@ static int stats_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
 
     /* Count output columns */
     size_t n_stat_cols = 1; /* "column" */
-    if (st->want_count)    n_stat_cols++;
-    if (st->want_sum)      n_stat_cols++;
+    if (st->want_count)         n_stat_cols++;
+    if (st->want_missing)       n_stat_cols++;
+    if (st->want_complete_rate) n_stat_cols++;
+    if (st->want_sum)           n_stat_cols++;
     if (st->want_avg)      n_stat_cols++;
     if (st->want_min)      n_stat_cols++;
     if (st->want_max)      n_stat_cols++;
@@ -558,86 +607,102 @@ static int stats_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
 
     tf_batch *ob = tf_batch_create(n_stat_cols, st->n_cols);
     if (!ob) return TF_ERROR;
+    int rc = TF_ERROR;
+#define STATS_WRITE(expr) do { if ((expr) != TF_OK) goto done; } while (0)
 
     /* Set schema */
     size_t ci = 0;
-    tf_batch_set_schema(ob, ci++, "column", TF_TYPE_STRING);
-    if (st->want_count)    tf_batch_set_schema(ob, ci++, "count", TF_TYPE_INT64);
-    if (st->want_sum)      tf_batch_set_schema(ob, ci++, "sum", TF_TYPE_FLOAT64);
-    if (st->want_avg)      tf_batch_set_schema(ob, ci++, "avg", TF_TYPE_FLOAT64);
-    if (st->want_min)      tf_batch_set_schema(ob, ci++, "min", TF_TYPE_FLOAT64);
-    if (st->want_max)      tf_batch_set_schema(ob, ci++, "max", TF_TYPE_FLOAT64);
-    if (st->want_var)      tf_batch_set_schema(ob, ci++, "var", TF_TYPE_FLOAT64);
-    if (st->want_stddev)   tf_batch_set_schema(ob, ci++, "stddev", TF_TYPE_FLOAT64);
-    if (st->want_median)   tf_batch_set_schema(ob, ci++, "median", TF_TYPE_FLOAT64);
-    if (st->want_p25)      tf_batch_set_schema(ob, ci++, "p25", TF_TYPE_FLOAT64);
-    if (st->want_p75)      tf_batch_set_schema(ob, ci++, "p75", TF_TYPE_FLOAT64);
-    if (st->want_skewness) tf_batch_set_schema(ob, ci++, "skewness", TF_TYPE_FLOAT64);
-    if (st->want_kurtosis) tf_batch_set_schema(ob, ci++, "kurtosis", TF_TYPE_FLOAT64);
-    if (st->want_distinct) tf_batch_set_schema(ob, ci++, "distinct", TF_TYPE_INT64);
-    if (st->want_hist)     tf_batch_set_schema(ob, ci++, "hist", TF_TYPE_STRING);
-    if (st->want_sample)   tf_batch_set_schema(ob, ci++, "sample", TF_TYPE_STRING);
+    STATS_WRITE(tf_batch_set_schema(ob, ci++, "column", TF_TYPE_STRING));
+    if (st->want_count)         STATS_WRITE(tf_batch_set_schema(ob, ci++, "count", TF_TYPE_INT64));
+    if (st->want_missing)       STATS_WRITE(tf_batch_set_schema(ob, ci++, "missing", TF_TYPE_INT64));
+    if (st->want_complete_rate) STATS_WRITE(tf_batch_set_schema(ob, ci++, "complete_rate", TF_TYPE_FLOAT64));
+    if (st->want_sum)           STATS_WRITE(tf_batch_set_schema(ob, ci++, "sum", TF_TYPE_FLOAT64));
+    if (st->want_avg)      STATS_WRITE(tf_batch_set_schema(ob, ci++, "avg", TF_TYPE_FLOAT64));
+    if (st->want_min)      STATS_WRITE(tf_batch_set_schema(ob, ci++, "min", TF_TYPE_FLOAT64));
+    if (st->want_max)      STATS_WRITE(tf_batch_set_schema(ob, ci++, "max", TF_TYPE_FLOAT64));
+    if (st->want_var)      STATS_WRITE(tf_batch_set_schema(ob, ci++, "var", TF_TYPE_FLOAT64));
+    if (st->want_stddev)   STATS_WRITE(tf_batch_set_schema(ob, ci++, "stddev", TF_TYPE_FLOAT64));
+    if (st->want_median)   STATS_WRITE(tf_batch_set_schema(ob, ci++, "median", TF_TYPE_FLOAT64));
+    if (st->want_p25)      STATS_WRITE(tf_batch_set_schema(ob, ci++, "p25", TF_TYPE_FLOAT64));
+    if (st->want_p75)      STATS_WRITE(tf_batch_set_schema(ob, ci++, "p75", TF_TYPE_FLOAT64));
+    if (st->want_skewness) STATS_WRITE(tf_batch_set_schema(ob, ci++, "skewness", TF_TYPE_FLOAT64));
+    if (st->want_kurtosis) STATS_WRITE(tf_batch_set_schema(ob, ci++, "kurtosis", TF_TYPE_FLOAT64));
+    if (st->want_distinct) STATS_WRITE(tf_batch_set_schema(ob, ci++, "distinct", TF_TYPE_INT64));
+    if (st->want_hist)     STATS_WRITE(tf_batch_set_schema(ob, ci++, "hist", TF_TYPE_STRING));
+    if (st->want_sample)   STATS_WRITE(tf_batch_set_schema(ob, ci++, "sample", TF_TYPE_STRING));
 
     /* One row per input column */
     for (size_t c = 0; c < st->n_cols; c++) {
-        tf_batch_ensure_capacity(ob, c + 1);
+        STATS_WRITE(tf_batch_ensure_capacity(ob, c + 1));
         ci = 0;
-        tf_batch_set_string(ob, c, ci++, st->col_names[c]);
+        STATS_WRITE(tf_batch_set_string(ob, c, ci++, st->col_names[c]));
 
         col_accum *a = &st->accums[c];
 
         if (st->want_count) {
-            tf_batch_set_int64(ob, c, ci++, (int64_t)a->count);
+            STATS_WRITE(tf_batch_set_int64(ob, c, ci++, (int64_t)a->count));
+        }
+        if (st->want_missing) {
+            size_t missing = a->total >= a->count ? a->total - a->count : 0;
+            STATS_WRITE(tf_batch_set_int64(ob, c, ci++, (int64_t)missing));
+        }
+        if (st->want_complete_rate) {
+            if (a->total > 0) {
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, (double)a->count / (double)a->total));
+            } else {
+                STATS_WRITE(tf_batch_set_null(ob, c, ci));
+            }
+            ci++;
         }
         if (st->want_sum) {
-            if (a->has_numeric) tf_batch_set_float64(ob, c, ci, a->sum);
-            else tf_batch_set_null(ob, c, ci);
+            if (a->has_numeric) STATS_WRITE(tf_batch_set_float64(ob, c, ci, a->sum));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_avg) {
             if (a->has_numeric && a->count > 0)
-                tf_batch_set_float64(ob, c, ci, a->sum / (double)a->count);
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, a->sum / (double)a->count));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_min) {
-            if (a->has_numeric && a->count > 0) tf_batch_set_float64(ob, c, ci, a->min);
-            else tf_batch_set_null(ob, c, ci);
+            if (a->has_numeric && a->count > 0) STATS_WRITE(tf_batch_set_float64(ob, c, ci, a->min));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_max) {
-            if (a->has_numeric && a->count > 0) tf_batch_set_float64(ob, c, ci, a->max);
-            else tf_batch_set_null(ob, c, ci);
+            if (a->has_numeric && a->count > 0) STATS_WRITE(tf_batch_set_float64(ob, c, ci, a->max));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_var) {
             if (a->has_numeric && a->count > 1)
-                tf_batch_set_float64(ob, c, ci, a->wf_m2 / (double)(a->count - 1));
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, a->wf_m2 / (double)(a->count - 1)));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_stddev) {
             if (a->has_numeric && a->count > 1)
-                tf_batch_set_float64(ob, c, ci, sqrt(a->wf_m2 / (double)(a->count - 1)));
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, sqrt(a->wf_m2 / (double)(a->count - 1))));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_median) {
             if (a->has_numeric && a->count > 0)
-                tf_batch_set_float64(ob, c, ci, p2_value(&a->p2_median));
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, p2_value(&a->p2_median)));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_p25) {
             if (a->has_numeric && a->count > 0)
-                tf_batch_set_float64(ob, c, ci, p2_value(&a->p2_p25));
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, p2_value(&a->p2_p25)));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_p75) {
             if (a->has_numeric && a->count > 0)
-                tf_batch_set_float64(ob, c, ci, p2_value(&a->p2_p75));
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_float64(ob, c, ci, p2_value(&a->p2_p75)));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_skewness) {
@@ -646,11 +711,11 @@ static int stats_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
                 double vr = m2 - m1 * m1;
                 if (vr > 1e-15) {
                     double sk = (m3 - 3.0*m1*vr - m1*m1*m1) / pow(vr, 1.5);
-                    tf_batch_set_float64(ob, c, ci, sk);
+                    STATS_WRITE(tf_batch_set_float64(ob, c, ci, sk));
                 } else {
-                    tf_batch_set_float64(ob, c, ci, 0.0);
+                    STATS_WRITE(tf_batch_set_float64(ob, c, ci, 0.0));
                 }
-            } else tf_batch_set_null(ob, c, ci);
+            } else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_kurtosis) {
@@ -661,17 +726,17 @@ static int stats_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
                 if (vr > 1e-15) {
                     double kt = (m4 - 4.0*m1*m3 + 6.0*m1*m1*m2
                                  - 3.0*m1*m1*m1*m1) / (vr*vr) - 3.0;
-                    tf_batch_set_float64(ob, c, ci, kt);
+                    STATS_WRITE(tf_batch_set_float64(ob, c, ci, kt));
                 } else {
-                    tf_batch_set_float64(ob, c, ci, 0.0);
+                    STATS_WRITE(tf_batch_set_float64(ob, c, ci, 0.0));
                 }
-            } else tf_batch_set_null(ob, c, ci);
+            } else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_distinct) {
             if (a->hll && a->count > 0)
-                tf_batch_set_int64(ob, c, ci, (int64_t)round(hll_estimate(a->hll)));
-            else tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_int64(ob, c, ci, (int64_t)round(hll_estimate(a->hll))));
+            else STATS_WRITE(tf_batch_set_null(ob, c, ci));
             ci++;
         }
         if (st->want_hist) {
@@ -684,9 +749,9 @@ static int stats_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
                     if (b > 0) buf[pos++] = ',';
                     pos += snprintf(buf + pos, sizeof(buf) - pos, "%zu", a->hist->counts[b]);
                 }
-                tf_batch_set_string(ob, c, ci, buf);
+                STATS_WRITE(tf_batch_set_string(ob, c, ci, buf));
             } else {
-                tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_null(ob, c, ci));
             }
             ci++;
         }
@@ -700,38 +765,83 @@ static int stats_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
                     pos += snprintf(buf + pos, sizeof(buf) - pos,
                                     "%.6g", a->reservoir->values[s]);
                 }
-                tf_batch_set_string(ob, c, ci, buf);
+                STATS_WRITE(tf_batch_set_string(ob, c, ci, buf));
             } else {
-                tf_batch_set_null(ob, c, ci);
+                STATS_WRITE(tf_batch_set_null(ob, c, ci));
             }
             ci++;
         }
-        ob->n_rows = c + 1;
+        STATS_WRITE(tf_batch_expose_row(ob, c));
     }
 
     *out = ob;
-    return TF_OK;
+    ob = NULL;
+    rc = TF_OK;
+
+done:
+    if (ob) tf_batch_free(ob);
+#undef STATS_WRITE
+    return rc;
 }
 
 static void stats_destroy(tf_step *self) {
     stats_state *st = self->state;
     if (st) {
-        if (st->accums) {
-            for (size_t i = 0; i < st->n_cols; i++)
-                accum_free(&st->accums[i]);
-            free(st->accums);
-        }
-        if (st->col_names) {
-            for (size_t i = 0; i < st->n_cols; i++) free(st->col_names[i]);
-            free(st->col_names);
-        }
+        stats_clear_accumulation(st);
         free(st);
     }
     free(self);
 }
 
-tf_step *tf_stats_create(const cJSON *args) {
-    stats_state *st = calloc(1, sizeof(stats_state));
+static void stats_enable_name(stats_state *st, const char *s) {
+    if (strcmp(s, "count") == 0)              st->want_count = 1;
+    else if (strcmp(s, "missing") == 0)       st->want_missing = 1;
+    else if (strcmp(s, "complete_rate") == 0) st->want_complete_rate = 1;
+    else if (strcmp(s, "sum") == 0)           st->want_sum = 1;
+    else if (strcmp(s, "avg") == 0)           st->want_avg = 1;
+    else if (strcmp(s, "min") == 0)           st->want_min = 1;
+    else if (strcmp(s, "max") == 0)           st->want_max = 1;
+    else if (strcmp(s, "var") == 0)           st->want_var = 1;
+    else if (strcmp(s, "stddev") == 0)        st->want_stddev = 1;
+    else if (strcmp(s, "median") == 0)        st->want_median = 1;
+    else if (strcmp(s, "p25") == 0)           st->want_p25 = 1;
+    else if (strcmp(s, "p75") == 0)           st->want_p75 = 1;
+    else if (strcmp(s, "skewness") == 0)      st->want_skewness = 1;
+    else if (strcmp(s, "kurtosis") == 0)      st->want_kurtosis = 1;
+    else if (strcmp(s, "distinct") == 0)      st->want_distinct = 1;
+    else if (strcmp(s, "hist") == 0)          st->want_hist = 1;
+    else if (strcmp(s, "sample") == 0)        st->want_sample = 1;
+}
+
+static void stats_enable_default(stats_state *st) {
+    st->want_count = 1;
+    st->want_sum = 1;
+    st->want_avg = 1;
+    st->want_min = 1;
+    st->want_max = 1;
+    st->want_var = 1;
+    st->want_stddev = 1;
+    st->want_median = 1;
+}
+
+static void scan_enable_default(stats_state *st) {
+    st->want_count = 1;
+    st->want_missing = 1;
+    st->want_complete_rate = 1;
+    st->want_distinct = 1;
+    st->want_min = 1;
+    st->want_max = 1;
+    st->want_avg = 1;
+    st->want_stddev = 1;
+    st->want_p25 = 1;
+    st->want_median = 1;
+    st->want_p75 = 1;
+    st->want_hist = 1;
+    st->want_sample = 1;
+}
+
+static tf_step *stats_create_common(const cJSON *args, int scan_defaults) {
+    stats_state *st = tf_callocarray_checked(1, sizeof(stats_state));
     if (!st) return NULL;
 
     cJSON *stats_arr = args ? cJSON_GetObjectItemCaseSensitive(args, "stats") : NULL;
@@ -739,37 +849,15 @@ tf_step *tf_stats_create(const cJSON *args) {
         int n = cJSON_GetArraySize(stats_arr);
         for (int i = 0; i < n; i++) {
             cJSON *item = cJSON_GetArrayItem(stats_arr, i);
-            if (!cJSON_IsString(item)) continue;
-            const char *s = item->valuestring;
-            if (strcmp(s, "count") == 0)         st->want_count = 1;
-            else if (strcmp(s, "sum") == 0)      st->want_sum = 1;
-            else if (strcmp(s, "avg") == 0)      st->want_avg = 1;
-            else if (strcmp(s, "min") == 0)      st->want_min = 1;
-            else if (strcmp(s, "max") == 0)      st->want_max = 1;
-            else if (strcmp(s, "var") == 0)      st->want_var = 1;
-            else if (strcmp(s, "stddev") == 0)   st->want_stddev = 1;
-            else if (strcmp(s, "median") == 0)   st->want_median = 1;
-            else if (strcmp(s, "p25") == 0)      st->want_p25 = 1;
-            else if (strcmp(s, "p75") == 0)      st->want_p75 = 1;
-            else if (strcmp(s, "skewness") == 0) st->want_skewness = 1;
-            else if (strcmp(s, "kurtosis") == 0) st->want_kurtosis = 1;
-            else if (strcmp(s, "distinct") == 0) st->want_distinct = 1;
-            else if (strcmp(s, "hist") == 0)     st->want_hist = 1;
-            else if (strcmp(s, "sample") == 0)   st->want_sample = 1;
+            if (cJSON_IsString(item)) stats_enable_name(st, item->valuestring);
         }
+    } else if (scan_defaults) {
+        scan_enable_default(st);
     } else {
-        /* Default: basic + variance + median */
-        st->want_count = 1;
-        st->want_sum = 1;
-        st->want_avg = 1;
-        st->want_min = 1;
-        st->want_max = 1;
-        st->want_var = 1;
-        st->want_stddev = 1;
-        st->want_median = 1;
+        stats_enable_default(st);
     }
 
-    tf_step *step = malloc(sizeof(tf_step));
+    tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
     if (!step) { free(st); return NULL; }
     step->process = stats_process;
     step->flush = stats_flush;
@@ -777,3 +865,11 @@ tf_step *tf_stats_create(const cJSON *args) {
     step->state = st;
     return step;
 }
+
+tf_step *tf_stats_create(const cJSON *args) {
+    return stats_create_common(args, 0);
+}
+
+tf_step *tf_scan_create(const cJSON *args) {
+    return stats_create_common(args, 1);
+}