faster-diffbloch 0.1.0__py3-none-any.whl

faster_diffbloch/native/bridge_lib.c

@@ -0,0 +1,701 @@
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Flow runtime helpers */
+typedef struct flow_temp_node { struct flow_temp_node* next; } flow_temp_node;
+static flow_temp_node* flow_temp_head = NULL;
+static int flow_temp_atexit_set = 0;
+__attribute__((unused)) static void flow_temp_free_all(void) {
+    while (flow_temp_head) {
+        flow_temp_node* n = flow_temp_head;
+        flow_temp_head = n->next;
+        free(n);
+    }
+}
+__attribute__((unused)) static void* flow_temp_alloc(size_t nbytes) {
+    flow_temp_node* node = (flow_temp_node*)malloc(sizeof(flow_temp_node) + nbytes);
+    if (!node) return NULL;
+    node->next = flow_temp_head;
+    flow_temp_head = node;
+    if (!flow_temp_atexit_set) {
+        flow_temp_atexit_set = 1;
+        atexit(flow_temp_free_all);
+    }
+    return (void*)(node + 1);
+}
+#ifndef FLOW_DIAG
+#define FLOW_DIAG(msg) fprintf(stderr, "%s", (msg))
+#endif
+#ifndef FLOW_LOG
+#define FLOW_LOG(fmt, ...) printf(fmt, __VA_ARGS__)
+#endif
+#ifndef FLOW_LOG_EMPTY
+#define FLOW_LOG_EMPTY(fmt) printf(fmt)
+#endif
+static char* flow_strcat(const char* a, const char* b) {
+    size_t la = strlen(a ? a : ""), lb = strlen(b ? b : "");
+    char* r = (char*)flow_temp_alloc(la + lb + 1);
+    if (!r) return NULL;
+    if (la) memcpy(r, a, la);
+    if (lb) memcpy(r + la, b, lb);
+    r[la + lb] = '\0';
+    return r;
+}
+
+#define __flow_in_arr(arr, val) __extension__ ({ \
+    int _found = 0; \
+    size_t _n = sizeof(arr)/sizeof((arr)[0]); \
+    for (size_t _i = 0; _i < _n; _i++) { \
+        if ((arr)[_i] == (val)) { _found = 1; break; } \
+    } _found; })
+
+#include <math.h>
+#include <complex.h>
+
+void* _ui_state = NULL;
+
+static inline float i32_to_f32(int32_t v) { return (float)v; }
+
+/* Host stub for @gpu kernels (device codegen replaces this). */
+static inline int32_t gpu_thread_id(void) { return 0; }
+
+typedef enum {
+    Adjoint_Blocked,
+    Adjoint_Dense
+} Adjoint_Tag;
+
+typedef struct {
+    Adjoint_Tag tag;
+} Adjoint;
+
+typedef struct CMat CMat;
+typedef struct Scalars Scalars;
+typedef struct c128 c128;
+typedef struct c64 c64;
+
+/* Spans: borrowed {pointer, length} views */
+typedef struct { double *data; int64_t len; } flow_span_mut_f64;
+typedef struct { float complex *data; int64_t len; } flow_span_mut_c64;
+typedef struct { double complex *data; int64_t len; } flow_span_mut_c128;
+typedef struct { const double complex *data; int64_t len; } flow_span_const_c128;
+typedef struct { const int32_t *data; int64_t len; } flow_span_const_i32;
+typedef struct { const double *data; int64_t len; } flow_span_const_f64;
+typedef struct { int32_t *data; int64_t len; } flow_span_mut_i32;
+
+struct CMat {
+    flow_span_mut_c64 data;
+    int32_t n;
+};
+
+struct Scalars {
+    float complex* one;
+    float complex* zero;
+    float complex* beta_one;
+};
+
+struct c128 {
+};
+
+struct c64 {
+};
+
+void cblas_cgemm(int32_t order, int32_t ta, int32_t tb, int32_t m, int32_t n, int32_t k, float complex* alpha, float complex* A, int32_t lda, float complex* B, int32_t ldb, float complex* beta, float complex* C, int32_t ldc);
+double theta18(void);
+double ln2(void);
+CMat cmat_new_i32(int32_t n);
+void cmat_free_CMat(CMat m);
+void cmat_copy_into_CMat_CMat(CMat dst, CMat src);
+Scalars scalars_new(void);
+void scalars_free_Scalars(Scalars s);
+void gemm_CMat_CMat_CMat_Scalars(CMat a, CMat b, CMat out, Scalars s);
+double norm1_CMat(CMat m);
+int32_t squarings_for_f64(double norm);
+float halving_factor_i32(int32_t s);
+flow_span_mut_f64 taylor_coefficients(void);
+void expm_CMat_CMat(CMat m, CMat out);
+void conjugate_transpose_CMat_CMat(CMat m, CMat out);
+void expm_backward_dense_CMat_CMat_CMat(CMat m, CMat ebar, CMat mbar);
+void pair_mul_CMat_CMat_CMat_CMat_CMat_CMat_Scalars(CMat y1, CMat l1, CMat y2, CMat l2, CMat yo, CMat lo, Scalars s);
+void pair_sq_CMat_CMat_CMat_CMat_Scalars(CMat y, CMat l, CMat yo, CMat lo, Scalars s);
+double pair_norm1_CMat_CMat(CMat y, CMat l);
+void expm_backward_blocked_CMat_CMat_CMat(CMat m, CMat ebar, CMat mbar);
+void expm_backward_Adjoint_CMat_CMat_CMat(Adjoint method, CMat m, CMat ebar, CMat mbar);
+bool shapes_ok_i32_i32(int32_t n, int32_t count);
+int32_t metal_matrix_exp_c64(float complex* a, int32_t n, int32_t count, float complex* out);
+int32_t metal_matrix_exp_backward_c64(float complex* m, float complex* ebar, int32_t n, int32_t count, int32_t dense, float complex* mbar);
+int32_t bridge_abi_version(void);
+int32_t bridge_matrix_exp_gpu_ptr_c64_i32_i32_ptr_c64(float complex* a, int32_t n, int32_t count, float complex* out);
+int32_t bridge_matrix_exp_backward_gpu_ptr_c64_ptr_c64_i32_i32_i32_ptr_c64(float complex* m, float complex* ebar, int32_t n, int32_t count, int32_t dense, float complex* mbar);
+int32_t bridge_matrix_exp_ptr_c64_i32_i32_ptr_c64(float complex* a, int32_t n, int32_t count, float complex* out);
+int32_t bridge_matrix_exp_backward_ptr_c64_ptr_c64_i32_i32_i32_ptr_c64(float complex* m, float complex* ebar, int32_t n, int32_t count, int32_t dense, float complex* mbar);
+double bridge_norm1_ptr_c64_i32(float complex* a, int32_t n);
+flow_span_mut_c128 zeroed_c128_i32(int32_t n);
+void gather_into_span_const_c128_span_const_i32_i32_span_const_i32_span_mut_c128(flow_span_const_c128 fgb, flow_span_const_i32 source, int32_t buffer_size, flow_span_const_i32 destination, flow_span_mut_c128 gathered);
+int32_t bridge_structure_matrix_ptr_c128_i32_ptr_i32_i32_ptr_i32_i32_ptr_f64_ptr_f64_i32_f64_i32_ptr_c128(double complex* fgb, int32_t n_grid, int32_t* source, int32_t buffer_size, int32_t* destination, int32_t n, double* mii, double* diagonal, int32_t n_batch, double prefactor, int32_t absorption, double complex* out);
+int32_t bridge_structure_matrix_backward_ptr_c128_ptr_i32_i32_ptr_f64_i32_f64_i32_ptr_i32_i32_i32_ptr_c128(double complex* abar, int32_t* destination, int32_t n, double* mii, int32_t n_batch, double prefactor, int32_t absorption, int32_t* source, int32_t buffer_size, int32_t n_grid, double complex* fbar);
+
+static const int32_t ROW_MAJOR = 101;
+static const int32_t NO_TRANS = 111;
+
+
+
+
+
+
+
+
+
+double theta18(void) {
+    return 3.010066362817634;
+}
+
+double ln2(void) {
+    return 0.6931471805599453;
+}
+
+CMat cmat_new_i32(int32_t n) {
+    int64_t bytes = ((((int64_t)(n)) * ((int64_t)(n))) * 8);
+    float complex* raw = (float complex*)(malloc(bytes));
+    memset(raw, 0, bytes);
+    return (CMat){ .data = ((flow_span_mut_c64){ .data = (((raw)) + (0)), .len = (int64_t)(((n * n)) - (0)) }), .n = n };
+}
+
+void cmat_free_CMat(CMat m) {
+    free(m.data.data);
+}
+
+void cmat_copy_into_CMat_CMat(CMat dst, CMat src) {
+    memcpy(dst.data.data, src.data.data, ((((int64_t)(dst.n)) * ((int64_t)(dst.n))) * 8));
+}
+
+Scalars scalars_new(void) {
+    float complex* one = (float complex*)(malloc(8));
+    float complex* zero = (float complex*)(malloc(8));
+    float complex* beta_one = (float complex*)(malloc(8));
+    one[0] = ((float)(1.0) + (float)(0.0) * I);
+    zero[0] = ((float)(0.0) + (float)(0.0) * I);
+    beta_one[0] = ((float)(1.0) + (float)(0.0) * I);
+    return (Scalars){ .one = one, .zero = zero, .beta_one = beta_one };
+}
+
+void scalars_free_Scalars(Scalars s) {
+    free(s.one);
+    free(s.zero);
+    free(s.beta_one);
+}
+
+void gemm_CMat_CMat_CMat_Scalars(CMat a, CMat b, CMat out, Scalars s) {
+    cblas_cgemm(ROW_MAJOR, NO_TRANS, NO_TRANS, a.n, a.n, a.n, s.one, a.data.data, a.n, b.data.data, b.n, s.zero, out.data.data, out.n);
+}
+
+double norm1_CMat(CMat m) {
+    double worst = 0.0;
+    int32_t __flow_step_1 = 1;
+    for (int32_t j = 0; (0 <= m.n) ? j < m.n : j > m.n; j += (0 <= m.n) ? 1 : -1) {
+        double column = 0.0;
+        int32_t __flow_step_2 = 1;
+        for (int32_t i = 0; (0 <= m.n) ? i < m.n : i > m.n; i += (0 <= m.n) ? 1 : -1) {
+            column = (column + ((double)(cabs((m.data).data[((i * m.n) + j)]))));
+        }
+        if (column > worst) {
+            worst = column;
+        }
+    }
+    return worst;
+}
+
+int32_t squarings_for_f64(double norm) {
+    double threshold = theta18();
+    if (norm <= threshold) {
+        return 0;
+    }
+    return ((int32_t)(ceil(log2((norm / threshold)))));
+}
+
+float halving_factor_i32(int32_t s) {
+    return ((float)(exp(((-((double)(s))) * ln2()))));
+}
+
+flow_span_mut_f64 taylor_coefficients(void) {
+    double* raw = (double*)(malloc((19 * 8)));
+    raw[0] = 1.0;
+    int32_t __flow_step_3 = 1;
+    for (int32_t k = 1; (1 <= 19) ? k < 19 : k > 19; k += (1 <= 19) ? 1 : -1) {
+        raw[k] = (raw[(k - 1)] / ((double)(k)));
+    }
+    return ((flow_span_mut_f64){ .data = (((raw)) + (0)), .len = (int64_t)((19) - (0)) });
+}
+
+void expm_CMat_CMat(CMat m, CMat out) {
+    int32_t n = m.n;
+    int32_t count = (n * n);
+    Scalars s = scalars_new();
+    int32_t squarings = squarings_for_f64(norm1_CMat(m));
+    if (squarings > 0) {
+        float factor = halving_factor_i32(squarings);
+        int32_t __flow_step_4 = 1;
+        for (int32_t i = 0; (0 <= count) ? i < count : i > count; i += (0 <= count) ? 1 : -1) {
+            (m.data).data[i] = ((m.data).data[i] * ((float)(factor) + (float)(0.0) * I));
+        }
+    }
+    flow_span_mut_f64 __flow_span_init_1 = taylor_coefficients();
+    flow_span_mut_f64 coefficients = __flow_span_init_1;
+    CMat m2 = cmat_new_i32(n);
+    CMat m3 = cmat_new_i32(n);
+    gemm_CMat_CMat_CMat_Scalars(m, m, m2, s);
+    gemm_CMat_CMat_CMat_Scalars(m2, m, m3, s);
+    CMat acc = cmat_new_i32(n);
+    CMat tmp = cmat_new_i32(n);
+    int32_t j = 5;
+    while (j >= 0) {
+        if (j == 5) {
+            int32_t __flow_step_5 = 1;
+            for (int32_t i = 0; (0 <= count) ? i < count : i > count; i += (0 <= count) ? 1 : -1) {
+                (acc.data).data[i] = ((float)(0.0) + (float)(0.0) * I);
+            }
+        } else {
+            gemm_CMat_CMat_CMat_Scalars(acc, m3, tmp, s);
+            cmat_copy_into_CMat_CMat(acc, tmp);
+        }
+        float c1 = ((float)((coefficients).data[((j * 3) + 1)]));
+        float c2 = ((float)((coefficients).data[((j * 3) + 2)]));
+        int32_t __flow_step_6 = 1;
+        for (int32_t i = 0; (0 <= count) ? i < count : i > count; i += (0 <= count) ? 1 : -1) {
+            (acc.data).data[i] = (((acc.data).data[i] + ((m.data).data[i] * ((float)(c1) + (float)(0.0) * I))) + ((m2.data).data[i] * ((float)(c2) + (float)(0.0) * I)));
+        }
+        float c0 = ((float)((coefficients).data[(j * 3)]));
+        int32_t __flow_step_7 = 1;
+        for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+            (acc.data).data[((i * n) + i)] = ((acc.data).data[((i * n) + i)] + ((float)(c0) + (float)(0.0) * I));
+        }
+        j = (j - 1);
+    }
+    CMat src = acc;
+    CMat dst = tmp;
+    int32_t __flow_step_8 = 1;
+    for (int32_t k = 0; (0 <= squarings) ? k < squarings : k > squarings; k += (0 <= squarings) ? 1 : -1) {
+        gemm_CMat_CMat_CMat_Scalars(src, src, dst, s);
+        CMat swap = src;
+        src = dst;
+        dst = swap;
+    }
+    cmat_copy_into_CMat_CMat(out, src);
+    cmat_free_CMat(m2);
+    cmat_free_CMat(m3);
+    cmat_free_CMat(acc);
+    cmat_free_CMat(tmp);
+    free(coefficients.data);
+    scalars_free_Scalars(s);
+}
+
+void conjugate_transpose_CMat_CMat(CMat m, CMat out) {
+    int32_t __flow_step_9 = 1;
+    for (int32_t i = 0; (0 <= m.n) ? i < m.n : i > m.n; i += (0 <= m.n) ? 1 : -1) {
+        int32_t __flow_step_10 = 1;
+        for (int32_t j = 0; (0 <= m.n) ? j < m.n : j > m.n; j += (0 <= m.n) ? 1 : -1) {
+            float complex z = (m.data).data[((j * m.n) + i)];
+            (out.data).data[((i * m.n) + j)] = ((float)(creal(z)) + (float)((-cimag(z))) * I);
+        }
+    }
+}
+
+void expm_backward_dense_CMat_CMat_CMat(CMat m, CMat ebar, CMat mbar) {
+    int32_t n = m.n;
+    int32_t wide = (2 * n);
+    CMat block = cmat_new_i32(wide);
+    CMat result = cmat_new_i32(wide);
+    int32_t __flow_step_11 = 1;
+    for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+        int32_t __flow_step_12 = 1;
+        for (int32_t j = 0; (0 <= n) ? j < n : j > n; j += (0 <= n) ? 1 : -1) {
+            float complex z = (m.data).data[((j * n) + i)];
+            float complex adjointed = ((float)(creal(z)) + (float)((-cimag(z))) * I);
+            (block.data).data[((i * wide) + j)] = adjointed;
+            (block.data).data[(((i + n) * wide) + (j + n))] = adjointed;
+            (block.data).data[((i * wide) + (j + n))] = (ebar.data).data[((i * n) + j)];
+        }
+    }
+    expm_CMat_CMat(block, result);
+    int32_t __flow_step_13 = 1;
+    for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+        int32_t __flow_step_14 = 1;
+        for (int32_t j = 0; (0 <= n) ? j < n : j > n; j += (0 <= n) ? 1 : -1) {
+            (mbar.data).data[((i * n) + j)] = (result.data).data[((i * wide) + (j + n))];
+        }
+    }
+    cmat_free_CMat(block);
+    cmat_free_CMat(result);
+}
+
+void pair_mul_CMat_CMat_CMat_CMat_CMat_CMat_Scalars(CMat y1, CMat l1, CMat y2, CMat l2, CMat yo, CMat lo, Scalars s) {
+    gemm_CMat_CMat_CMat_Scalars(y1, y2, yo, s);
+    gemm_CMat_CMat_CMat_Scalars(y1, l2, lo, s);
+    cblas_cgemm(ROW_MAJOR, NO_TRANS, NO_TRANS, l1.n, l1.n, l1.n, s.one, l1.data.data, l1.n, y2.data.data, y2.n, s.beta_one, lo.data.data, lo.n);
+}
+
+void pair_sq_CMat_CMat_CMat_CMat_Scalars(CMat y, CMat l, CMat yo, CMat lo, Scalars s) {
+    gemm_CMat_CMat_CMat_Scalars(y, y, yo, s);
+    gemm_CMat_CMat_CMat_Scalars(y, l, lo, s);
+    cblas_cgemm(ROW_MAJOR, NO_TRANS, NO_TRANS, l.n, l.n, l.n, s.one, l.data.data, l.n, y.data.data, y.n, s.beta_one, lo.data.data, lo.n);
+}
+
+double pair_norm1_CMat_CMat(CMat y, CMat l) {
+    double worst = 0.0;
+    int32_t __flow_step_15 = 1;
+    for (int32_t j = 0; (0 <= y.n) ? j < y.n : j > y.n; j += (0 <= y.n) ? 1 : -1) {
+        double column_y = 0.0;
+        double column_l = 0.0;
+        int32_t __flow_step_16 = 1;
+        for (int32_t i = 0; (0 <= y.n) ? i < y.n : i > y.n; i += (0 <= y.n) ? 1 : -1) {
+            column_y = (column_y + ((double)(cabs((y.data).data[((i * y.n) + j)]))));
+            column_l = (column_l + ((double)(cabs((l.data).data[((i * y.n) + j)]))));
+        }
+        if (column_y > worst) {
+            worst = column_y;
+        }
+        if ((column_y + column_l) > worst) {
+            worst = (column_y + column_l);
+        }
+    }
+    return worst;
+}
+
+void expm_backward_blocked_CMat_CMat_CMat(CMat m, CMat ebar, CMat mbar) {
+    int32_t n = m.n;
+    int32_t count = (n * n);
+    Scalars s = scalars_new();
+    CMat y = cmat_new_i32(n);
+    CMat l = cmat_new_i32(n);
+    conjugate_transpose_CMat_CMat(m, y);
+    cmat_copy_into_CMat_CMat(l, ebar);
+    int32_t squarings = squarings_for_f64(pair_norm1_CMat_CMat(y, l));
+    if (squarings > 0) {
+        float factor = halving_factor_i32(squarings);
+        int32_t __flow_step_17 = 1;
+        for (int32_t i = 0; (0 <= count) ? i < count : i > count; i += (0 <= count) ? 1 : -1) {
+            (y.data).data[i] = ((y.data).data[i] * ((float)(factor) + (float)(0.0) * I));
+            (l.data).data[i] = ((l.data).data[i] * ((float)(factor) + (float)(0.0) * I));
+        }
+    }
+    flow_span_mut_f64 __flow_span_init_2 = taylor_coefficients();
+    flow_span_mut_f64 coefficients = __flow_span_init_2;
+    CMat y2 = cmat_new_i32(n);
+    CMat l2 = cmat_new_i32(n);
+    CMat y3 = cmat_new_i32(n);
+    CMat l3 = cmat_new_i32(n);
+    pair_sq_CMat_CMat_CMat_CMat_Scalars(y, l, y2, l2, s);
+    pair_mul_CMat_CMat_CMat_CMat_CMat_CMat_Scalars(y2, l2, y, l, y3, l3, s);
+    CMat acc_y = cmat_new_i32(n);
+    CMat acc_l = cmat_new_i32(n);
+    CMat tmp_y = cmat_new_i32(n);
+    CMat tmp_l = cmat_new_i32(n);
+    int32_t j = 5;
+    while (j >= 0) {
+        if (j == 5) {
+            int32_t __flow_step_18 = 1;
+            for (int32_t i = 0; (0 <= count) ? i < count : i > count; i += (0 <= count) ? 1 : -1) {
+                (acc_y.data).data[i] = ((float)(0.0) + (float)(0.0) * I);
+                (acc_l.data).data[i] = ((float)(0.0) + (float)(0.0) * I);
+            }
+        } else {
+            pair_mul_CMat_CMat_CMat_CMat_CMat_CMat_Scalars(acc_y, acc_l, y3, l3, tmp_y, tmp_l, s);
+            cmat_copy_into_CMat_CMat(acc_y, tmp_y);
+            cmat_copy_into_CMat_CMat(acc_l, tmp_l);
+        }
+        float c1 = ((float)((coefficients).data[((j * 3) + 1)]));
+        float c2 = ((float)((coefficients).data[((j * 3) + 2)]));
+        int32_t __flow_step_19 = 1;
+        for (int32_t i = 0; (0 <= count) ? i < count : i > count; i += (0 <= count) ? 1 : -1) {
+            (acc_y.data).data[i] = (((acc_y.data).data[i] + ((y.data).data[i] * ((float)(c1) + (float)(0.0) * I))) + ((y2.data).data[i] * ((float)(c2) + (float)(0.0) * I)));
+            (acc_l.data).data[i] = (((acc_l.data).data[i] + ((l.data).data[i] * ((float)(c1) + (float)(0.0) * I))) + ((l2.data).data[i] * ((float)(c2) + (float)(0.0) * I)));
+        }
+        float c0 = ((float)((coefficients).data[(j * 3)]));
+        int32_t __flow_step_20 = 1;
+        for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+            (acc_y.data).data[((i * n) + i)] = ((acc_y.data).data[((i * n) + i)] + ((float)(c0) + (float)(0.0) * I));
+        }
+        j = (j - 1);
+    }
+    int32_t __flow_step_21 = 1;
+    for (int32_t k = 0; (0 <= squarings) ? k < squarings : k > squarings; k += (0 <= squarings) ? 1 : -1) {
+        pair_sq_CMat_CMat_CMat_CMat_Scalars(acc_y, acc_l, tmp_y, tmp_l, s);
+        cmat_copy_into_CMat_CMat(acc_y, tmp_y);
+        cmat_copy_into_CMat_CMat(acc_l, tmp_l);
+    }
+    cmat_copy_into_CMat_CMat(mbar, acc_l);
+    cmat_free_CMat(y);
+    cmat_free_CMat(l);
+    cmat_free_CMat(y2);
+    cmat_free_CMat(l2);
+    cmat_free_CMat(y3);
+    cmat_free_CMat(l3);
+    cmat_free_CMat(acc_y);
+    cmat_free_CMat(acc_l);
+    cmat_free_CMat(tmp_y);
+    cmat_free_CMat(tmp_l);
+    free(coefficients.data);
+    scalars_free_Scalars(s);
+}
+
+void expm_backward_Adjoint_CMat_CMat_CMat(Adjoint method, CMat m, CMat ebar, CMat mbar) {
+    { // match block
+        if ((method.tag) == Adjoint_Blocked) {
+            expm_backward_blocked_CMat_CMat_CMat(m, ebar, mbar);
+        } else { // exhaustive
+            expm_backward_dense_CMat_CMat_CMat(m, ebar, mbar);
+        }
+    } // end match
+}
+
+bool shapes_ok_i32_i32(int32_t n, int32_t count) {
+    if (n <= 0) {
+        return 0;
+    }
+    if (count <= 0) {
+        return 0;
+    }
+    return 1;
+}
+
+
+
+int32_t bridge_abi_version(void) {
+    return 3;
+}
+
+int32_t bridge_matrix_exp_gpu_ptr_c64_i32_i32_ptr_c64(float complex* a, int32_t n, int32_t count, float complex* out) {
+    if ((!(shapes_ok_i32_i32(n, count)))) {
+        return (-1);
+    }
+    return metal_matrix_exp_c64(a, n, count, out);
+}
+
+int32_t bridge_matrix_exp_backward_gpu_ptr_c64_ptr_c64_i32_i32_i32_ptr_c64(float complex* m, float complex* ebar, int32_t n, int32_t count, int32_t dense, float complex* mbar) {
+    if ((!(shapes_ok_i32_i32(n, count)))) {
+        return (-1);
+    }
+    return metal_matrix_exp_backward_c64(m, ebar, n, count, dense, mbar);
+}
+
+int32_t bridge_matrix_exp_ptr_c64_i32_i32_ptr_c64(float complex* a, int32_t n, int32_t count, float complex* out) {
+    if ((!(shapes_ok_i32_i32(n, count)))) {
+        return (-1);
+    }
+    int32_t block = (n * n);
+    int32_t total = (block * count);
+    flow_span_mut_c64 __flow_span_init_3 = ((flow_span_mut_c64){ .data = (((a)) + (0)), .len = (int64_t)((total) - (0)) });
+    flow_span_mut_c64 src = __flow_span_init_3;
+    flow_span_mut_c64 __flow_span_init_4 = ((flow_span_mut_c64){ .data = (((out)) + (0)), .len = (int64_t)((total) - (0)) });
+    flow_span_mut_c64 dst = __flow_span_init_4;
+    CMat work = cmat_new_i32(n);
+    CMat result = cmat_new_i32(n);
+    int32_t __flow_step_22 = 1;
+    for (int32_t k = 0; (0 <= count) ? k < count : k > count; k += (0 <= count) ? 1 : -1) {
+        int32_t base = (k * block);
+        int32_t __flow_step_23 = 1;
+        for (int32_t i = 0; (0 <= block) ? i < block : i > block; i += (0 <= block) ? 1 : -1) {
+            (work.data).data[i] = (src).data[(base + i)];
+        }
+        expm_CMat_CMat(work, result);
+        int32_t __flow_step_24 = 1;
+        for (int32_t i = 0; (0 <= block) ? i < block : i > block; i += (0 <= block) ? 1 : -1) {
+            (dst).data[(base + i)] = (result.data).data[i];
+        }
+    }
+    cmat_free_CMat(work);
+    cmat_free_CMat(result);
+    return 0;
+}
+
+int32_t bridge_matrix_exp_backward_ptr_c64_ptr_c64_i32_i32_i32_ptr_c64(float complex* m, float complex* ebar, int32_t n, int32_t count, int32_t dense, float complex* mbar) {
+    if ((!(shapes_ok_i32_i32(n, count)))) {
+        return (-1);
+    }
+    int32_t block = (n * n);
+    int32_t total = (block * count);
+    flow_span_mut_c64 __flow_span_init_5 = ((flow_span_mut_c64){ .data = (((m)) + (0)), .len = (int64_t)((total) - (0)) });
+    flow_span_mut_c64 m_all = __flow_span_init_5;
+    flow_span_mut_c64 __flow_span_init_6 = ((flow_span_mut_c64){ .data = (((ebar)) + (0)), .len = (int64_t)((total) - (0)) });
+    flow_span_mut_c64 ebar_all = __flow_span_init_6;
+    flow_span_mut_c64 __flow_span_init_7 = ((flow_span_mut_c64){ .data = (((mbar)) + (0)), .len = (int64_t)((total) - (0)) });
+    flow_span_mut_c64 mbar_all = __flow_span_init_7;
+    Adjoint method = (Adjoint){ .tag = Adjoint_Blocked };
+    if (dense != 0) {
+        method = (Adjoint){ .tag = Adjoint_Dense };
+    }
+    CMat m_k = cmat_new_i32(n);
+    CMat ebar_k = cmat_new_i32(n);
+    CMat mbar_k = cmat_new_i32(n);
+    int32_t __flow_step_25 = 1;
+    for (int32_t k = 0; (0 <= count) ? k < count : k > count; k += (0 <= count) ? 1 : -1) {
+        int32_t base = (k * block);
+        int32_t __flow_step_26 = 1;
+        for (int32_t i = 0; (0 <= block) ? i < block : i > block; i += (0 <= block) ? 1 : -1) {
+            (m_k.data).data[i] = (m_all).data[(base + i)];
+            (ebar_k.data).data[i] = (ebar_all).data[(base + i)];
+        }
+        expm_backward_Adjoint_CMat_CMat_CMat(method, m_k, ebar_k, mbar_k);
+        int32_t __flow_step_27 = 1;
+        for (int32_t i = 0; (0 <= block) ? i < block : i > block; i += (0 <= block) ? 1 : -1) {
+            (mbar_all).data[(base + i)] = (mbar_k.data).data[i];
+        }
+    }
+    cmat_free_CMat(m_k);
+    cmat_free_CMat(ebar_k);
+    cmat_free_CMat(mbar_k);
+    return 0;
+}
+
+double bridge_norm1_ptr_c64_i32(float complex* a, int32_t n) {
+    if ((!(shapes_ok_i32_i32(n, 1)))) {
+        return 0.0;
+    }
+    CMat m = (CMat){ .data = ((flow_span_mut_c64){ .data = (((a)) + (0)), .len = (int64_t)(((n * n)) - (0)) }), .n = n };
+    return norm1_CMat(m);
+}
+
+
+
+
+flow_span_mut_c128 zeroed_c128_i32(int32_t n) {
+    int64_t bytes = (((int64_t)(n)) * 16);
+    double complex* raw = (double complex*)(malloc(bytes));
+    memset(raw, 0, bytes);
+    return ((flow_span_mut_c128){ .data = (((raw)) + (0)), .len = (int64_t)((n) - (0)) });
+}
+
+void gather_into_span_const_c128_span_const_i32_i32_span_const_i32_span_mut_c128(flow_span_const_c128 fgb, flow_span_const_i32 source, int32_t buffer_size, flow_span_const_i32 destination, flow_span_mut_c128 gathered) {
+    flow_span_mut_c128 __flow_span_init_8 = zeroed_c128_i32(buffer_size);
+    flow_span_mut_c128 buffer = __flow_span_init_8;
+    int32_t __flow_step_28 = 1;
+    for (int32_t k = 0; (0 <= ((int32_t)(fgb.len))) ? k < ((int32_t)(fgb.len)) : k > ((int32_t)(fgb.len)); k += (0 <= ((int32_t)(fgb.len))) ? 1 : -1) {
+        int32_t slot = (source).data[k];
+        (buffer).data[slot] = ((buffer).data[slot] + (fgb).data[k]);
+    }
+    int32_t __flow_step_29 = 1;
+    for (int32_t p = 0; (0 <= ((int32_t)(gathered.len))) ? p < ((int32_t)(gathered.len)) : p > ((int32_t)(gathered.len)); p += (0 <= ((int32_t)(gathered.len))) ? 1 : -1) {
+        (gathered).data[p] = (buffer).data[(destination).data[p]];
+    }
+    free(buffer.data);
+}
+
+int32_t bridge_structure_matrix_ptr_c128_i32_ptr_i32_i32_ptr_i32_i32_ptr_f64_ptr_f64_i32_f64_i32_ptr_c128(double complex* fgb, int32_t n_grid, int32_t* source, int32_t buffer_size, int32_t* destination, int32_t n, double* mii, double* diagonal, int32_t n_batch, double prefactor, int32_t absorption, double complex* out) {
+    if ((!(shapes_ok_i32_i32(n, n_batch)))) {
+        return (-1);
+    }
+    if (n_grid <= 0) {
+        return (-2);
+    }
+    if (buffer_size <= 0) {
+        return (-3);
+    }
+    int32_t pairs = (n * n);
+    flow_span_mut_c128 __flow_span_init_9 = ((flow_span_mut_c128){ .data = (((fgb)) + (0)), .len = (int64_t)((n_grid) - (0)) });
+    flow_span_const_c128 f = ((flow_span_const_c128){ .data = (const double complex*)(__flow_span_init_9).data, .len = (__flow_span_init_9).len });
+    flow_span_mut_i32 __flow_span_init_10 = ((flow_span_mut_i32){ .data = (((source)) + (0)), .len = (int64_t)((n_grid) - (0)) });
+    flow_span_const_i32 src = ((flow_span_const_i32){ .data = (const int32_t*)(__flow_span_init_10).data, .len = (__flow_span_init_10).len });
+    flow_span_mut_i32 __flow_span_init_11 = ((flow_span_mut_i32){ .data = (((destination)) + (0)), .len = (int64_t)((pairs) - (0)) });
+    flow_span_const_i32 dst = ((flow_span_const_i32){ .data = (const int32_t*)(__flow_span_init_11).data, .len = (__flow_span_init_11).len });
+    flow_span_mut_f64 __flow_span_init_12 = ((flow_span_mut_f64){ .data = (((mii)) + (0)), .len = (int64_t)(((n_batch * n)) - (0)) });
+    flow_span_const_f64 m = ((flow_span_const_f64){ .data = (const double*)(__flow_span_init_12).data, .len = (__flow_span_init_12).len });
+    flow_span_mut_f64 __flow_span_init_13 = ((flow_span_mut_f64){ .data = (((diagonal)) + (0)), .len = (int64_t)(((n_batch * n)) - (0)) });
+    flow_span_const_f64 d = ((flow_span_const_f64){ .data = (const double*)(__flow_span_init_13).data, .len = (__flow_span_init_13).len });
+    flow_span_mut_c128 __flow_span_init_14 = ((flow_span_mut_c128){ .data = (((out)) + (0)), .len = (int64_t)(((n_batch * pairs)) - (0)) });
+    flow_span_mut_c128 a = __flow_span_init_14;
+    flow_span_mut_c128 __flow_span_init_15 = zeroed_c128_i32(pairs);
+    flow_span_mut_c128 gathered = __flow_span_init_15;
+    gather_into_span_const_c128_span_const_i32_i32_span_const_i32_span_mut_c128(f, src, buffer_size, dst, gathered);
+    double u0_prime = 0.0;
+    if (absorption != 0) {
+        double total = 0.0;
+        int32_t __flow_step_30 = 1;
+        for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+            total = (total + cimag((gathered).data[((i * n) + i)]));
+        }
+        u0_prime = (prefactor * (total / ((double)(n))));
+    }
+    int32_t __flow_step_31 = 1;
+    for (int32_t b = 0; (0 <= n_batch) ? b < n_batch : b > n_batch; b += (0 <= n_batch) ? 1 : -1) {
+        int32_t base_m = (b * n);
+        int32_t base_a = (b * pairs);
+        int32_t __flow_step_32 = 1;
+        for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+            int32_t __flow_step_33 = 1;
+            for (int32_t j = 0; (0 <= n) ? j < n : j > n; j += (0 <= n) ? 1 : -1) {
+                double scale = ((prefactor * (m).data[(base_m + j)]) * (m).data[(base_m + i)]);
+                double complex z = (gathered).data[((i * n) + j)];
+                (a).data[((base_a + (i * n)) + j)] = ((double)((creal(z) * scale)) + (double)((cimag(z) * scale)) * I);
+            }
+            (a).data[((base_a + (i * n)) + i)] = ((double)((d).data[(base_m + i)]) + (double)((u0_prime * (m).data[(base_m + i)])) * I);
+        }
+    }
+    free(gathered.data);
+    return 0;
+}
+
+int32_t bridge_structure_matrix_backward_ptr_c128_ptr_i32_i32_ptr_f64_i32_f64_i32_ptr_i32_i32_i32_ptr_c128(double complex* abar, int32_t* destination, int32_t n, double* mii, int32_t n_batch, double prefactor, int32_t absorption, int32_t* source, int32_t buffer_size, int32_t n_grid, double complex* fbar) {
+    if ((!(shapes_ok_i32_i32(n, n_batch)))) {
+        return (-1);
+    }
+    if (n_grid <= 0) {
+        return (-2);
+    }
+    if (buffer_size <= 0) {
+        return (-3);
+    }
+    int32_t pairs = (n * n);
+    flow_span_mut_c128 __flow_span_init_16 = ((flow_span_mut_c128){ .data = (((abar)) + (0)), .len = (int64_t)(((n_batch * pairs)) - (0)) });
+    flow_span_const_c128 ab = ((flow_span_const_c128){ .data = (const double complex*)(__flow_span_init_16).data, .len = (__flow_span_init_16).len });
+    flow_span_mut_i32 __flow_span_init_17 = ((flow_span_mut_i32){ .data = (((destination)) + (0)), .len = (int64_t)((pairs) - (0)) });
+    flow_span_const_i32 dst = ((flow_span_const_i32){ .data = (const int32_t*)(__flow_span_init_17).data, .len = (__flow_span_init_17).len });
+    flow_span_mut_f64 __flow_span_init_18 = ((flow_span_mut_f64){ .data = (((mii)) + (0)), .len = (int64_t)(((n_batch * n)) - (0)) });
+    flow_span_const_f64 m = ((flow_span_const_f64){ .data = (const double*)(__flow_span_init_18).data, .len = (__flow_span_init_18).len });
+    flow_span_mut_i32 __flow_span_init_19 = ((flow_span_mut_i32){ .data = (((source)) + (0)), .len = (int64_t)((n_grid) - (0)) });
+    flow_span_const_i32 src = ((flow_span_const_i32){ .data = (const int32_t*)(__flow_span_init_19).data, .len = (__flow_span_init_19).len });
+    flow_span_mut_c128 __flow_span_init_20 = ((flow_span_mut_c128){ .data = (((fbar)) + (0)), .len = (int64_t)((n_grid) - (0)) });
+    flow_span_mut_c128 fb = __flow_span_init_20;
+    flow_span_mut_c128 __flow_span_init_21 = zeroed_c128_i32(buffer_size);
+    flow_span_mut_c128 buffer = __flow_span_init_21;
+    int32_t __flow_step_34 = 1;
+    for (int32_t b = 0; (0 <= n_batch) ? b < n_batch : b > n_batch; b += (0 <= n_batch) ? 1 : -1) {
+        int32_t base_m = (b * n);
+        int32_t base_a = (b * pairs);
+        int32_t __flow_step_35 = 1;
+        for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+            int32_t __flow_step_36 = 1;
+            for (int32_t j = 0; (0 <= n) ? j < n : j > n; j += (0 <= n) ? 1 : -1) {
+                if (i != j) {
+                    double scale = ((prefactor * (m).data[(base_m + j)]) * (m).data[(base_m + i)]);
+                    int32_t slot = (dst).data[((i * n) + j)];
+                    double complex z = (ab).data[((base_a + (i * n)) + j)];
+                    (buffer).data[slot] = ((buffer).data[slot] + ((double)((creal(z) * scale)) + (double)((cimag(z) * scale)) * I));
+                }
+            }
+        }
+    }
+    if (absorption != 0) {
+        double total = 0.0;
+        int32_t __flow_step_37 = 1;
+        for (int32_t b = 0; (0 <= n_batch) ? b < n_batch : b > n_batch; b += (0 <= n_batch) ? 1 : -1) {
+            int32_t __flow_step_38 = 1;
+            for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+                total = (total + (cimag((ab).data[(((b * pairs) + (i * n)) + i)]) * (m).data[((b * n) + i)]));
+            }
+        }
+        double share = ((prefactor * total) / ((double)(n)));
+        int32_t __flow_step_39 = 1;
+        for (int32_t i = 0; (0 <= n) ? i < n : i > n; i += (0 <= n) ? 1 : -1) {
+            int32_t slot = (dst).data[((i * n) + i)];
+            (buffer).data[slot] = ((buffer).data[slot] + ((double)(0.0) + (double)(share) * I));
+        }
+    }
+    int32_t __flow_step_40 = 1;
+    for (int32_t k = 0; (0 <= n_grid) ? k < n_grid : k > n_grid; k += (0 <= n_grid) ? 1 : -1) {
+        (fb).data[k] = (buffer).data[(src).data[k]];
+    }
+    free(buffer.data);
+    return 0;
+}