mumpy-toolkit 0.2.0__py3-none-any.whl

mumpy/_parallel.py

@@ -0,0 +1,142 @@
+"""mumpy parallel engine: chunked multithreaded execution for memory-bound ops.
+
+Why faster than plain numpy for many ops:
+- numpy element-wise ufuncs are single-threaded. We split large arrays
+  into chunks and run the ufunc in a ThreadPool (GIL is released in C loops).
+- reductions are done per-chunk in parallel then combined.
+- fft uses scipy.fft with all workers when available.
+"""
+from __future__ import annotations
+
+import os
+import numpy as np
+from concurrent.futures import ThreadPoolExecutor
+
+_ncpu = os.cpu_count() or 4
+MAX_WORKERS = max(1, min(32, _ncpu))
+
+# Only parallelize above this many elements (avoid thread overhead).
+PARALLEL_THRESHOLD = 50_000
+
+
+def get_workers(n_elements: int | None = None) -> int:
+    if n_elements is not None and n_elements < PARALLEL_THRESHOLD:
+        return 1
+    return MAX_WORKERS
+
+
+def _split_first_axis(n: int, workers: int) -> list[tuple[int, int]]:
+    chunk = (n + workers - 1) // workers
+    return [(i, min(i + chunk, n)) for i in range(0, n, chunk)]
+
+
+def parallel_ewise(func, *arrays, out=None):
+    """Apply element-wise C func over chunks in parallel.
+
+    func: callable (*views, out_view) executed on numpy views.
+    arrays: broadcastable arrays (broadcasted first, then chunked on flat view).
+    """
+    # strip subclasses: work on plain ndarrays to avoid __array_ufunc__ recursion
+    arrays = [np.asanyarray(a).view(np.ndarray) for a in arrays]
+    if out is not None:
+        out = np.asanyarray(out).view(np.ndarray)
+    # fast path: all same shape & contiguous -> zero-copy ravel views
+    try:
+        shape0 = arrays[0].shape
+        if all(a.shape == shape0 for a in arrays):
+            size = arrays[0].size
+            workers = get_workers(size)
+            if workers <= 1 or size < PARALLEL_THRESHOLD:
+                if out is None:
+                    res = np.empty(shape0, dtype=np.result_type(*arrays))
+                    func(*arrays, out=res)
+                    return res
+                func(*arrays, out=out)
+                return out
+            flats = [a.reshape(-1) if a.flags["C_CONTIGUOUS"] or a.flags["F_CONTIGUOUS"]
+                     else np.ascontiguousarray(a).reshape(-1) for a in arrays]
+            if out is None:
+                res_flat = np.empty(size, dtype=np.result_type(*arrays))
+            else:
+                res_flat = np.asanyarray(out).reshape(-1)
+            ranges = _split_first_axis(size, workers)
+
+            def _job(r):
+                s, e = r
+                views = [f[s:e] for f in flats]
+                func(*views, out=res_flat[s:e])
+
+            with ThreadPoolExecutor(max_workers=workers) as ex:
+                list(ex.map(_job, ranges))
+            if out is None:
+                return res_flat.reshape(shape0)
+            return out
+    except Exception:
+        pass
+    try:
+        bcast = np.broadcast_arrays(*arrays)
+    except ValueError:
+        # fall back to plain call (e.g. matmul-like)
+        return func(*arrays, out=out) if out is not None else func(*arrays)
+    shape = bcast[0].shape
+    size = bcast[0].size
+    workers = get_workers(size)
+    if workers <= 1 or size < PARALLEL_THRESHOLD:
+        outs = [np.ascontiguousarray(b) for b in bcast]
+        if out is None:
+            res = np.empty(shape, dtype=np.result_type(*arrays))
+            func(*outs, out=res.reshape(-1) if res.ndim else res)
+            return res
+        func(*outs, out=np.asanyarray(out).reshape(-1))
+        return out
+
+    # flatten broadcasted (copies only if needed for non-contiguous)
+    flats = [np.ascontiguousarray(b).reshape(-1) for b in bcast]
+    if out is None:
+        res_flat = np.empty(size, dtype=np.result_type(*arrays))
+    else:
+        res_flat = np.asanyarray(out).reshape(-1)
+    ranges = _split_first_axis(size, workers)
+
+    def _job(r):
+        s, e = r
+        views = [f[s:e] for f in flats]
+        func(*views, out=res_flat[s:e])
+
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        list(ex.map(_job, ranges))
+    if out is None:
+        return res_flat.reshape(shape)
+    return out
+
+
+def parallel_reduce(op, a, axis=None, dtype=None, keepdims=False, op_name="sum"):
+    """Parallel reduction over the flattened array when axis is None,
+    else chunked along the given axis."""
+    a = np.asanyarray(a)
+    workers = get_workers(a.size)
+    _no_dtype = op_name in ("min", "max", "any", "all")
+    if workers <= 1 or a.size < PARALLEL_THRESHOLD or axis is not None:
+        if _no_dtype:
+            return op(a, axis=axis, keepdims=keepdims)
+        return op(a, axis=axis, dtype=dtype, keepdims=keepdims)
+    # flat parallel reduce: partials then combine
+    flats = np.ascontiguousarray(a).reshape(-1)
+    ranges = _split_first_axis(flats.size, workers)
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        if _no_dtype:
+            partials = list(ex.map(lambda r: op(flats[r[0]:r[1]]), ranges))
+        else:
+            partials = list(ex.map(lambda r: op(flats[r[0]:r[1]], dtype=dtype), ranges))
+    partials = np.asanyarray(partials)
+    if op_name in ("min", "max", "any", "all"):
+        res = op(partials)
+    elif op_name in ("sum", "prod"):
+        res = op(partials, dtype=dtype)
+    elif op_name == "mean":
+        res = np.mean(partials, dtype=dtype) if False else np.sum(partials, dtype=dtype) / flats.size
+    else:
+        res = op(partials)
+    if keepdims:
+        res = np.asanyarray(res).reshape((1,) * a.ndim)
+    return res

mumpy/db.py

@@ -0,0 +1,289 @@
+"""mumpy.db: unified database layer for analysts & developers.
+
+Zero-dependency core: sqlite3 (stdlib) — full CRUD, bulk insert from numpy,
+read to numpy / dict / pandas (optional).
+Optional backends (auto-detected): sqlalchemy (postgres/mysql/sqlite),
+duckdb (analytical OLAP, parquet direct).
+
+Example:
+    import mumpy as cp
+    db = cp.db.connect("data.db")
+    db.create_table("users", {"id": "INTEGER PRIMARY KEY", "name": "TEXT", "age": "INTEGER"})
+    db.insert_many("users", [{"name": "salim", "age": 30}])
+    arr = db.read_numpy("SELECT age FROM users")
+    df  = db.read_pandas("SELECT * FROM users")
+"""
+from __future__ import annotations
+
+import sqlite3
+import numpy as np
+
+__all__ = ["Database", "connect", "has_sqlalchemy", "has_duckdb", "has_pandas"]
+
+
+def has_sqlalchemy() -> bool:
+    try:
+        import sqlalchemy  # noqa: F401
+        return True
+    except Exception:
+        return False
+
+
+def has_duckdb() -> bool:
+    try:
+        import duckdb  # noqa: F401
+        return True
+    except Exception:
+        return False
+
+
+def has_pandas() -> bool:
+    try:
+        import pandas  # noqa: F401
+        return True
+    except Exception:
+        return False
+
+
+class Database:
+    """Thin, fast wrapper over sqlite3 (+ optional sqlalchemy/duckdb engines)."""
+
+    def __init__(self, path=":memory:", engine="sqlite", echo=False, **kw):
+        self.path = path
+        self.engine = engine
+        self.echo = echo
+        self._sa_engine = None
+        self._duck = None
+        if engine == "sqlite":
+            self.conn = sqlite3.connect(path, **{k: v for k, v in kw.items()
+                                                 if k in ("timeout", "detect_types",
+                                                          "isolation_level", "check_same_thread")})
+            self.conn.row_factory = sqlite3.Row
+        elif engine == "sqlalchemy":
+            if not has_sqlalchemy():
+                raise ImportError("pip install sqlalchemy")
+            from sqlalchemy import create_engine
+            url = kw.pop("url", f"sqlite:///{path}" if path != ":memory:" else "sqlite://")
+            self._sa_engine = create_engine(url, echo=echo, **kw)
+            self.conn = self._sa_engine.connect()
+        elif engine == "duckdb":
+            if not has_duckdb():
+                raise ImportError("pip install duckdb")
+            import duckdb
+            self._duck = duckdb.connect(path if path != ":memory:" else ":memory:", **kw)
+            self.conn = None
+        else:
+            raise ValueError(f"unknown engine: {engine}")
+
+    # ---------- core execution ----------
+    def execute(self, sql, params=None):
+        if self.echo:
+            print(sql, params or "")
+        if self.engine == "duckdb":
+            return self._duck.execute(sql, params or [])
+        if self.engine == "sqlalchemy":
+            from sqlalchemy import text
+            if params:
+                return self.conn.execute(text(sql), params)
+            return self.conn.execute(text(sql))
+        cur = self.conn.cursor()
+        cur.execute(sql, params or [])
+        self.conn.commit()
+        return cur
+
+    def executemany(self, sql, seq):
+        if self.engine == "duckdb":
+            self._duck.executemany(sql, seq)
+            return None
+        if self.engine == "sqlalchemy":
+            from sqlalchemy import text
+            self.conn.execute(text(sql), seq)
+            try:
+                self.conn.commit()
+            except Exception:
+                pass
+            return None
+        cur = self.conn.cursor()
+        cur.executemany(sql, seq)
+        self.conn.commit()
+        return cur
+
+    # ---------- DDL helpers ----------
+    def create_table(self, name, schema: dict, if_not_exists=True):
+        cols = ", ".join(f'"{k}" {v}' for k, v in schema.items())
+        ine = "IF NOT EXISTS " if if_not_exists else ""
+        self.execute(f'CREATE TABLE {ine}"{name}" ({cols})')
+
+    def drop_table(self, name, if_exists=True):
+        ie = "IF EXISTS " if if_exists else ""
+        self.execute(f'DROP TABLE {ie}"{name}"')
+
+    def tables(self):
+        if self.engine == "duckdb":
+            rows = self._duck.execute(
+                "SELECT table_name FROM information_schema.tables").fetchall()
+            return [r[0] for r in rows]
+        cur = self.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+        try:
+            return [r[0] for r in cur.fetchall()]
+        except Exception:
+            return [r[0] for r in cur]
+
+    # ---------- writes ----------
+    def insert_many(self, table, rows):
+        """rows: list[dict] | list[tuple] | 2D ndarray (+ columns kw)."""
+        if isinstance(rows, np.ndarray):
+            raise ValueError("use write_numpy(table, arr, columns=...) for ndarrays")
+        if not rows:
+            return 0
+        if isinstance(rows[0], dict):
+            cols = list(rows[0].keys())
+            placeholders = ", ".join(["?"] * len(cols))
+            sql = f'INSERT INTO "{table}" ({", ".join(chr(34)+c+chr(34) for c in cols)}) VALUES ({placeholders})'
+            seq = [tuple(r[c] for c in cols) for r in rows]
+        else:
+            n = len(rows[0])
+            placeholders = ", ".join(["?"] * n)
+            sql = f'INSERT INTO "{table}" VALUES ({placeholders})'
+            seq = [tuple(r) for r in rows]
+        self.executemany(sql, seq)
+        return len(seq)
+
+    def write_numpy(self, table, arr, columns=None, dtypes=None):
+        """Bulk-write 2D numpy array as a table (creates table if needed)."""
+        arr = np.asanyarray(arr)
+        if arr.ndim == 1:
+            arr = arr.reshape(-1, 1)
+        ncols = arr.shape[1]
+        if columns is None:
+            columns = [f"c{i}" for i in range(ncols)]
+        if dtypes is None:
+            # infer SQL types from numpy dtype
+            def _t(dt):
+                dt = np.dtype(dt)
+                if np.issubdtype(dt, np.integer):
+                    return "INTEGER"
+                if np.issubdtype(dt, np.floating):
+                    return "REAL"
+                return "TEXT"
+            if arr.ndim == 2:
+                dtypes = [_t(arr.dtype)] * ncols
+            else:
+                dtypes = ["REAL"] * ncols
+        schema = {c: t for c, t in zip(columns, dtypes)}
+        self.create_table(table, schema)
+        placeholders = ", ".join(["?"] * ncols)
+        sql = f'INSERT INTO "{table}" ({", ".join(chr(34)+c+chr(34) for c in columns)}) VALUES ({placeholders})'
+        self.executemany(sql, [tuple(r) for r in arr.tolist()])
+        return arr.shape[0]
+
+    def from_csv(self, table, csv_path, delimiter=",", header=True):
+        from .io import load_csv
+        data, names = load_csv(csv_path, delimiter=delimiter, header=header,
+                               dtype=str)
+        if names is None:
+            names = [f"c{i}" for i in range(data.shape[1])] if data.size else []
+        # try numeric conversion per column
+        cols = {}
+        for j, name in enumerate(names):
+            col = data[:, j] if data.size else np.array([])
+            try:
+                cols[name] = col.astype(float)
+            except Exception:
+                cols[name] = col
+        import numpy as _np
+        n = data.shape[0] if data.size else 0
+        schema = {}
+        for name in names:
+            v = cols[name]
+            try:
+                v.astype(float)
+                schema[name] = "REAL"
+            except Exception:
+                schema[name] = "TEXT"
+        self.create_table(table, schema)
+        if n:
+            placeholders = ", ".join(["?"] * len(names))
+            sql = f'INSERT INTO "{table}" VALUES ({placeholders})'
+            self.executemany(sql, [tuple(data[i, j] for j in range(len(names)))
+                                   for i in range(n)])
+        return n
+
+    # ---------- reads ----------
+    def query(self, sql, params=None):
+        """Return list[dict]."""
+        if self.engine == "duckdb":
+            cur = self._duck.execute(sql, params or [])
+            names = [d[0] for d in cur.description]
+            return [dict(zip(names, r)) for r in cur.fetchall()]
+        if self.engine == "sqlalchemy":
+            from sqlalchemy import text
+            res = self.conn.execute(text(sql), params or {})
+            cols = list(res.keys())
+            return [dict(zip(cols, r)) for r in res.fetchall()]
+        cur = self.conn.cursor()
+        cur.execute(sql, params or [])
+        rows = cur.fetchall()
+        return [dict(r) for r in rows]
+
+    def read_numpy(self, sql, params=None):
+        rows = self.query(sql, params)
+        if not rows:
+            return np.empty((0, 0))
+        keys = list(rows[0].keys())
+        try:
+            arr = np.array([[r[k] for k in keys] for r in rows], dtype=float)
+        except Exception:
+            arr = np.array([[r[k] for k in keys] for r in rows], dtype=object)
+        try:
+            from ._core import _wrap as _w
+            return _w(arr)
+        except Exception:
+            return arr
+
+    def read_pandas(self, sql, params=None):
+        if not has_pandas():
+            raise ImportError("pip install pandas")
+        import pandas as pd
+        if self.engine == "duckdb":
+            return self._duck.execute(sql).df()
+        if self.engine == "sqlalchemy":
+            return pd.read_sql(sql, self._sa_engine, params=params)
+        return pd.read_sql_query(sql, self.conn, params=params)
+
+    def to_parquet(self, sql, path, params=None):
+        rows = self.query(sql, params)
+        if not has_pandas():
+            raise ImportError("pip install pandas pyarrow")
+        import pandas as pd
+        pd.DataFrame(rows).to_parquet(path, index=False)
+        return path
+
+    # ---------- maintenance ----------
+    def vacuum(self):
+        try:
+            self.execute("VACUUM")
+        except Exception:
+            pass
+
+    def close(self):
+        try:
+            if self.engine == "duckdb":
+                self._duck.close()
+            elif self.engine == "sqlalchemy":
+                self.conn.close()
+            else:
+                self.conn.close()
+        except Exception:
+            pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *a):
+        self.close()
+
+
+def connect(path=":memory:", engine="sqlite", **kw):
+    return Database(path, engine=engine, **kw)

mumpy/fft.py

@@ -0,0 +1,132 @@
+"""mumpy.fft: numpy.fft-compatible but uses scipy.fft workers when available."""
+from __future__ import annotations
+
+try:
+    import scipy.fft as _sf
+    _HAS_SCIPY = True
+except Exception:
+    _sf = None
+    _HAS_SCIPY = False
+
+import numpy as _np
+import numpy.fft as _nf
+from numpy.fft import fftfreq, rfftfreq, fftshift, ifftshift  # noqa: F401
+
+__all__ = ["fft", "ifft", "rfft", "irfft", "fft2", "ifft2", "fftn", "ifftn",
+           "rfftn", "irfftn", "hfft", "ihfft",
+           "fftfreq", "rfftfreq", "fftshift", "ifftshift",
+           "next_fast_len", "convolve", "has_scipy"]
+
+
+def has_scipy() -> bool:
+    return _HAS_SCIPY
+
+
+def next_fast_len(n):
+    try:
+        if _HAS_SCIPY:
+            return _sf.next_fast_len(n)
+    except Exception:
+        pass
+    # fallback: next pow2-ish 5-smooth
+    m = 1
+    while m < n:
+        m *= 2
+    return m
+
+
+def _call(name, a, **kw):
+    if _HAS_SCIPY:
+        kw.setdefault("workers", -1)
+        kw.setdefault("overwrite_x", False)
+        try:
+            return getattr(_sf, name)(a, **{k: v for k, v in kw.items()
+                                            if k in ("n", "axis", "norm", "workers",
+                                                     "overwrite_x", "s", "axes", "shape")})
+        except TypeError:
+            pass
+    return getattr(_nf, name)(a, **{k: v for k, v in kw.items()
+                                    if k in ("n", "axis", "norm", "s", "axes", "shape")})
+
+
+def fft(a, n=None, axis=-1, norm=None, workers=-1, overwrite_x=False):
+    return _call("fft", a, n=n, axis=axis, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def ifft(a, n=None, axis=-1, norm=None, workers=-1, overwrite_x=False):
+    return _call("ifft", a, n=n, axis=axis, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def rfft(a, n=None, axis=-1, norm=None, workers=-1, overwrite_x=False):
+    return _call("rfft", a, n=n, axis=axis, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def irfft(a, n=None, axis=-1, norm=None, workers=-1, overwrite_x=False):
+    return _call("irfft", a, n=n, axis=axis, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def fft2(a, s=None, axes=(-2, -1), norm=None, workers=-1, overwrite_x=False):
+    return _call("fft2", a, s=s, axes=axes, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def ifft2(a, s=None, axes=(-2, -1), norm=None, workers=-1, overwrite_x=False):
+    return _call("ifft2", a, s=s, axes=axes, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def fftn(a, s=None, axes=None, norm=None, workers=-1, overwrite_x=False):
+    return _call("fftn", a, s=s, axes=axes, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def ifftn(a, s=None, axes=None, norm=None, workers=-1, overwrite_x=False):
+    return _call("ifftn", a, s=s, axes=axes, norm=norm, workers=workers, overwrite_x=overwrite_x)
+
+
+def rfftn(a, s=None, axes=None, norm=None, workers=-1, overwrite_x=False):
+    try:
+        return _call("rfftn", a, s=s, axes=axes, norm=norm, workers=workers, overwrite_x=overwrite_x)
+    except AttributeError:
+        return _nf.rfftn(a, s=s, axes=axes, norm=norm)
+
+
+def irfftn(a, s=None, axes=None, norm=None, workers=-1, overwrite_x=False):
+    try:
+        return _call("irfftn", a, s=s, axes=axes, norm=norm, workers=workers, overwrite_x=overwrite_x)
+    except AttributeError:
+        return _nf.irfftn(a, s=s, axes=axes, norm=norm)
+
+
+def hfft(a, n=None, axis=-1, norm=None):
+    try:
+        if _HAS_SCIPY:
+            return _sf.hfft(a, n=n, axis=axis, norm=norm)
+    except Exception:
+        pass
+    return _nf.hfft(a, n=n, axis=axis, norm=norm)
+
+
+def ihfft(a, n=None, axis=-1, norm=None):
+    try:
+        if _HAS_SCIPY:
+            return _sf.ihfft(a, n=n, axis=axis, norm=norm)
+    except Exception:
+        pass
+    return _nf.ihfft(a, n=n, axis=axis, norm=norm)
+
+
+def convolve(a, b, mode="full"):
+    """FFT-based convolution (fast for large signals)."""
+    a = _np.asanyarray(a)
+    b = _np.asanyarray(b)
+    n = a.size + b.size - 1
+    N = next_fast_len(n)
+    FA = fft(a, n=N)
+    FB = fft(b, n=N)
+    full = _np.real(ifft(FA * FB))[:n]
+    if mode == "full":
+        return full
+    if mode == "same":
+        start = (n - max(a.size, b.size)) // 2
+        return full[start:start + max(a.size, b.size)]
+    # valid
+    start = b.size - 1
+    return full[start:start + (a.size - b.size + 1)]