kardioutils 1.0.18__tar.gz → 1.0.20__tar.gz

{kardioutils-1.0.18/kardioutils.egg-info → kardioutils-1.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kardioutils
-Version: 1.0.18
+Version: 1.0.20
 Summary: Utils lib
 Author: João Neto
 Author-email: joao.filipe.neto@gmail.com

kardioutils-1.0.20/dl2050utils/__version__.py

@@ -0,0 +1 @@
+version = "1.0.20"

{kardioutils-1.0.18 → kardioutils-1.0.20}/dl2050utils/api.py

@@ -149,6 +149,7 @@ def db_create(db):
             key CHAR(256),
             kind CHAR(32),
             status CHAR(32),
+            bucket_key CHAR(64),
             urls TEXT, -- json stringyfied list of dicts with url and file name
             results_url TEXT,
             eta DATETIME,
@@ -342,8 +343,10 @@ class APIServer:
         data = await request.json()
         urls = get_param(request, 'urls', list, data=data, LOG=self.LOG)
         req_uid = get_uid()
+        bucket_key = get_param(request, 'bucket_key', str, required=False, data=data, LOG=self.LOG)
+
         task = {'req_uid':req_uid, 'created_at':now(), 'key':key, 'kind':'ASYNC', 'status':'REQUESTED',
-                'urls': json_dumps(urls)}
+                'urls': json_dumps(urls), 'bucket_key': bucket_key,}
         self.db.insert('api_tasks', task)
         self.LOG(2, 0, label='APIServer', label2='/apiserver/req', msg=f"req_uid={req_uid}")
         return rest_ok({'req_uid':req_uid})
@@ -456,7 +459,13 @@ class APIClient:
         return sync_request(f'{self.url}{route}', method=method, headers=headers, payload=payload)
     def get_signed_urls(self, fnames): return self.do_request('/apiserver/get_signed_urls', {'fnames':fnames})
     def proc(self, data): return self.do_request('/apiserver/proc', {'base64':base64_encode(data)})
-    def req(self, urls): return self.do_request('/apiserver/req', {'urls':listify(urls)})
+    
+    def req(self, urls, bucket_key=None):
+        payload = {'urls': listify(urls)}
+        if bucket_key is not None:
+            payload['bucket_key'] = bucket_key
+        return self.do_request('/apiserver/req', payload)
+    
     def check(self, req_uid): return self.do_request('/apiserver/check', {'req_uid':req_uid})
 
 # #################################################################################################################

{kardioutils-1.0.18 → kardioutils-1.0.20}/dl2050utils/db.py

@@ -749,8 +749,10 @@ def db_import_tbl(db, p, tbl, delete=False):
     if rows is None:
         return log_and_return(f"Cant read {p}")
     if delete:
-        if db.sync_execute(f"delete from {tbl}"):
+        res = db.sync_execute(f"DELETE FROM {tbl}")
+        if res is None:
             return log_and_return(f"Error deleting tbl {tbl}")
+        print("Delete result:", res) 
     n = 0
     for row in rows:
         res = db.sync_insert(tbl, row)
@@ -783,3 +785,48 @@ def db_enable_serial(db, tbl, col):
     db.sync_execute(
         f"ALTER TABLE {tbl} ALTER COLUMN {col} SET DEFAULT nextval('{tbl}_{col}_seq')"
     )
+
+def row_exists_full(db, tbl, row: dict, cols=None):
+    cols = cols or list(row.keys())
+    where = " AND ".join([f"{c} IS NOT DISTINCT FROM ${i+1}" for i, c in enumerate(cols)])
+    q = f"SELECT 1 FROM {tbl} WHERE {where} LIMIT 1"
+    params = [convert_type(row.get(c)) for c in cols]
+    res = db.sync_execute(q, *params)
+    return bool(res)
+
+def db_import_tbl_full_compare(db, p, tbl, delete=False, cols=None):
+    from pathlib import Path
+    p = Path(p)
+    if not p.with_suffix(".pickle").is_file():
+        return log_and_return(f"Error importing {tbl}: file {p} not found")
+
+    rows = pickle_load(p)
+    if rows is None:
+        return log_and_return(f"Cant read {p}")
+    if not rows:
+        print("No rows to import.")
+        return 0
+
+    if delete:
+        res = db.sync_execute(f"DELETE FROM {tbl}")
+        if res is None:
+            return log_and_return(f"Error deleting tbl {tbl}")
+
+    cols = cols or list(rows[0].keys())
+    col_list = ", ".join(cols)
+    placeholders = ", ".join([f"${i}" for i in range(1, len(cols)+1)])
+    qins = f"INSERT INTO {tbl} ({col_list}) VALUES ({placeholders})"
+
+    n_new, n_skip = 0, 0
+    for r in rows:
+        r2 = {c: r.get(c) for c in cols}
+        if not delete and row_exists_full(db, tbl, r2, cols=cols):
+            n_skip += 1
+            continue
+        res = db.sync_execute(qins, *[convert_type(r2.get(c)) for c in cols])
+        if res is not None:
+            try: n_new += int(str(res).split()[-1])
+            except: pass
+
+    print(f"new={n_new} skipped={n_skip}")
+    return 0

kardioutils-1.0.20/dl2050utils/df_utils.py

@@ -0,0 +1,155 @@
+import pandas as pd
+import os
+from __future__ import annotations
+from typing import Tuple, List, Optional
+import numpy as np
+
+def list_prefixes(df: pd.DataFrame) -> list:
+    """Return all distinct prefixes in the dataframe."""
+    return df["prefix"].dropna().unique().tolist()
+
+
+def filter_by_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
+    """Return all rows that match a given prefix exactly."""
+    return df[df["prefix"] == prefix]
+
+
+def filter_prefix_contains(df: pd.DataFrame, text: str) -> pd.DataFrame:
+    """Return all rows where prefix contains the given text."""
+    return df[df["prefix"].str.contains(text, na=False)]
+
+
+def find_by_uid_suffix(df: pd.DataFrame, uid_suffix: str) -> pd.DataFrame:
+    """Return all rows that match a given uid_suffix."""
+    return df[df["uid_suffix"] == uid_suffix]
+
+
+def find_by_uid_full(df: pd.DataFrame, uid_full: str) -> pd.DataFrame:
+    """Return all rows that match a given uid_full."""
+    return df[df["uid_full"] == uid_full]
+
+
+def holter_only(df: pd.DataFrame) -> pd.DataFrame:
+    """Return only rows where holter == True."""
+    return df[df["holter"] == True]
+
+
+def non_holter_only(df: pd.DataFrame) -> pd.DataFrame:
+    """Return only rows where holter == False."""
+    return df[df["holter"] == False]
+
+
+def get_path_by_uid_suffix(df: pd.DataFrame, uid_suffix: str) -> str | None:
+    """
+    Return the path for a given uid_suffix.
+    If there are multiple rows, returns the first one.
+    If nothing is found, returns None.
+    """
+    rows = df[df["uid_suffix"] == uid_suffix]
+    if rows.empty:
+        return None
+    return rows.iloc[0]["path"]
+
+
+def get_paths_by_prefix(df: pd.DataFrame, prefix: str, holter_only_flag: bool | None = None) -> list:
+    """
+    Return a list of paths filtered by prefix and optionally holter flag.
+    - holter_only_flag = True  → only holter rows
+    - holter_only_flag = False → only non-holter rows
+    - holter_only_flag = None  → ignore holter column
+    """
+    subset = df[df["prefix"] == prefix]
+    if holter_only_flag is not None:
+        subset = subset[subset["holter"] == holter_only_flag]
+    return subset["path"].dropna().tolist()
+
+
+def check_missing_files(df):
+    """
+    Return subset of rows whose 'path' does not point to an existing file.
+    """
+    mask = ~df["path"].astype(str).apply(os.path.exists)
+    return df[mask]
+
+
+def check_existing_files(df):
+    """
+    Return subset of rows whose 'path' exists.
+    """
+    mask = df["path"].astype(str).apply(os.path.exists)
+    return df[mask]
+
+def load_X_from_index_df(
+    index_df: pd.DataFrame,
+    fdb,
+    *,
+    uid_col: str = "uid_full",
+    pre: str = "x_",
+    ext: str = ".npy",
+    allow_pickle: bool = False,
+    stack: bool = True,
+    on_missing: str = "skip",   # "skip" | "raise" | "keep_none"
+) -> Tuple[np.ndarray, List[str], pd.DataFrame]:
+    """
+    Loads x_ arrays for each row in index_df using fdb.load(row[uid_col], pre, ext).
+
+    Returns:
+      - X: stacked np.ndarray (N, ...) if stack=True; otherwise object array of length N
+      - ids: list of ids in the same order as X
+      - meta: dataframe aligned with X (rows kept), including a 'loaded' boolean column
+    """
+    if uid_col not in index_df.columns:
+        raise KeyError(f"uid_col '{uid_col}' not found in index_df columns")
+
+    loaded_arrays = []
+    kept_ids: List[str] = []
+    kept_rows = []
+    missing_rows = []
+
+    for _, row in index_df.iterrows():
+        uid = row[uid_col]
+        arr = fdb.load(uid, pre=pre, ext=ext, allow_pickle=allow_pickle)
+
+        if arr is None:
+            if on_missing == "raise":
+                raise FileNotFoundError(f"Missing array for {uid} (pre={pre}, ext={ext})")
+            if on_missing == "keep_none":
+                loaded_arrays.append(None)
+                kept_ids.append(uid)
+                r = row.copy()
+                r["loaded"] = False
+                kept_rows.append(r)
+            else:  # skip
+                r = row.copy()
+                r["loaded"] = False
+                missing_rows.append(r)
+            continue
+
+        loaded_arrays.append(arr)
+        kept_ids.append(uid)
+        r = row.copy()
+        r["loaded"] = True
+        kept_rows.append(r)
+
+    meta = pd.DataFrame(kept_rows).reset_index(drop=True)
+
+    if not stack:
+        # keep as object array (useful if shapes can differ or you used keep_none)
+        X = np.array(loaded_arrays, dtype=object)
+        return X, kept_ids, meta
+
+    # stack=True: require all arrays exist and have same shape
+    arrays_only = [a for a in loaded_arrays if a is not None]
+    if len(arrays_only) == 0:
+        return np.empty((0,), dtype=float), kept_ids, meta
+
+    try:
+        X = np.stack(arrays_only, axis=0)
+    except Exception as e:
+        raise ValueError(
+            "Could not stack arrays (shapes likely differ). "
+            "Use stack=False or handle padding/truncation."
+        ) from e
+
+   
+    return X, kept_ids, meta

{kardioutils-1.0.18 → kardioutils-1.0.20}/dl2050utils/gs.py

@@ -64,6 +64,9 @@ class GS:
         # Try Google Cloud first
         key_dict = oget(cfg, ["gcloud", "gs_key"])
         fs_cfg = oget(cfg, ["fs"]) or {}
+        bucket_cfg = oget(cfg, ["bucket"]) or {}
+        self.bucket_map = bucket_cfg if isinstance(bucket_cfg, dict) else {}
+        self.default_bucket = self.bucket_map.get("general")
         self.internal_token = fs_cfg.get("internal_token")
         if self.internal_token:
             os.environ["FS_INTERNAL_TOKEN"] = self.internal_token
@@ -573,3 +576,14 @@ class GS:
             self.upload_url(bucket_name, blob_name, timeout=timeout, size=size),
             self.download_url(bucket_name, blob_name, timeout=timeout),
         )
+    def resolve_bucket(self, bucket_name=None, bucket_key=None):
+        if bucket_name:
+            return bucket_name
+        if bucket_key:
+            if bucket_key in self.bucket_map:
+                return self.bucket_map[bucket_key]
+            raise RuntimeError(f"GS: unknown bucket_key '{bucket_key}'")
+        if self.default_bucket:
+            return self.default_bucket
+        raise RuntimeError("GS: missing bucket.general in config")
+

{kardioutils-1.0.18 → kardioutils-1.0.20}/dl2050utils/restapp.py

@@ -173,7 +173,12 @@ class App():
         files = args['files']
         uid = get_uid()
         upload_urls,download_urls = [],[]
-        bucket = f'{self.service}-apiserver'
+        u = await self.auth.check_auth(request)
+        org_id= u.get("org")
+        org = await self.db.select_one("orgs", {"id": org_id})
+        bucket_key = org.get("bucket_key")
+        bucket = self.fs.resolve_bucket(bucket_key=bucket_key)
+        #bucket = f'{self.service}-apiserver'
         for file in files:
             upload_url,download_url = self.fs.urls(bucket, f'{uid}/{file}')
             if upload_url is None or download_url is None:

{kardioutils-1.0.18 → kardioutils-1.0.20/kardioutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kardioutils
-Version: 1.0.18
+Version: 1.0.20
 Summary: Utils lib
 Author: João Neto
 Author-email: joao.filipe.neto@gmail.com

kardioutils-1.0.18/dl2050utils/__version__.py

@@ -1 +0,0 @@
-version = "1.0.18"

kardioutils-1.0.18/dl2050utils/df_utils.py

@@ -1,77 +0,0 @@
-import pandas as pd
-import os
-
-def list_prefixes(df: pd.DataFrame) -> list:
-    """Return all distinct prefixes in the dataframe."""
-    return df["prefix"].dropna().unique().tolist()
-
-
-def filter_by_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
-    """Return all rows that match a given prefix exactly."""
-    return df[df["prefix"] == prefix]
-
-
-def filter_prefix_contains(df: pd.DataFrame, text: str) -> pd.DataFrame:
-    """Return all rows where prefix contains the given text."""
-    return df[df["prefix"].str.contains(text, na=False)]
-
-
-def find_by_uid_suffix(df: pd.DataFrame, uid_suffix: str) -> pd.DataFrame:
-    """Return all rows that match a given uid_suffix."""
-    return df[df["uid_suffix"] == uid_suffix]
-
-
-def find_by_uid_full(df: pd.DataFrame, uid_full: str) -> pd.DataFrame:
-    """Return all rows that match a given uid_full."""
-    return df[df["uid_full"] == uid_full]
-
-
-def holter_only(df: pd.DataFrame) -> pd.DataFrame:
-    """Return only rows where holter == True."""
-    return df[df["holter"] == True]
-
-
-def non_holter_only(df: pd.DataFrame) -> pd.DataFrame:
-    """Return only rows where holter == False."""
-    return df[df["holter"] == False]
-
-
-def get_path_by_uid_suffix(df: pd.DataFrame, uid_suffix: str) -> str | None:
-    """
-    Return the path for a given uid_suffix.
-    If there are multiple rows, returns the first one.
-    If nothing is found, returns None.
-    """
-    rows = df[df["uid_suffix"] == uid_suffix]
-    if rows.empty:
-        return None
-    return rows.iloc[0]["path"]
-
-
-def get_paths_by_prefix(df: pd.DataFrame, prefix: str, holter_only_flag: bool | None = None) -> list:
-    """
-    Return a list of paths filtered by prefix and optionally holter flag.
-    - holter_only_flag = True  → only holter rows
-    - holter_only_flag = False → only non-holter rows
-    - holter_only_flag = None  → ignore holter column
-    """
-    subset = df[df["prefix"] == prefix]
-    if holter_only_flag is not None:
-        subset = subset[subset["holter"] == holter_only_flag]
-    return subset["path"].dropna().tolist()
-
-
-def check_missing_files(df):
-    """
-    Return subset of rows whose 'path' does not point to an existing file.
-    """
-    mask = ~df["path"].astype(str).apply(os.path.exists)
-    return df[mask]
-
-
-def check_existing_files(df):
-    """
-    Return subset of rows whose 'path' exists.
-    """
-    mask = df["path"].astype(str).apply(os.path.exists)
-    return df[mask]