kardioutils 1.0.19__tar.gz → 1.0.21__tar.gz

{kardioutils-1.0.19/kardioutils.egg-info → kardioutils-1.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kardioutils
-Version: 1.0.19
+Version: 1.0.21
 Summary: Utils lib
 Author: João Neto
 Author-email: joao.filipe.neto@gmail.com

kardioutils-1.0.21/dl2050utils/__version__.py

@@ -0,0 +1 @@
+version = "1.0.21"

{kardioutils-1.0.19 → kardioutils-1.0.21}/dl2050utils/api.py

@@ -149,6 +149,7 @@ def db_create(db):
             key CHAR(256),
             kind CHAR(32),
             status CHAR(32),
+            bucket_key CHAR(64),
             urls TEXT, -- json stringyfied list of dicts with url and file name
             results_url TEXT,
             eta DATETIME,
@@ -342,8 +343,10 @@ class APIServer:
         data = await request.json()
         urls = get_param(request, 'urls', list, data=data, LOG=self.LOG)
         req_uid = get_uid()
+        bucket_key = get_param(request, 'bucket_key', str, required=False, data=data, LOG=self.LOG)
+
         task = {'req_uid':req_uid, 'created_at':now(), 'key':key, 'kind':'ASYNC', 'status':'REQUESTED',
-                'urls': json_dumps(urls)}
+                'urls': json_dumps(urls), 'bucket_key': bucket_key,}
         self.db.insert('api_tasks', task)
         self.LOG(2, 0, label='APIServer', label2='/apiserver/req', msg=f"req_uid={req_uid}")
         return rest_ok({'req_uid':req_uid})
@@ -456,7 +459,13 @@ class APIClient:
         return sync_request(f'{self.url}{route}', method=method, headers=headers, payload=payload)
     def get_signed_urls(self, fnames): return self.do_request('/apiserver/get_signed_urls', {'fnames':fnames})
     def proc(self, data): return self.do_request('/apiserver/proc', {'base64':base64_encode(data)})
-    def req(self, urls): return self.do_request('/apiserver/req', {'urls':listify(urls)})
+    
+    def req(self, urls, bucket_key=None):
+        payload = {'urls': listify(urls)}
+        if bucket_key is not None:
+            payload['bucket_key'] = bucket_key
+        return self.do_request('/apiserver/req', payload)
+    
     def check(self, req_uid): return self.do_request('/apiserver/check', {'req_uid':req_uid})
 
 # #################################################################################################################

{kardioutils-1.0.19 → kardioutils-1.0.21}/dl2050utils/db.py

@@ -785,3 +785,48 @@ def db_enable_serial(db, tbl, col):
     db.sync_execute(
         f"ALTER TABLE {tbl} ALTER COLUMN {col} SET DEFAULT nextval('{tbl}_{col}_seq')"
     )
+
+def row_exists_full(db, tbl, row: dict, cols=None):
+    cols = cols or list(row.keys())
+    where = " AND ".join([f"{c} IS NOT DISTINCT FROM ${i+1}" for i, c in enumerate(cols)])
+    q = f"SELECT 1 FROM {tbl} WHERE {where} LIMIT 1"
+    params = [convert_type(row.get(c)) for c in cols]
+    res = db.sync_execute(q, *params)
+    return bool(res)
+
+def db_import_tbl_full_compare(db, p, tbl, delete=False, cols=None):
+    from pathlib import Path
+    p = Path(p)
+    if not p.with_suffix(".pickle").is_file():
+        return log_and_return(f"Error importing {tbl}: file {p} not found")
+
+    rows = pickle_load(p)
+    if rows is None:
+        return log_and_return(f"Cant read {p}")
+    if not rows:
+        print("No rows to import.")
+        return 0
+
+    if delete:
+        res = db.sync_execute(f"DELETE FROM {tbl}")
+        if res is None:
+            return log_and_return(f"Error deleting tbl {tbl}")
+
+    cols = cols or list(rows[0].keys())
+    col_list = ", ".join(cols)
+    placeholders = ", ".join([f"${i}" for i in range(1, len(cols)+1)])
+    qins = f"INSERT INTO {tbl} ({col_list}) VALUES ({placeholders})"
+
+    n_new, n_skip = 0, 0
+    for r in rows:
+        r2 = {c: r.get(c) for c in cols}
+        if not delete and row_exists_full(db, tbl, r2, cols=cols):
+            n_skip += 1
+            continue
+        res = db.sync_execute(qins, *[convert_type(r2.get(c)) for c in cols])
+        if res is not None:
+            try: n_new += int(str(res).split()[-1])
+            except: pass
+
+    print(f"new={n_new} skipped={n_skip}")
+    return 0

{kardioutils-1.0.19 → kardioutils-1.0.21}/dl2050utils/fdb.py

@@ -59,7 +59,7 @@ class FileDB():
         """
         return f'{pre}{key}{ext}'
     
-    def save(self, key, data, pre='', ext='', save_f=pickle_save, allow_pickle=False):
+    def save(self, key, data, pre='', ext='', save_f=pickle_save, allow_pickle=False, show=False):
         """
         Saves data to a file in the directory corresponding to the key.
         Args:
@@ -81,10 +81,11 @@ class FileDB():
                 save_f(p, data)
             return 0
         except Exception as exc:
-            print(f'FileDB save Exception: {exc}')
+            if show:
+                print(f'FileDB save Exception: {exc}')
             return 1
 
-    def load(self, key, pre='', ext='', load_f=pickle_load, allow_pickle=False):
+    def load(self, key, pre='', ext='', load_f=pickle_load, allow_pickle=False,show=False):
         """
         Loads data from a file based on the key, prefix, and extension.
         Args:
@@ -102,10 +103,11 @@ class FileDB():
             if ext=='.npy': return np.load(p, allow_pickle=allow_pickle)
             return load_f(p)
         except Exception as exc:
-            print(f'FileDB load Exception: {exc}')
+            if show:
+                print(f'FileDB load Exception: {exc}')
             return None
     
-    def delete(self, key, pre='', ext=''):
+    def delete(self, key, pre='', ext='',show=False):
         """
         Deletes file based on the key, prefix, and extension.
         Args:
@@ -121,5 +123,6 @@ class FileDB():
             p.unlink()
             return 0
         except Exception as exc:
-            print(f'FileDB delete Exception: {exc}')
+            if show:
+                print(f'FileDB delete Exception: {exc}')
             return 1

{kardioutils-1.0.19 → kardioutils-1.0.21}/dl2050utils/gs.py

@@ -64,6 +64,9 @@ class GS:
         # Try Google Cloud first
         key_dict = oget(cfg, ["gcloud", "gs_key"])
         fs_cfg = oget(cfg, ["fs"]) or {}
+        bucket_cfg = oget(cfg, ["bucket"]) or {}
+        self.bucket_map = bucket_cfg if isinstance(bucket_cfg, dict) else {}
+        self.default_bucket = self.bucket_map.get("general")
         self.internal_token = fs_cfg.get("internal_token")
         if self.internal_token:
             os.environ["FS_INTERNAL_TOKEN"] = self.internal_token
@@ -573,3 +576,14 @@ class GS:
             self.upload_url(bucket_name, blob_name, timeout=timeout, size=size),
             self.download_url(bucket_name, blob_name, timeout=timeout),
         )
+    def resolve_bucket(self, bucket_name=None, bucket_key=None):
+        if bucket_name:
+            return bucket_name
+        if bucket_key:
+            if bucket_key in self.bucket_map:
+                return self.bucket_map[bucket_key]
+            raise RuntimeError(f"GS: unknown bucket_key '{bucket_key}'")
+        if self.default_bucket:
+            return self.default_bucket
+        raise RuntimeError("GS: missing bucket.general in config")
+

{kardioutils-1.0.19 → kardioutils-1.0.21}/dl2050utils/restapp.py

@@ -173,7 +173,12 @@ class App():
         files = args['files']
         uid = get_uid()
         upload_urls,download_urls = [],[]
-        bucket = f'{self.service}-apiserver'
+        u = await self.auth.check_auth(request)
+        org_id= u.get("org")
+        org = await self.db.select_one("orgs", {"id": org_id})
+        bucket_key = org.get("bucket_key")
+        bucket = self.fs.resolve_bucket(bucket_key=bucket_key)
+        #bucket = f'{self.service}-apiserver'
         for file in files:
             upload_url,download_url = self.fs.urls(bucket, f'{uid}/{file}')
             if upload_url is None or download_url is None:

{kardioutils-1.0.19 → kardioutils-1.0.21/kardioutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kardioutils
-Version: 1.0.19
+Version: 1.0.21
 Summary: Utils lib
 Author: João Neto
 Author-email: joao.filipe.neto@gmail.com

kardioutils-1.0.19/dl2050utils/__version__.py

@@ -1 +0,0 @@
-version = "1.0.19"