kardioutils 1.0.10__tar.gz → 1.0.11__tar.gz

{kardioutils-1.0.10/kardioutils.egg-info → kardioutils-1.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kardioutils
-Version: 1.0.10
+Version: 1.0.11
 Summary: Utils lib
 Author: João Neto
 Author-email: joao.filipe.neto@gmail.com

kardioutils-1.0.11/dl2050utils/__version__.py

@@ -0,0 +1 @@
+version = "1.0.11"

kardioutils-1.0.11/dl2050utils/df_utils.py

@@ -0,0 +1,77 @@
+import pandas as pd
+import os
+
+def list_prefixes(df: pd.DataFrame) -> list:
+    """Return all distinct prefixes in the dataframe."""
+    return df["prefix"].dropna().unique().tolist()
+
+
+def filter_by_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
+    """Return all rows that match a given prefix exactly."""
+    return df[df["prefix"] == prefix]
+
+
+def filter_prefix_contains(df: pd.DataFrame, text: str) -> pd.DataFrame:
+    """Return all rows where prefix contains the given text."""
+    return df[df["prefix"].str.contains(text, na=False)]
+
+
+def find_by_uid_suffix(df: pd.DataFrame, uid_suffix: str) -> pd.DataFrame:
+    """Return all rows that match a given uid_suffix."""
+    return df[df["uid_suffix"] == uid_suffix]
+
+
+def find_by_uid_full(df: pd.DataFrame, uid_full: str) -> pd.DataFrame:
+    """Return all rows that match a given uid_full."""
+    return df[df["uid_full"] == uid_full]
+
+
+def holter_only(df: pd.DataFrame) -> pd.DataFrame:
+    """Return only rows where holter == True."""
+    return df[df["holter"] == True]
+
+
+def non_holter_only(df: pd.DataFrame) -> pd.DataFrame:
+    """Return only rows where holter == False."""
+    return df[df["holter"] == False]
+
+
+def get_path_by_uid_suffix(df: pd.DataFrame, uid_suffix: str) -> str | None:
+    """
+    Return the path for a given uid_suffix.
+    If there are multiple rows, returns the first one.
+    If nothing is found, returns None.
+    """
+    rows = df[df["uid_suffix"] == uid_suffix]
+    if rows.empty:
+        return None
+    return rows.iloc[0]["path"]
+
+
+def get_paths_by_prefix(df: pd.DataFrame, prefix: str, holter_only_flag: bool | None = None) -> list:
+    """
+    Return a list of paths filtered by prefix and optionally holter flag.
+    - holter_only_flag = True  → only holter rows
+    - holter_only_flag = False → only non-holter rows
+    - holter_only_flag = None  → ignore holter column
+    """
+    subset = df[df["prefix"] == prefix]
+    if holter_only_flag is not None:
+        subset = subset[subset["holter"] == holter_only_flag]
+    return subset["path"].dropna().tolist()
+
+
+def check_missing_files(df):
+    """
+    Return subset of rows whose 'path' does not point to an existing file.
+    """
+    mask = ~df["path"].astype(str).apply(os.path.exists)
+    return df[mask]
+
+
+def check_existing_files(df):
+    """
+    Return subset of rows whose 'path' exists.
+    """
+    mask = df["path"].astype(str).apply(os.path.exists)
+    return df[mask]

{kardioutils-1.0.10 → kardioutils-1.0.11}/dl2050utils/gs.py

@@ -8,12 +8,45 @@ import datetime
 import re
 import pickle
 import mimetypes
+import requests
 from google.cloud import storage
 from dl2050utils.core import oget
 from dl2050utils.env import config_load
 from dl2050utils.fs import json_save
+import hashlib
+import hmac
+import urllib.parse
+from pathlib import Path
 
 
+class _URLSigner:
+    """Internal HMAC-based URL signer for local backend."""
+
+    def __init__(self, secret_key: str, base_url: str):
+        self.secret = secret_key.encode("utf-8")
+        self.base_url = base_url.rstrip("/")
+
+    def _make_signature(self, method: str, bucket: str, blob: str, exp: int, max_size: int | None):
+        payload = f"{method}\n{bucket}\n{blob}\n{exp}\n{max_size or ''}"
+        return hmac.new(self.secret, payload.encode("utf-8"), hashlib.sha256).hexdigest()
+
+    def generate_url(self, path: str, method: str, bucket: str, blob: str,
+                     timeout: int, max_size: int | None = None) -> str:
+        import time
+        exp = int(time.time()) + timeout
+        sig = self._make_signature(method, bucket, blob, exp, max_size)
+        query = {
+            "bucket": bucket,
+            "blob": blob,
+            "exp": exp,
+            "method": method,
+            "sig": sig,
+        }
+        if max_size is not None:
+            query["max_size"] = max_size
+        qs = urllib.parse.urlencode(query)
+        return f"{self.base_url}{path}?{qs}"
+
 class GS:
     """
     Google Cloud Storage helper class to manage buckets, files, and URLs.
@@ -27,23 +60,39 @@ class GS:
     """
 
     def __init__(self, service, default_location="europe-west1"):
-        """
-        Initializes the GS class with the specified Google Cloud service and location.
-        Args:
-            service (str): The Google Cloud service name.
-            default_location (str): Default location for bucket creation. Defaults to "europe-west1".
-        """
         cfg = config_load(service)
-        # Create credentials file from config yml
+        # Try Google Cloud first
         key_dict = oget(cfg, ["gcloud", "gs_key"])
-        assert key_dict["type"] == "service_account"
-        credentials_p = "./gs-keyfile.json"
-        json_save(credentials_p, key_dict)
-        # Set the GOOGLE_APPLICATION_CREDENTIALS env var to use the credentials file
-        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_p
-        # Connect
-        self.default_location = default_location
-        self.gc = storage.Client()
+        fs_cfg = oget(cfg, ["fs"]) or {}
+
+        self.mode = None  # 'gcloud' or 'local'
+
+        if key_dict is not None:
+            # ---------- GCS MODE ----------
+            assert key_dict["type"] == "service_account"
+            credentials_p = "./gs-keyfile.json"
+            json_save(credentials_p, key_dict)
+            os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_p
+            self.default_location = default_location
+            self.gc = storage.Client()
+            self.mode = "gcloud"
+        elif fs_cfg.get("backend") == "local":
+            # ---------- LOCAL MODE ----------
+            self.mode = "local"
+            self.default_location = "local"
+            self.gc = None  # not used
+
+            self.root_dir = Path(fs_cfg.get("root_dir", f"/data/{service}/fs"))
+            self.root_dir.mkdir(parents=True, exist_ok=True)
+
+            base_url = fs_cfg.get("url", "http://localhost:8001")
+            secret = fs_cfg.get("secret")
+            if not secret:
+                raise RuntimeError("GS local backend enabled but fs.secret not configured")
+
+            self._signer = _URLSigner(secret_key=secret, base_url=base_url)
+        else:
+            raise RuntimeError("GS: neither gcloud.gs_key nor fs.backend=local configured")
 
     # ####################################################################################################
     # Admin
@@ -172,30 +221,56 @@ class GS:
     # Memmory Download, Upload
     # ###################################################################################################################
 
-    def upload_mem(self, bucket_name, blob_name, data, content_type="application/octet-stream", use_pickle=True):
+    def upload_mem(self, bucket_name, blob_name, data,
+                   content_type="application/octet-stream",
+                   use_pickle=True):
         """
-        Uploads data from memory to a specified bucket and blob.
-        Args:
-            bucket_name (str): Name of the GCS bucket.
-            blob_name (str): Name of the blob to upload.
-            data (str or bytes): Data to upload. Can be a string or bytes.
-            content_type (str, optional): MIME type of the data. Defaults to 'application/octet-stream'.
-            use_pickle (bool, optional): If True, serializes the data using pickle before uploading. Defaults to False.
-        Returns:
-            int: 0 if upload is successful, 1 otherwise.
-        Examples:
-            gs.upload_mem(bucket_name, blob_name, data="Hello, world!", content_type='text/plain')
-            gs.upload_mem(bucket_name, blob_name, data=b'\x89PNG\r\n\x1a...', content_type='image/png')
+        Uploads data from memory to storage (GCS or local FS).
         """
         try:
             if use_pickle:
                 data = pickle.dumps(data)
             elif isinstance(data, str):
                 data = data.encode("utf-8")
-            bucket = self.gc.bucket(bucket_name)
-            blob = bucket.blob(blob_name)
-            blob.upload_from_string(data, content_type=content_type)
-            return 0
+
+
+            if self.mode == "gcloud":
+                # --------- Google Cloud ---------
+                bucket = self.gc.bucket(bucket_name)
+                blob = bucket.blob(blob_name)
+                blob.upload_from_string(data, content_type=content_type)
+                return 0
+
+            elif self.mode == "local":
+                # --------- Local fs-server (fs.py) via HTTP ---------
+                size = len(data)
+                upload_url = self.upload_url(
+                    bucket_name,
+                    blob_name,
+                    timeout=15 * 60,
+                    size=size,
+                )
+                if not upload_url:
+                    print("upload_mem (local) ERROR: could not generate upload_url")
+                    return 1
+
+                resp = requests.put(
+                    upload_url,
+                    data=data,
+                    headers={"Content-Type": content_type},
+                    timeout=60,
+                )
+
+                if resp.status_code not in (200, 201):
+                    print("upload_mem (local) ERROR:", resp.status_code, resp.text)
+                    return 1
+
+                return 0
+
+            else:
+                print("upload_mem ERROR: unknown mode", self.mode)
+                return 1
+
         except Exception as exc:
             print(f"upload_mem EXCEPTION: {str(exc)}")
             return 1
@@ -215,21 +290,48 @@ class GS:
             Any: The data from the blob, possibly decoded or deserialized.
         """
         try:
-            bucket = self.gc.bucket(bucket_name)
-            blob = bucket.blob(blob_name)
-            data = blob.download_as_bytes()
+            if self.mode == "gcloud":
+                # --------- Google Cloud ---------
+                bucket = self.gc.bucket(bucket_name)
+                blob = bucket.blob(blob_name)
+                data = blob.download_as_bytes()
+
+            elif self.mode == "local":
+                # --------- Local fs-server (fs.py) via HTTP ---------
+                download_url = self.download_url(
+                    bucket_name,
+                    blob_name,
+                    timeout=24 * 3600,
+                )
+                if not download_url:
+                    print("download_mem (local) ERROR: could not generate download_url")
+                    return None
+                internal_token = os.environ.get("FS_INTERNAL_TOKEN")
+                headers = {}
+                if internal_token:
+                    headers["X-Internal-Token"] = internal_token
+
+
+                resp = requests.get(download_url, headers=headers, timeout=60)
+
+                if resp.status_code != 200:
+                    print("download_mem (local) ERROR:", resp.status_code, resp.text)
+                    return None
+
+                data = resp.content
+
+            else:
+                print("download_mem ERROR: unknown mode", self.mode)
+                return None
+                # Pós-processamento igual para os dois modos
             if use_pickle:
-                # Deserialize the data using pickle
                 data = pickle.loads(data)
             elif as_string:
-                # Decode the data using the specified encoding
                 data = data.decode(encoding)
-            # If neither use_pickle nor as_string is True, return the raw bytes
             return data
         except Exception as exc:
             print(f"download_mem EXCEPTION: {str(exc)}")
             return None
-
     # ###################################################################################################################
     # File Download, Upload
     # ###################################################################################################################
@@ -243,18 +345,20 @@ class GS:
     ):
         """
         Uploads a local file to a specified bucket and blob.
-        Args:
-            bucket_name (str): Name of the GCS bucket.
-            blob_name (str): Name of the blob to upload.
-            local_file_path (str): Local path of the file to upload.
-            content_type (str, optional): MIME type of the data. Defaults to 'application/octet-stream'.
-        Returns:
-            int: 0 if upload is successful, 1 otherwise.
+        - In GCS mode: uploads to Google Cloud.
+        - In local mode: copies the file into root_dir / bucket / blob.
         """
         try:
-            bucket = self.gc.bucket(bucket_name)
-            blob = bucket.blob(blob_name)
-            blob.upload_from_filename(local_file_path, content_type=content_type)
+            if self.mode == "gcloud":
+                bucket = self.gc.bucket(bucket_name)
+                blob = bucket.blob(blob_name)
+                blob.upload_from_filename(local_file_path, content_type=content_type)
+                return 0
+            # LOCAL MODE
+            dst = self.root_dir / bucket_name / blob_name
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            import shutil
+            shutil.copy2(local_file_path, dst)
             return 0
         except Exception as exc:
             print(f"upload_file EXCEPTION: {str(exc)}")
@@ -263,18 +367,50 @@ class GS:
     def download_file(self, bucket_name, blob_name, local_file_path):
         """
         Downloads a blob from the bucket to a local file.
-        Args:
-            bucket_name (str): Name of the GCS bucket.
-            blob_name (str): Name of the blob to download.
-            local_file_path (str): Local path to save the downloaded file.
-        Returns:
-            int: 0 if upload is successful, 1 otherwise.
+        - In GCS mode: downloads from Google Cloud.
+        - In local mode: copies from root_dir / bucket / blob.
         """
         try:
-            bucket = self.gc.bucket(bucket_name)
-            blob = bucket.blob(blob_name)
-            blob.download_to_filename(local_file_path)
-            return 0
+            if self.mode == "gcloud":
+                bucket = self.gc.bucket(bucket_name)
+                blob = bucket.blob(blob_name)
+                blob.download_to_filename(local_file_path)
+                return 0
+
+            # LOCAL MODE
+            elif self.mode == "local":
+                # --------- Local fs-server (fs.py) via HTTP ---------
+                download_url = self.download_url(
+                    bucket_name,
+                    blob_name,
+                    timeout=24 * 3600,
+                )
+                if not download_url:
+                    print("download_file (local) ERROR: could not generate download_url")
+                    return 1
+
+                internal_token = os.environ.get("FS_INTERNAL_TOKEN")
+                headers = {}
+                if internal_token:
+                    headers["X-Internal-Token"] = internal_token
+
+                # stream to not load everyting in ram 
+                with requests.get(download_url, headers=headers, stream=True, timeout=60) as r:
+                    if r.status_code != 200:
+                        print("download_file (local) ERROR:", r.status_code, r.text)
+                        return 1
+
+                    Path(local_file_path).parent.mkdir(parents=True, exist_ok=True)
+                    with open(local_file_path, "wb") as f:
+                        for chunk in r.iter_content(chunk_size=1024 * 1024):
+                            if chunk:
+                                f.write(chunk)
+
+                return 0
+
+            else:
+                print("download_file ERROR: unknown mode", self.mode)
+                return 1
         except Exception as exc:
             print(f"download_file EXCEPTION: {str(exc)}")
             return 1
@@ -357,68 +493,79 @@ class GS:
     def upload_url(self, bucket_name, blob_name, timeout=15 * 60, size=None):
         """
         Generates a signed URL for uploading a blob.
-        Args:
-            bucket_name (str): Name of the GCS bucket.
-            blob_name (str): Name of the blob to upload.
-            timeout (int, optional): URL expiration time in seconds. Defaults to 15 minutes.
-            size (int, optional): Maximum allowed size of the upload in bytes.
-        Returns:
-            str or None: Signed URL for uploading or None if an error occurs.
+        - Local mode: signed URL for local fileserver (/upload).
         """
+        if self.mode == "gcloud":
+            try:
+                bucket = self.gc.bucket(bucket_name)
+                blob = bucket.blob(blob_name)
+                query_parameters = (
+                    None if size is None else {"x-goog-content-length-range": f"0,{size}"}
+                )
+                url = blob.generate_signed_url(
+                    version="v4",
+                    expiration=datetime.timedelta(seconds=timeout),
+                    method="PUT",
+                    content_type="application/octet-stream",
+                    query_parameters=query_parameters,
+                )
+                return url
+            except Exception as exc:
+                print(f"upload_url EXCEPTION: {str(exc)}")
+                return None
+
+        # LOCAL MODE
         try:
-            bucket = self.gc.bucket(bucket_name)
-            blob = bucket.blob(blob_name)
-            query_parameters = (
-                None if size is None else {"x-goog-content-length-range": f"0,{size}"}
-            )
-            url = blob.generate_signed_url(
-                version="v4",
-                expiration=datetime.timedelta(seconds=timeout),
+            return self._signer.generate_url(
+                path="/upload",
                 method="PUT",
-                content_type="application/octet-stream",
-                query_parameters=query_parameters,
+                bucket=bucket_name,
+                blob=blob_name,
+                timeout=timeout,
+                max_size=size,
             )
-            return url
         except Exception as exc:
-            print(f"upload_url EXCEPTION: {str(exc)}")
+            print(f"upload_url (local) EXCEPTION: {str(exc)}")
             return None
 
     def download_url(self, bucket_name, blob_name, timeout=24 * 3600):
         """
         Generates a signed URL for downloading a blob.
-        Args:
-            bucket_name (str): Name of the GCS bucket.
-            blob_name (str): Name of the blob to download.
-            timeout (int, optional): URL expiration time in seconds. Defaults to 24 hours.
-        Returns:
-            str or None: Signed URL for downloading or None if an error occurs.
+        - Local mode: signed URL for local fileserver (/download).
         """
+        if self.mode == "gcloud":
+            try:
+                bucket = self.gc.bucket(bucket_name)
+                blob = bucket.blob(blob_name)
+                url = blob.generate_signed_url(
+                    version="v4",
+                    expiration=datetime.timedelta(seconds=timeout),
+                    method="GET",
+                )
+                return url
+            except Exception as exc:
+                print(f"download_url EXCEPTION: {str(exc)}")
+                return None
+
+        # LOCAL MODE
         try:
-            bucket = self.gc.bucket(bucket_name)
-            blob = bucket.blob(blob_name)
-            url = blob.generate_signed_url(
-                version="v4",
-                expiration=datetime.timedelta(seconds=timeout),
+            return self._signer.generate_url(
+                path="/download",
                 method="GET",
+                bucket=bucket_name,
+                blob=blob_name,
+                timeout=timeout,
+                max_size=None,
             )
-            # Append the blob_name for the download client to be able to recover the file name
-            # url = f'{url}&filename={blob_name}'
-            return url
         except Exception as exc:
-            print(f"download_url EXCEPTION: {str(exc)}")
+            print(f"download_url (local) EXCEPTION: {str(exc)}")
             return None
 
     def urls(self, bucket_name, blob_name, timeout=24 * 3600, size=None):
         """
         Generates both upload and download signed URLs for a blob.
-        Args:
-            bucket_name (str): Name of the GCS bucket.
-            blob_name (str): Name of the blob.
-            timeout (int, optional): URL expiration time in seconds. Defaults to 24 hours.
-            size (int, optional): Maximum allowed size of the upload in bytes.
-        Returns:
-            tuple: (upload_url, download_url)
         """
-        return self.upload_url(
-            bucket_name, blob_name, timeout=timeout, size=size
-        ), self.download_url(bucket_name, blob_name, timeout=timeout)
+        return (
+            self.upload_url(bucket_name, blob_name, timeout=timeout, size=size),
+            self.download_url(bucket_name, blob_name, timeout=timeout),
+        )

{kardioutils-1.0.10 → kardioutils-1.0.11/kardioutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kardioutils
-Version: 1.0.10
+Version: 1.0.11
 Summary: Utils lib
 Author: João Neto
 Author-email: joao.filipe.neto@gmail.com

{kardioutils-1.0.10 → kardioutils-1.0.11}/kardioutils.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ dl2050utils/db.py
 dl2050utils/dbdf.py
 dl2050utils/dbutils.py
 dl2050utils/df.py
+dl2050utils/df_utils.py
 dl2050utils/env.py
 dl2050utils/etl.py
 dl2050utils/fdb.py

kardioutils-1.0.10/dl2050utils/__version__.py

@@ -1 +0,0 @@
-version = "1.0.10"