wpipe 0.0.1__py3-none-any.whl

wpipe/__init__.py

@@ -0,0 +1,5 @@
+from .api_client import APIClient
+from .log import new_logger
+from .pipe import Pipeline
+from .sqlite import Wsqlite
+from .ram import memory

wpipe/api_client/__init__.py

@@ -0,0 +1 @@
+from .api_client import APIClient

wpipe/api_client/api_client.py

@@ -0,0 +1,87 @@
+# pip install requests
+import requests
+import json
+
+
+class APIClient:
+    def __init__(self, base_url: str = None, token: str = None):
+        """
+        Inicializa el cliente API con la URL base y el token de autorización.
+        """
+        self.base_url = base_url
+        self.headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        }
+
+    def send_post(self, endpoint: str, data: dict):
+        """
+        Envía una solicitud POST a un endpoint específico.
+        """
+
+        if not self.base_url:
+            raise Exception("No 'base_url' defined.")
+
+        url = f"{self.base_url}{endpoint}"
+        try:
+            response = requests.post(url, headers=self.headers, data=json.dumps(data))
+            response.raise_for_status()  # Lanza una excepción si el código de estado indica un error
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            print(f"Error en la solicitud POST a {url}: {e}")
+            return None
+
+    def send_get(self, endpoint: str):
+        """
+        Envía una solicitud GET a un endpoint específico.
+        """
+
+        if not self.base_url:
+            raise Exception("No 'base_url' defined.")
+
+        url = f"{self.base_url}{endpoint}"
+        try:
+            response = requests.get(url, headers=self.headers)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            print(f"Error en la solicitud GET a {url}: {e}")
+            return None
+
+    # Métodos específicos para cada endpoint
+
+    def register_worker(self, data: dict):
+        return self.send_post("/matricula", data)
+
+    def healthcheck_worker(self, data: dict):
+        return self.send_post("/healthchecker", data)
+
+    def register_process(self, data: dict):
+        return self.send_post("/newprocess", data)
+
+    def end_process(self, data: dict):
+        return self.send_post("/endprocess", data)
+
+    def update_task(self, data: dict):
+        return self.send_post("/actualizar_task", data)
+
+    def get_dashboard_workers(self):
+        return self.send_get("/dashboard_workers")
+
+
+# Ejemplo de uso
+if __name__ == "__main__":
+    # Configura la URL base y el token de autorización
+    client = APIClient(base_url="http://192.168.1.60:8418", token="mysecrettoken")
+
+    # Ejemplo de cómo registrar un trabajador
+    data = {
+        # Llena con los datos necesarios para la solicitud
+    }
+
+    # Llama al método para obtener el dashboard de trabajadores
+    workers_info = client.get_dashboard_workers()
+    print(workers_info)
+
+
+# pip install requests

wpipe/log/__init__.py

@@ -0,0 +1 @@
+from .log import new_logger

wpipe/log/log.py

@@ -0,0 +1,31 @@
+"""
+This program has a functions that returns
+a logger object.
+"""
+
+import sys
+from loguru import logger
+
+
+def new_logger(process_name: str, path_file: str = "/logs/file_{time}.log"):
+
+    # for print in console
+    logger.add(
+        sys.stderr,
+        format="{time} {level} {message}",
+        filter=f"{process_name}",
+        level="INFO",
+    )
+
+    # for save in file
+    logger.add(
+        path_file,
+        colorize=True,
+        format="<green>{time:YYYY-MM-DD at HH:mm:ss}</green>"
+        + "| <level>{level}</level> | <blue>{message}</blue>",
+        rotation="50 MB",
+        compression="zip",
+        retention="10 days",
+    )
+
+    return logger

wpipe/pipe/__init__.py

@@ -0,0 +1 @@
+from .pipe import Pipeline

wpipe/pipe/pipe.py

@@ -0,0 +1,224 @@
+from wpipe.api_client.api_client import APIClient
+
+
+# Definición de la clase
+class Pipeline(APIClient):
+    worker_id: str = None
+    process_id: str = None
+    send_to_api: bool = False
+    api_config: dict = None
+    tasks_list: list = []  # Inicializa un atributo lista vacío
+
+    def __init__(
+        self, worker_id: str = None, api_config: dict = None, verbose: bool = False
+    ):
+        if api_config:
+            # invoca el constructor de APIClient
+            super().__init__(
+                base_url=api_config.get("base_url"), token=api_config.get("token")
+            )
+            self.api_config = api_config
+
+        if worker_id:
+            self.set_worker_id(worker_id)
+
+        self.verbose = verbose
+
+    def set_worker_id(self, id: str):
+        if id and isinstance(id, str):
+            if len(id) > 5:
+                if self.api_config and not self.worker_id:
+                    self.send_to_api = True
+                    print("[INFO] worker_id defined correct")
+
+                self.worker_id = id
+            else:
+                self.worker_id = None
+        else:
+            raise Exception(f"[ERROR] {id} is not correct, have to be a string")
+
+    def worker_register(self, name: str, version: str):
+        data = {
+            "name": name,
+            "version": version,
+            "tasks": [
+                {
+                    "name": name,
+                    "version": version,
+                }
+                for func, name, version, _id in self.tasks_list
+            ],
+        }
+
+        if self.api_config:
+            worker_id = self.register_worker(data)
+
+            if worker_id and "id" in worker_id:
+                self.set_worker_id(worker_id.get("id"))
+
+                return worker_id
+
+    def _decorator_task_report(func):
+        """Decorador para reportar la ejecución de cada tarea."""
+
+        def wrapper(self, *args, **kwargs):
+
+            if self.send_to_api:
+                task_updated = self.update_task(
+                    {"task_id": self.task_id, "status": "start"}
+                )
+                if self.verbose:
+                    print(
+                        "\t" * 2,
+                        f"[INFO] [START] task '{self.task_name}': {task_updated}",
+                    )
+
+            resultado = {}
+            try:
+                resultado = func(self, *args, **kwargs)
+
+                if self.send_to_api:
+                    task_updated = self.update_task(
+                        {"task_id": self.task_id, "status": "success"}
+                    )
+
+                    if not task_updated:
+                        raise Exception("Problem task")
+
+                    if self.verbose:
+                        print(
+                            "\t" * 2,
+                            f"[INFO] [END] task '{self.task_name}': {task_updated}",
+                        )
+            except Exception as e:
+                details = str(e)
+                resultado["error"] = details
+
+                if self.send_to_api:
+                    task_updated = self.update_task(
+                        {"task_id": self.task_id, "status": "error"}
+                    )
+                    print(
+                        "\t" * 2,
+                        f"[ERROR] [END] task '{self.task_name}': {task_updated}",
+                    )
+
+            return resultado
+
+        return wrapper
+
+    def _decorator_pipeline_report(func):
+        """Decorador para reportar la ejecución del pipeline completo."""
+
+        def wrapper(self, *args, **kwargs):
+
+            worker_id = self.worker_id
+
+            if self.verbose:
+                print("\n", "\t", "*" * 50)
+                print("\n", f"\t [WORKER] {self.worker_id}")
+                print("\n\t", "*" * 50)
+
+            if self.send_to_api:
+                process_registed = self.register_process({"id": worker_id})
+
+                if process_registed:
+                    self.tasks_list = [
+                        (rta[0], rta[1], rta[2], son["id"])
+                        for son, rta in zip(process_registed["sons"], self.tasks_list)
+                    ]
+
+                    self.process_id = process_registed["father"]
+
+                if self.verbose:
+                    print(
+                        "\t",
+                        f"[INFO] [START] pipeline: new process ({self.process_id})",
+                    )
+
+            resultado = {}
+            try:
+                resultado = func(self, *args, **kwargs)
+
+                if self.send_to_api:
+                    status = self.end_process({"id": self.process_id, "details": ""})
+                    if not status:
+                        raise Exception("API problem")
+
+                    if self.verbose:
+                        print("\t", f"[INFO] [END] pipeline: {status}")
+
+            except Exception as e:
+                details = str(e)
+                resultado["error"] = details
+
+                if self.send_to_api:
+                    status = self.end_process(
+                        {"id": self.process_id, "details": details}
+                    )
+                    print("\t", f"[ERROR] [END] pipeline: {status}")
+
+            return resultado
+
+        return wrapper
+
+    def set_steps(self, lista):
+        """
+        Define los pasos del pipeline, verificando que cada elemento sea una tupla
+        compuesta por una función y un nombre en string.
+        """
+
+        new_list = []
+
+        for item in lista:
+            if not (
+                isinstance(item, tuple)
+                and len(item) == 3
+                and callable(item[0])
+                and isinstance(item[1], str)
+            ):
+                raise ValueError(
+                    "Cada elemento de la lista debe ser una tupla (función, nombre)."
+                )
+            else:
+                new_list.append((item[0], item[1], item[2], ""))
+
+        self.tasks_list = new_list
+
+    @_decorator_task_report
+    def _task_invoke(self, func, name, *args, **kwargs):
+        return func(*args, **kwargs)
+
+    @_decorator_pipeline_report
+    def run(self, *args, **kwargs):
+        """
+        Ejecuta el pipeline completo, pasando los resultados de una función como
+        entrada a la siguiente.
+        """
+
+        resultado = None
+        data = {}
+        # TODO: poner un tqdm para ver como van avanzando el progreso de las tareas
+        for i, (func, name, version, _id) in enumerate(self.tasks_list):
+            self.task_name = name
+            self.task_id = _id
+
+            data.update(args[0])
+
+            if i == 0:
+                resultado = self._task_invoke(func, name, *(data,), **kwargs)
+            else:
+                resultado = self._task_invoke(func, name, *(data,), **kwargs)
+
+            data.update(resultado)
+
+            if self.verbose:
+                print()
+
+            if "error" in data:
+                break
+
+        if "error" in data:
+            raise Exception(f"[{self.task_name}] Fail the pipeline:{data['error']}")
+
+        return data

wpipe/ram/__init__.py

@@ -0,0 +1 @@
+from .ram import memory

wpipe/ram/ram.py

@@ -0,0 +1,52 @@
+import resource
+import platform
+import sys
+
+
+def memory_limit(percentage: float):
+    """
+    只在linux操作系统起作用
+    """
+    if platform.system() != "Linux":
+        print("Only works on linux!")
+        return
+    soft, hard = resource.getrlimit(resource.RLIMIT_AS)
+    resource.setrlimit(resource.RLIMIT_AS, (get_memory() * 1024 * percentage, hard))
+
+
+def get_memory():
+    with open("/proc/meminfo", "r") as mem:
+        free_memory = 0
+        for i in mem:
+            sline = i.split()
+            if str(sline[0]) in ("MemFree:", "Buffers:", "Cached:"):
+                free_memory += int(sline[1])
+    return free_memory
+
+
+"""
+        Este es el decorador
+        
+        example:
+
+        @memory(percentage=0.8)
+        def main():
+            print('My memory is limited to 80%.')
+"""
+
+
+def memory(percentage=0.8):
+    def decorator(function):
+        def wrapper(*args, **kwargs):
+            memory_limit(percentage)
+            try:
+                return function(*args, **kwargs)
+            except MemoryError:
+                mem = get_memory() / 1024 / 1024
+                print("Remain: %.2f GB" % mem)
+                sys.stderr.write("\n\nERROR: Memory Exception\n")
+                sys.exit(1)
+
+        return wrapper
+
+    return decorator

wpipe/sqlite/Sqlite.py

@@ -0,0 +1,207 @@
+import json
+import os
+import sqlite3
+from concurrent.futures import ThreadPoolExecutor
+from typing import Literal
+from datetime import datetime
+import pandas as pd
+
+
+class SQLite:
+    def __init__(self, db_name: str = "register.db"):
+        # nombre absoluto de db_name
+        # self.db_name = os.path.basename(db_name)
+        self.db_name = db_name
+        self._create_table_if_not_exists()
+
+        self.executor = ThreadPoolExecutor(max_workers=10)
+
+    def _create_table_if_not_exists(self):
+        if not self.db_name:
+            return
+
+        with sqlite3.connect(self.db_name) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """CREATE TABLE IF NOT EXISTS records
+                              (id INTEGER PRIMARY KEY AUTOINCREMENT,
+                               input TEXT,
+                               output TEXT,
+                               details TEXT DEFAULT NULL,
+                               datetime TEXT DEFAULT CURRENT_TIMESTAMP)"""
+            )
+
+    def async_write(
+        self, input: str = None, output: str = None, details: str = None, id: int = None
+    ):
+        self.executor.submit(self.write, input, output, details, id)
+
+    def write(
+        self,
+        input: str = None,
+        output: Literal["str", "dict"] = None,
+        details: str = None,
+        id: int = None,
+    ):
+        if not self.check_table_exists():
+            return
+
+        if isinstance(input, dict):
+            input = json.dumps(input)
+
+        if isinstance(output, dict):
+            output["datetime"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            output = json.dumps(output)
+
+        elif isinstance(output, str):
+            output = json.dumps(
+                {
+                    "output": output,
+                    "datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                }
+            )
+
+        if isinstance(details, dict):
+            details = json.dumps(details)
+
+        with sqlite3.connect(self.db_name) as conn:
+            cursor = conn.cursor()
+
+            if not id:
+                # insert
+                cursor.execute(
+                    "INSERT INTO records (input, output, details) VALUES (?, ?, ?)",
+                    (input, output, details),
+                )
+
+                id = cursor.lastrowid
+            else:
+                # update
+                if not self.read_by_id(id):
+                    return
+
+                if not input and not details and output:
+                    cursor.execute(
+                        "UPDATE records SET output = ? WHERE id = ?",
+                        (output, id),
+                    )
+                elif not input and output and details:
+                    cursor.execute(
+                        "UPDATE records SET output = ?, details = ? WHERE id = ?",
+                        (output, details, id),
+                    )
+                elif input and output and details:
+                    cursor.execute(
+                        "UPDATE records SET input = ?, output = ?, details = ? WHERE id = ?",
+                        (input, output, details, id),
+                    )
+                elif input and output and not details:
+                    cursor.execute(
+                        "UPDATE records SET input = ?, output = ? WHERE id = ?",
+                        (input, output, id),
+                    )
+            conn.commit()
+
+        return id
+
+    def read_by_id(self, id: int) -> list:
+        if not self.check_table_exists():
+            return []
+
+        results = []
+
+        with sqlite3.connect(self.db_name) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT * FROM records WHERE id = ?", (id,))
+
+            results = cursor.fetchall()
+
+            if cursor.rowcount == 0:
+                # print("No records found for the specified id")
+                pass
+
+        return results
+
+    def _view_records(self) -> pd.DataFrame:
+        with sqlite3.connect(self.db_name) as conn:
+            return pd.read_sql_query("SELECT * FROM records", conn)
+
+    def export_to_dataframe(
+        self, save_csv: bool = False, csv_name: str = "records.csv"
+    ) -> pd.DataFrame:
+        if not self.check_table_exists():
+            return pd.DataFrame()
+
+        df = self._view_records()
+
+        if save_csv:
+            df.to_csv(csv_name, index=False)
+
+        return df
+
+    def get_records_by_date_range(self, start_date: str, end_date: str) -> list:
+        if not self.check_table_exists():
+            return []
+
+        with sqlite3.connect(self.db_name) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT * FROM records WHERE datetime BETWEEN ? AND ?",
+                (start_date, end_date),
+            )
+            return cursor.fetchall()
+
+    def count_records(self) -> int:
+        if not self.check_table_exists():
+            return 0
+
+        with sqlite3.connect(self.db_name) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM records")
+            return cursor.fetchone()[0]
+
+    def delete_by_id(self, id_saved: int):
+        if not self.check_table_exists():
+            return
+
+        with sqlite3.connect(self.db_name) as conn:
+            cursor = conn.cursor()
+            cursor.execute("DELETE FROM records WHERE id = ?", (id_saved,))
+            conn.commit()
+
+    def check_table_exists(self) -> bool:
+        if not self.db_name:
+            return False
+
+        if len(os.path.dirname(self.db_name)) > 0:
+            if not os.path.exists(os.path.dirname(self.db_name)):
+                os.makedirs(os.path.dirname(self.db_name))
+
+        self._create_table_if_not_exists()
+
+        return True
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.executor:
+            self.executor.shutdown(wait=True)
+
+
+if __name__ == "__main__":
+    import uuid
+
+    my_uuid = str(uuid.uuid4().hex)
+
+    with SQLite() as registro:
+        registro.write(topic="topic 1", type_data="image", value=my_uuid)
+        registro.write(topic="topic 2", type_data="image", value=my_uuid)
+        registro.write(topic="topic 3", type_data="image", value=my_uuid)
+        registro.write(topic="topic 4", type_data="image", value=my_uuid)
+
+        print(registro.read_by_id(str(uuid.uuid4().hex)))
+
+        print(registro.export_to_dataframe().head())
+
+        print("total", registro.count_records())

wpipe/sqlite/Wsqlite.py

@@ -0,0 +1,54 @@
+from wpipe.sqlite.Sqlite import SQLite
+
+
+class Wsqlite:
+
+    id: str = None
+
+    output_db: dict = {}
+    details_db: dict = {}
+    input_db: dict = {}
+
+    def __init__(self, db_name: str = "register.db") -> None:
+        self.db_name = db_name
+
+    @property
+    def _input(self):
+        pass
+
+    @_input.setter
+    def input(self, input: dict):
+        self._create(input=input)
+
+    @property
+    def _output(self):
+        pass
+
+    @_output.setter
+    def output(self, output: dict):
+        self.output_db = output
+
+    @property
+    def _details(self):
+        pass
+
+    @_details.setter
+    def details(self, details: dict):
+        self.details_db = details
+
+    def _create(self, input: dict):
+        id = None
+        with SQLite(self.db_name) as conection_db:
+            id = conection_db.async_write(input=input)
+
+        self.id = str(id)
+
+    def _update(self, output: dict, details: dict = {}):
+        with SQLite(self.db_name) as conection_db:
+            conection_db.async_write(output=output, details=details, id=self.id)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._update(output=self.output_db, details=self.details_db)

wpipe/sqlite/__init__.py

@@ -0,0 +1 @@
+from .Wsqlite import Wsqlite

wpipe/util/__init__.py

@@ -0,0 +1 @@
+from .utils import leer_yaml, escribir_yaml