saviialib 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

saviialib/libs/directory_client/client/os_client.py

@@ -21,3 +21,7 @@ class OsClient(DirectoryClientContract):
     @staticmethod
     async def isdir(path: str) -> list:
         return await asyncio.to_thread(os.path.isdir, path)
+
+    @staticmethod
+    async def makedirs(path: str) -> None:
+        return await asyncio.to_thread(os.makedirs, path, exist_ok=True)

saviialib/libs/directory_client/directory_client.py

@@ -26,3 +26,6 @@ class DirectoryClient(DirectoryClientContract):
 
     async def isdir(self, path: str) -> bool:
         return await self.client_obj.isdir(path)
+
+    async def makedirs(self, path: str) -> None:
+        return await self.client_obj.makedirs(path)

saviialib/libs/directory_client/directory_client_contract.py

@@ -17,3 +17,7 @@ class DirectoryClientContract(ABC):
     @abstractmethod
     async def isdir(self, path) -> bool:
         pass
+
+    @abstractmethod
+    async def makedirs(self, path: str) -> None:
+        pass

saviialib/libs/files_client/clients/csv_client.py

@@ -0,0 +1,42 @@
+import csv
+from asyncio import to_thread
+
+from saviialib.libs.directory_client.directory_client import (
+    DirectoryClient,
+    DirectoryClientArgs,
+)
+from saviialib.libs.files_client.files_client_contract import FilesClientContract
+from saviialib.libs.files_client.types.files_client_types import (
+    FilesClientInitArgs,
+    ReadArgs,
+    WriteArgs,
+)
+
+
+class CsvClient(FilesClientContract):
+    def __init__(self, args: FilesClientInitArgs):
+        self.dir_client = DirectoryClient(DirectoryClientArgs(client_name="os_client"))
+
+    async def read(self, args: ReadArgs) -> str | bytes:
+        raise OSError("This method is not implemented yet.")
+
+    async def write(self, args: WriteArgs) -> None:
+        file_type = args.file_name.split(".")[-1]
+        file_content = args.file_content
+        header = file_content[0].keys()
+        if file_type == "tsv":
+            delimiter = "\t"
+        else:  # Default CSV.
+            delimiter = ","
+
+        if args.destination_path == "":
+            dest_path = self.dir_client.join_paths(args.file_name)
+        else:
+            dest_path = self.dir_client.join_paths(
+                args.destination_path, args.file_name
+            )
+
+        with open(dest_path, "w", newline="") as file:
+            writer = csv.DictWriter(file, fieldnames=header, delimiter=delimiter)  # type: ignore
+            await to_thread(writer.writeheader)
+            await to_thread(writer.writerows, file_content)  # type: ignore

saviialib/libs/files_client/files_client.py

@@ -1,10 +1,11 @@
 from .clients.aiofiles_client import AioFilesClient
+from .clients.csv_client import CsvClient
 from .files_client_contract import FilesClientContract
 from .types.files_client_types import FilesClientInitArgs, ReadArgs, WriteArgs
 
 
 class FilesClient(FilesClientContract):
-    CLIENTS = {"aiofiles_client"}
+    CLIENTS = {"aiofiles_client", "csv_client"}
 
     def __init__(self, args: FilesClientInitArgs) -> None:
         if args.client_name not in FilesClient.CLIENTS:
@@ -13,15 +14,12 @@ class FilesClient(FilesClientContract):
 
         if args.client_name == "aiofiles_client":
             self.client_obj = AioFilesClient(args)
+        elif args.client_name == "csv_client":
+            self.client_obj = CsvClient(args)
+
         self.client_name = args.client_name
 
     async def read(self, args: ReadArgs):
-        """Reads data from a specified source using the provided arguments.
-
-        :param args (ReadArgs): An object containing the parameters required for the read operation.
-
-        :return file: The result of the read operation, as returned by the client object.
-        """
         return await self.client_obj.read(args)
 
     async def write(self, args: WriteArgs):

saviialib/libs/files_client/types/files_client_types.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Literal
+from typing import Literal, Union, List, Dict
 
 
 @dataclass
@@ -27,6 +27,6 @@ class ReadArgs:
 @dataclass
 class WriteArgs:
     file_name: str
-    file_content: str | bytes
+    file_content: Union[str, bytes, List[Dict]]
     mode: Literal["w", "wb", "a"]
     destination_path: str = ""

saviialib/services/epii/use_cases/components/create_thies_statistics_file.py

@@ -0,0 +1,160 @@
+from .thies_bp import THIESDayData
+from typing import List
+from logging import Logger
+from asyncio import to_thread
+from saviialib.libs.directory_client import DirectoryClient
+from saviialib.libs.zero_dependency.utils.datetime_utils import datetime_to_str, today
+from saviialib.libs.files_client import FilesClient, FilesClientInitArgs, WriteArgs
+
+
+AVG_COLUMNS = {
+    "Date": "date",
+    "Time": "time",
+    "AirTemperature": "air_temperature",
+    "Radiation": "radiation",
+    "CO2": "carbon_dioxide",
+    "Precipitation": "precipitation",
+    "WS": "wind_velocity",
+    "WD": "wind_direction",
+    "Humidity": "humidity",
+}
+
+EXT_COLUMNS = {
+    "Date": "date",
+    "Time": "time",
+    "AirTemperature MIN": "air_temperature",
+    "AirTemperature MAX": "air_temperature",
+    "Radiation MIN": "radiation",
+    "Radiation MAX": "radiation",
+    "CO2 MIN": "carbon_dioxide",
+    "CO2 MAX": "carbon_dioxide",
+    "WS MIN": "wind_velocity",
+    "WS MAX gust": "wind_velocity",
+    "WD MIN": "wind_direction",
+    "WD MAX gust": "wind_direction",
+    "Humidity MIN": "humidity",
+    "Humidity MAX": "humidity",
+}
+
+AGG_DICT = {
+    "AirTemperature": "mean",
+    "AirTemperature MIN": "mean",
+    "AirTemperature MAX": "mean",
+    "Precipitation": "sum",
+    "Humidity": "mean",
+    "Humidity MIN": "mean",
+    "Humidity MAX": "mean",
+    "Radiation": "sum",
+    "Radiation MIN": "sum",
+    "Radiation MAX": "sum",
+    "CO2": "sum",
+    "CO2 MIN": "sum",
+    "CO2 MAX": "sum",
+    "WS": "mean",
+    "WS MIN": "mean",
+    "WS MAX gust": "mean",
+    "WD": "mean",
+    "WD MIN": "mean",
+    "WD MAX gust": "mean",
+}
+
+UNITS = {
+    "AirTemperature": "°C",
+    "Precipitation": "mm",
+    "Humidity": "%",
+    "Radiation": "W/m2",
+    "CO2": "ppm",
+    "WS": "m/s",
+    "WD": "°",
+}
+
+
+async def create_thies_daily_statistics_file(
+    os_client: DirectoryClient, logger: Logger, daily_files: List[str]
+) -> None:
+    logger.debug("[thies_synchronization_lib] Creating Daily Statistics ...")
+    csv_client = FilesClient(FilesClientInitArgs(client_name="csv_client"))
+    filename = datetime_to_str(today(), date_format="%Y%m%d") + ".BIN"
+    path_bin_av = os_client.join_paths("thies-daily-files", "ARCH_AV1", filename)
+    path_ini_av = os_client.join_paths("thies-daily-files", "ARCH_AV1", "DESCFILE.INI")
+    path_bin_ex = os_client.join_paths("thies-daily-files", "ARCH_EX1", filename)
+    path_ini_ex = os_client.join_paths("thies-daily-files", "ARCH_EX1", "DESCFILE.INI")
+
+    ext_df = THIESDayData("ex")
+    await to_thread(ext_df.read_binfile, path_bin_ex, path_ini_ex)
+
+    avg_df = THIESDayData("av")
+    await to_thread(avg_df.read_binfile, path_bin_av, path_ini_av)
+
+    ext_df = ext_df.dataDF[EXT_COLUMNS.keys()]
+    avg_df = avg_df.dataDF[AVG_COLUMNS.keys()]
+
+    # Merge both dataframes
+    df = avg_df.merge(ext_df, on=["Date", "Time"], how="outer")
+    # Set the date as dd.mm.yyyy format.
+    df["Date"] = df["Date"].str.replace(
+        r"(\d{4})/(\d{2})/(\d{2})", r"\3.\2.\1", regex=True
+    )
+    df["Hour"] = df["Time"].str[:2]
+
+    # Group by hour.
+    hourly_agg = df.groupby(["Date", "Hour"]).agg(AGG_DICT).reset_index()
+
+    rows = []
+    # For each attribute in avg_columns (except Date, Time)
+    for col, col_id in AVG_COLUMNS.items():
+        if col in ["Date", "Time"]:
+            continue
+        # Determine the corresponding min/max columns if they exist
+        min_col = f"{col} MIN"
+        max_col = f"{col} MAX"
+        mean_col = col
+        if col in ["WS", "WD"]:
+            max_col += " gust"
+
+        unit = UNITS.get(col, "")
+
+        for idx, row in hourly_agg.iterrows():
+            statistic_id = f"sensor.saviia_epii_{col_id}"
+            start = f"{row['Date']} {row['Hour']}:00"
+            mean = row[mean_col] if mean_col in row else 0
+            min_val = row[min_col] if min_col in row else 0
+            max_val = row[max_col] if max_col in row else 0
+
+            # If no min/max for this attribute is 0
+            if min_col not in row:
+                min_val = 0
+            if max_col not in row:
+                max_val = 0
+
+            if col in ["WD"]:  # Avoid error
+                rows.append(
+                    {
+                        "statistic_id": statistic_id,
+                        "unit": unit,
+                        "start": start,
+                        "min": mean,
+                        "max": mean,
+                        "mean": mean,
+                    }
+                )
+            else:
+                rows.append(
+                    {
+                        "statistic_id": statistic_id,
+                        "unit": unit,
+                        "start": start,
+                        "min": min_val,
+                        "max": max_val,
+                        "mean": mean,
+                    }
+                )
+
+    logger.debug("[thies_synchronization_lib] Saving file in /config folder ...")
+    logger.debug(rows[0].keys())
+    await csv_client.write(
+        WriteArgs(file_name="thies_daily_statistics.tsv", file_content=rows, mode="w")
+    )
+    logger.debug(
+        "[thies_synchronization_lib] thies_daily_statistics.tsv created successfully!"
+    )

saviialib/services/epii/use_cases/components/thies_bp.py

@@ -0,0 +1,442 @@
+import configparser
+import os
+import struct
+from datetime import datetime, timedelta
+
+import numpy as np
+import pandas as pd
+from bitarray import bitarray
+
+TIME_LIST = []
+
+start_time = datetime.strptime("00:00", "%H:%M")
+for i in range(0, 24 * 60, 10):
+    TIME_LIST.append((start_time + timedelta(minutes=i)).strftime("%H:%M"))
+
+ROWS = len(TIME_LIST)
+
+
+def date_range(start_date: str, end_date: str) -> list:
+    start = datetime.strptime(start_date, "%Y/%m/%d") + timedelta(days=1)
+    end = datetime.strptime(end_date, "%Y/%m/%d") - timedelta(days=1)
+    return [
+        (start + timedelta(days=i)).strftime("%Y/%m/%d")
+        for i in range((end - start).days + 1)
+        if start <= end
+    ]
+
+
+def add_date_sep(date: str) -> str:
+    """
+    Input: date as YYYYMMDD.BIN
+    Returns: date as YYYY/MM/DD
+    """
+    return date[:4] + "/" + date[4:6] + "/" + date[6:8]
+
+
+def verify_datestr(filename: str) -> bool:
+    """
+    Returns True if filename has the YYYYMMDD.BIN format
+    """
+    try:
+        datetime.strptime(filename[:8], "%Y%m%d")
+        return filename.endswith(".BIN")
+    except ValueError:
+        return False
+
+
+def read_descfile(path) -> dict:
+    """
+    Input: path DESCFILE.INI
+    Returns: dict
+        key is index [i]
+        value is dict with parameters from .ini
+    """
+    if type(path) == dict:  # noqa: E721
+        return path
+    config = configparser.ConfigParser()
+    config.read(path)
+    data_dict = {}
+    for section in config.sections():
+        section_dict = dict(config.items(section))
+        for v in section_dict:
+            if v == "name":
+                continue
+            section_dict[v] = int(section_dict[v])
+        data_dict[int(section)] = section_dict
+    return data_dict
+
+
+class THIESDayData:
+    # Bytes per parameter
+    BPP = {"av": 5, "ex": 9}
+    # Timestamp Offset
+    OFFSET = 4
+
+    def __init__(self, datatype: str) -> None:
+        d = datatype.lower().strip()
+        if d not in ["av", "ex"]:
+            raise ValueError(
+                "Invalid datatype. Expected 'av' (average values) or 'ex' (minmax values)."
+            )
+
+        self._bpr = -1  # Bytes per row
+        self._bpp = THIESDayData.BPP[d]  # Bytes per parameter
+        self._datatype = d
+        self._binfile = None
+        self.descfile = {}
+        self.nparameters = -1
+        self._parameters = []
+        self.nbytes = -1
+        self.nrows = -1
+        self._date = ""
+        self.statusDF = pd.DataFrame()
+        self.dataDF = pd.DataFrame()
+        self.datesDF = pd.DataFrame()
+
+    @staticmethod
+    def _bytes2datetime(b: bytes, only_time: bool = False) -> str:
+        """
+        Input: bytes (size 4)
+        Output: str (YYYY/MM/DD hh:mm:ss)
+        """
+        bits = bitarray()
+        bits.frombytes(b[::-1])  # Invert 4 bytes
+        hr = int(bits[15:20].to01(), 2)
+        min = int(bits[20:26].to01(), 2)
+        sec = int(bits[26:].to01(), 2)
+        time = f"{str(hr).zfill(2)}:{str(min).zfill(2)}"
+        if only_time:
+            return time
+        yr = int(bits[0:6].to01(), 2)
+        mon = int(bits[6:10].to01(), 2)
+        day = int(bits[10:15].to01(), 2)
+        date = f"20{yr}/{str(mon).zfill(2)}/{str(day).zfill(2)}"
+        return date + " " + time + f":{str(sec).zfill(2)}"
+
+    def _set_descfile(self, inipath: str) -> None:
+        self.descfile = read_descfile(inipath)
+        self.nparameters = len(self.descfile)
+        row_size = sum([self.descfile[num]["size"] for num in self.descfile])
+        self._bpr = row_size + THIESDayData.OFFSET
+
+    def read_binfile(self, binpath: str, inipath: str) -> None:
+        self._set_descfile(inipath)
+        with open(binpath, "rb") as bin_file:
+            binfile = bin_file.read()
+        self._binfile = binfile
+        self.nbytes = len(self._binfile)
+        self.nrows = int(self.nbytes / self._bpr)
+        self._make_dataframes()
+
+    def make_empty(self, inipath: str, date: str) -> None:
+        self._set_descfile(inipath)
+        dataDF = pd.DataFrame(
+            None, index=range(ROWS), columns=range(self.nparameters + 2)
+        )
+        col_names = {0: "Date", 1: "Time"}
+        par_names = {key + 1: self.descfile[key]["name"] for key in self.descfile}
+        col_names.update(par_names)
+        dataDF = dataDF.rename(columns=col_names)
+        dataDF["Time"] = TIME_LIST
+        dataDF["Date"] = [date] * ROWS
+
+        self.dataDF = dataDF
+        self.statusDF = dataDF
+        self.datesDF = dataDF
+
+    def _make_dataframes(self) -> None:
+        """
+        Builds data DF, status DF and, if datatype=ex, dates DF.
+        """
+        byterows = [
+            self._binfile[i * self._bpr + THIESDayData.OFFSET : (i + 1) * self._bpr]
+            for i in range(0, self.nrows)
+        ]
+        data_arr = np.zeros((self.nrows, self.nparameters))
+        status_arr = np.zeros((self.nrows, self.nparameters))
+        time_idx = np.empty(self.nrows, dtype=object)
+        date_idx = np.empty(self.nrows, dtype=object)
+        dates_arr = np.empty((self.nrows, self.nparameters), dtype=object)
+
+        for i, row in enumerate(byterows):
+            # Timestamp
+            ts_bytes = self._binfile[i * self._bpr : i * self._bpr + 4]
+            ts = THIESDayData._bytes2datetime(ts_bytes)
+            date_idx[i], time_idx[i] = ts[:-3].split()
+
+            for j in range(self.nparameters):
+                # Status = byte 1
+                status = row[j * self._bpp]
+                status_arr[i, j] = status
+
+                # Value = bytes 2-5, float
+                value = struct.unpack("<f", row[j * self._bpp + 1 : j * self._bpp + 5])[
+                    0
+                ]
+                data_arr[i, j] = round(value, 1)
+
+                if self._datatype == "ex":
+                    # Datetime = bytes 6-9
+                    dt = THIESDayData._bytes2datetime(
+                        row[j * self._bpp + 5 : j * self._bpp + 9], only_time=True
+                    )
+                    dates_arr[i, j] = dt
+
+        self.dataDF = pd.DataFrame(data_arr).rename(
+            columns={i: self.descfile[i + 1]["name"] for i in range(self.nparameters)}
+        )
+        self.statusDF = pd.DataFrame(status_arr).rename(
+            columns={i: self.descfile[i + 1]["name"] for i in range(self.nparameters)}
+        )
+        self.dataDF = self.dataDF.where(self.statusDF == 0.0, other=None)
+
+        if self._datatype == "ex":
+            self.datesDF = pd.DataFrame(dates_arr).rename(
+                columns={
+                    i: self.descfile[i + 1]["name"] for i in range(self.nparameters)
+                }
+            )
+            self.datesDF = self.datesDF.where(self.statusDF == 0.0, other=None)
+            self.datesDF.insert(0, "Time", time_idx)
+            self.datesDF.insert(0, "Date", date_idx)
+
+        self.dataDF.insert(0, "Time", time_idx)
+        self.dataDF.insert(0, "Date", date_idx)
+        self.statusDF.insert(0, "Time", time_idx)
+        self.statusDF.insert(0, "Date", date_idx)
+
+    def _generate_blank_rows(self) -> pd.DataFrame:
+        if len(self) == ROWS:
+            # Nothing to fill (already full rows)
+            return []
+
+        new = []
+        none_row = {col: None for col in self.dataDF.columns}
+        none_row["Date"] = self.date
+        current_times = self.dataDF["Time"]
+        for time in TIME_LIST:
+            if time not in current_times.values:
+                row = none_row.copy()
+                # 'time' was not measured in the original data
+                # fill it with None row
+                row["Time"] = time
+                new.append(row)
+        return pd.DataFrame(new)
+
+    def complete_empty(self):
+        """
+        Completes DataFrames with all the timestamps of missing data
+        Fills all columns with 'None' except Date and Time cols
+        """
+        if len(self) == ROWS:
+            return
+        new_rows = self._generate_blank_rows()
+        # self.dataDF = self.dataDF.append(new_rows, ignore_index=True)
+        self.dataDF = pd.concat([self.dataDF, new_rows], ignore_index=True)
+        self.dataDF = self.dataDF.sort_values(by="Time").reset_index(drop=True)
+        # self.statusDF = self.statusDF.append(new_rows, ignore_index=True)
+        self.statusDF = pd.concat([self.statusDF, new_rows], ignore_index=True)
+        self.statusDF = self.statusDF.sort_values(by="Time").reset_index(drop=True)
+
+        if self._datatype == "ex":
+            # self.datesDF = self.datesDF.append(new_rows, ignore_index=True)
+            self.datesDF = pd.concat([self.datesDF, new_rows], ignore_index=True)
+            self.datesDF = self.datesDF.sort_values(by="Time").reset_index(drop=True)
+
+    def sort_by(self, cols: list):
+        self.dataDF = self.dataDF.sort_values(
+            by=cols, ascending=[True, True]
+        ).reset_index(drop=True)
+        self.statusDF = self.statusDF.sort_values(
+            by=cols, ascending=[True, True]
+        ).reset_index(drop=True)
+        if len(self.datesDF):
+            self.datesDF = self.datesDF.sort_values(
+                by=cols, ascending=[True, True]
+            ).reset_index(drop=True)
+
+    @property
+    def date(self) -> str:
+        """
+        Returns str of date of measurement
+        """
+        if len(self.dataDF) and self._date == "":
+            self._date = self.dataDF["Date"][0]
+        return self._date
+
+    @property
+    def shape(self):
+        return self.dataDF.shape
+
+    @property
+    def info(self) -> None:
+        bf = self._binfile
+        if bf:
+            bf = bf[:8]
+        print(f"""=== THIES Day Data Instance ===\n
+Bytes per row (BPR): {self._bpr}
+Bytes per parameter (BPP): {self._bpp}
+Datatype: {self._datatype}
+Binfile: {bf}...
+Descfile: {self.descfile}
+N parameters: {self.nparameters}
+N Bytes: {self.nbytes}
+Rows: {self.nrows}
+Date: {self.date}
+    """)
+
+    @property
+    def parameters(self) -> list:
+        if self._parameters == []:
+            self._parameters = [self.descfile[i]["name"] for i in self.descfile]
+        return self._parameters
+
+    def write_csv(self, filename: str) -> None:
+        with open(filename + ".csv", "w") as outfile:
+            outfile.write(self.dataDF.to_csv())
+
+    def __repr__(self) -> str:
+        return str(self.dataDF)
+
+    def _repr_html_(self):
+        return self.dataDF._repr_html_()
+
+    def __len__(self):
+        return len(self.dataDF)
+
+    def __add__(self, other):
+        if isinstance(other, THIESDayData):
+            new = THIESDayData(datatype=self._datatype)
+            new.descfile = other.descfile
+            new.nparameters = other.nparameters
+            new._parameters = other.parameters
+            new.nrows = self.nrows + other.nrows
+            new.nbytes = self.nbytes + other.nbytes
+            new.statusDF = pd.concat([self.statusDF, other.statusDF]).reset_index(
+                drop=True
+            )
+            new.dataDF = pd.concat([self.dataDF, other.dataDF]).reset_index(drop=True)
+            if self._datatype == "ex":
+                new.datesDF = pd.concat([self.datesDF, other.datesDF]).reset_index(
+                    drop=True
+                )
+            return new
+        raise TypeError(
+            f"unsupported operand type(s) for +: 'THIESDayData' and '{type(other)}'"
+        )
+
+
+class THIESData:
+    def __init__(self, datatype: str, dirpath: str) -> None:
+        d = datatype.lower().strip()
+        if d not in ["av", "ex"]:
+            raise ValueError(
+                "Invalid datatype. Expected 'av' (average values) or 'ex' (minmax values)."
+            )
+
+        self._path = dirpath
+        self._datatype = d
+        self.filelist = []
+
+        self._verify_path(dirpath)
+        descpath = self._path + "/DESCFILE.INI"
+        self.descfile = read_descfile(descpath)
+
+        self.daylist = []
+        self.fullData = pd.DataFrame()
+
+        self.completed = False
+
+    def reset(self):
+        self.daylist = []
+        self.fullData = pd.DataFrame()
+        self.completed = False
+
+    def _verify_path(self, path: str) -> None:
+        fl = sorted(os.listdir(path))
+        if "DESCFILE.INI" not in fl:
+            raise FileNotFoundError("No DESCFILE.INI found in this directory.")
+        self.filelist = [file for file in fl if verify_datestr(file)]
+
+    def load_df(self, complete_rows=False) -> pd.DataFrame:
+        """Reads folder given in DIRPATH and
+        transforms data into DF. Saves it in self.fullData
+        - complete_rows (bool): if True, completes DFs with Empty Rows by calling
+            THIESDayData.complete_empty()
+        """
+        self.reset()
+        for f in self.filelist:
+            filepath = f"{self._path}/{f}"
+            daydata = THIESDayData(datatype=self._datatype)
+            daydata.read_binfile(binpath=filepath, inipath=self.descfile)
+            if complete_rows:
+                daydata.complete_empty()
+            self.daylist.append(daydata)
+
+        self.fullData = sum(self.daylist, start=THIESDayData(self._datatype))
+
+        return self.fullData
+
+    def complete_empty_dates(self):
+        if self.completed:
+            return
+        date_s = add_date_sep(self.filelist[0])
+        date_e = add_date_sep(self.filelist[-1])
+        d_range = date_range(date_s, date_e)
+        for date in d_range:
+            if date not in self.fullData.dataDF["Date"].values:
+                # Missing day
+                new = THIESDayData(self._datatype)
+                new.make_empty(self.descfile, date=date)
+                self.fullData += new
+
+        self.fullData.sort_by(["Date", "Time"])
+        self.completed = True
+
+    def df2csv(self, outpath: str) -> None:
+        # if self._datatype == 'av':
+        # FORMAT FOR EX FILES ???
+        self.fullData.write_csv(outpath)
+        print(f"Data written in: {outpath}.csv")
+
+    def read_write(self, outpath: str):
+        """Quick version of the read-write process.
+        Reads the path given and writes all BIN file data in same CSV
+        Does NOT save as DF the data.
+        Does NOT complete missing timestamps with empty rows.
+        """
+        write_header = True
+        bcount = 0
+        with open(outpath + ".csv", "w") as outfile:
+            for i, f in enumerate(self.filelist):
+                filepath = f"{self._path}/{f}"
+                daydata = THIESDayData(datatype=self._datatype)
+                daydata.read_binfile(binpath=filepath, inipath=self.descfile)
+                outfile.write(daydata.dataDF.to_csv(header=write_header))
+                bcount += daydata.nbytes
+                if i == 0:
+                    write_header = False
+        print(f"Data written in: {outpath}.csv")
+
+    @property
+    def dataDF(self):
+        return self.fullData.dataDF
+
+    @property
+    def shape(self):
+        return self.fullData.shape
+
+    @property
+    def size(self):
+        return len(self.filelist)
+
+    @property
+    def parameters(self):
+        return self.fullData.parameters
+
+    def __repr__(self) -> str:
+        return str(self.fullData)
+
+    def _repr_html_(self):
+        return self.fullData

saviialib/services/epii/use_cases/update_thies_data.py

@@ -23,6 +23,9 @@ from saviialib.libs.sharepoint_client import (
     SpListFoldersArgs,
     SpUploadFileArgs,
 )
+from saviialib.libs.directory_client import DirectoryClient, DirectoryClientArgs
+
+from saviialib.libs.files_client import FilesClient, FilesClientInitArgs, WriteArgs
 from saviialib.services.epii.use_cases.types import (
     FtpClientConfig,
     SharepointConfig,
@@ -31,6 +34,8 @@ from saviialib.services.epii.use_cases.types import (
 from saviialib.services.epii.utils import (
     parse_execute_response,
 )
+from saviialib.libs.zero_dependency.utils.datetime_utils import today, datetime_to_str
+from .components.create_thies_statistics_file import create_thies_daily_statistics_file
 
 
 class UpdateThiesDataUseCase:
@@ -44,6 +49,8 @@ class UpdateThiesDataUseCase:
         self.ftp_server_folders_path = input.ftp_server_folders_path
         self.sharepoint_base_url = f"/sites/{self.sharepoint_client.site_name}"
         self.uploading = set()
+        self.os_client = self._initialize_os_client()
+        self.files_client = self._initialize_files_client()
 
     def _initialize_sharepoint_client(
         self, config: SharepointConfig
@@ -63,6 +70,12 @@ class UpdateThiesDataUseCase:
         except RuntimeError as error:
             raise FtpClientError(error)
 
+    def _initialize_os_client(self) -> DirectoryClient:
+        return DirectoryClient(DirectoryClientArgs(client_name="os_client"))
+
+    def _initialize_files_client(self) -> FilesClient:
+        return FilesClient(FilesClientInitArgs(client_name="aiofiles_client"))
+
     async def _validate_sharepoint_current_folders(self):
         async with self.sharepoint_client:
             folder_base_path = "/".join(
@@ -204,6 +217,98 @@ class UpdateThiesDataUseCase:
 
         return upload_results
 
+    async def _sync_pending_files(self, thies_files: set, cloud_files: set) -> set:
+        uploading = thies_files - cloud_files
+
+        # Update content of the daily files
+        daily_files = {
+            prefix + datetime_to_str(today(), date_format="%Y%m%d") + ".BIN"
+            for prefix in ["EXT_", "AVG_"]
+        }
+        for file in daily_files:
+            if file in thies_files:
+                uploading.add(file)
+
+        return uploading
+
+    async def _extract_thies_daily_statistics(self) -> None:
+        # Create the folder thies-daily-files if doesnt exists
+        self.logger.info("[thies_synchronization_lib] Creating Daily files directory")
+        base_folder = "thies-daily-files"
+        if not await self.os_client.isdir(base_folder):
+            for dest_folder in {"ARCH_AV1", "ARCH_EX1"}:
+                await self.os_client.makedirs(
+                    self.os_client.join_paths(base_folder, dest_folder)
+                )
+        else:
+            self.logger.info(
+                "[thies_synchronization_lib] Thies daily files already exists"
+            )
+
+        # Read the daily files and save each data in the folder
+        daily_files = [
+            prefix + datetime_to_str(today(), date_format="%Y%m%d") + ".BIN"
+            for prefix in ["AVG_", "EXT_"]
+        ]
+        # Receive from FTP server and  write the file in thies-daily-files
+        for file in daily_files:
+            prefix, filename = file.split("_", 1)
+            # The first path is for AVG files. The second file is for EXT files
+            folder_path = next(
+                (
+                    path
+                    for path in self.ftp_server_folders_path
+                    if prefix == ("AVG" if "AV" in path else "EXT")
+                ),
+                self.ftp_server_folders_path[0],  # Default to the first path
+            )
+            # Retrieve the AVG or EXT file
+            file_path = f"{folder_path}/{filename}"
+            try:
+                content = await self.thies_ftp_client.read_file(
+                    FtpReadFileArgs(file_path)
+                )
+            except FileNotFoundError as error:
+                raise ThiesFetchingError(reason=str(error))
+            # Destination local folder
+            self.logger.debug(file_path)
+
+            dest_folder = "ARCH_AV1" if prefix == "AVG" else "ARCH_EX1"
+            await self.files_client.write(
+                WriteArgs(
+                    file_name=filename,
+                    file_content=content,
+                    mode="wb",
+                    destination_path=f"{base_folder}/{dest_folder}",
+                )
+            )
+            # Retrieve the DESCFILE and save if is not in the base folder
+            descfile_name = "DESCFILE.INI"
+            if not await self.os_client.path_exists(
+                self.os_client.join_paths(base_folder, dest_folder, descfile_name)
+            ):
+                descfile_path = f"{folder_path}/{descfile_name}"
+                descfile_content = await self.thies_ftp_client.read_file(
+                    FtpReadFileArgs(descfile_path)
+                )
+                await self.files_client.write(
+                    WriteArgs(
+                        file_name=descfile_name,
+                        file_content=descfile_content,
+                        mode="wb",
+                        destination_path=f"{base_folder}/{dest_folder}",
+                    )
+                )
+            else:
+                self.logger.debug(
+                    "[thies_synchronization_lib] DESCFILE.INI already exists in %s",
+                    dest_folder,
+                )
+        # Read the files with ThiesDayData class
+        await create_thies_daily_statistics_file(
+            self.os_client, self.logger, daily_files
+        )
+
     async def execute(self):
         """Synchronize data from the THIES Center to the cloud."""
         self.logger.debug("[thies_synchronization_lib] Starting ...")
@@ -223,7 +328,10 @@ class UpdateThiesDataUseCase:
             "[thies_synchronization_lib] Total files fetched from Sharepoint: %s",
             str(len(cloud_files)),
         )
-        self.uploading = thies_files - cloud_files
+        self.uploading = await self._sync_pending_files(thies_files, cloud_files)
+        # Extract thies statistics for SAVIIA Sensors
+        await self._extract_thies_daily_statistics()
+
         if not self.uploading:
             raise EmptyDataError(reason="No files to upload.")
         # Fetch the content of the files to be uploaded from THIES FTP Server
@@ -233,7 +341,9 @@ class UpdateThiesDataUseCase:
         upload_statistics = await self.upload_thies_files_to_sharepoint(
             thies_fetched_files
         )
+        self.logger.info(upload_statistics)
         self.logger.debug(
             "[thies_synchronization_lib] All the files were uploaded successfully 🎉"
         )
+
         return parse_execute_response(thies_fetched_files, upload_statistics)  # type: ignore

{saviialib-1.0.1.dist-info → saviialib-1.2.0.dist-info}/METADATA

@@ -1,21 +1,23 @@
 Metadata-Version: 2.3
 Name: saviialib
-Version: 1.0.1
+Version: 1.2.0
 Summary: A client library for IoT projects in the RCER initiative
 License: MIT
 Author: pedropablozavalat
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.11,<4.0
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiofiles
 Requires-Dist: aioftp
 Requires-Dist: aiohttp
+Requires-Dist: bitarray (>=1.9.2,<2.0.0)
 Requires-Dist: build
 Requires-Dist: dotenv (>=0.9.9,<0.10.0)
+Requires-Dist: numpy (>=2.3.0,<3.0.0)
+Requires-Dist: pandas (>=2.3.0,<3.0.0)
 Requires-Dist: pytest-cov (>=6.1.1,<7.0.0)
 Description-Content-Type: text/markdown
 

{saviialib-1.0.1.dist-info → saviialib-1.2.0.dist-info}/RECORD

@@ -8,15 +8,16 @@ saviialib/general_types/error_types/api/epii_api_error_types.py,sha256=iprpNvVoN
 saviialib/general_types/error_types/common/__init__.py,sha256=yOBLZbt64Ki9Q0IJ0tMAubgq7PtrQ7XQ3RgtAzyOjiE,170
 saviialib/general_types/error_types/common/common_types.py,sha256=n5yuw-gVtkrtNfmaZ83ZkYxYHGl4jynOLUB9C8Tr32w,474
 saviialib/libs/directory_client/__init__.py,sha256=ys07nzp74fHew2mUkbGpntp5w4t_PnhZIS6D4_mJw2A,162
-saviialib/libs/directory_client/client/os_client.py,sha256=cL1ocxfLu6csBg7RKhP7sbpgCUr6_Me5cMOs4wrXmhw,628
-saviialib/libs/directory_client/directory_client.py,sha256=mcYA2AY0f44ae39ImsetVRU8YW7xI25xTFTY5HmWivY,984
-saviialib/libs/directory_client/directory_client_contract.py,sha256=eRTKPkpArMSi7gEDoowJ15hupLB_gr4ztO1Zc1BoXpc,396
+saviialib/libs/directory_client/client/os_client.py,sha256=TgDnIQvgMP7sIVjn5kAKEJ5BYET7pN-FIQP0lvFMl4Q,763
+saviialib/libs/directory_client/directory_client.py,sha256=5HmB1Pt9M8Tk18WKzfh3Fwr8gKa_RZCdAqvbU0rg7mw,1086
+saviialib/libs/directory_client/directory_client_contract.py,sha256=wtNaUak1a7r6t9OI1L2ou7XMMJFXWpQFb7WyT6X7fCQ,479
 saviialib/libs/directory_client/types/directory_client_types.py,sha256=ncMwVs_o6EYMuypXXmVInsjVDKJsdxVkmwj1M-LEInA,109
 saviialib/libs/files_client/__init__.py,sha256=sIi9ne7Z3EfxnqGTaSmH-cZ8QsKyu0hoOz61GyA3njs,192
 saviialib/libs/files_client/clients/aiofiles_client.py,sha256=Mu5pSnnEa3dT3GONmt1O-lCQstuQNXHtJHM3D2L6TU8,1107
-saviialib/libs/files_client/files_client.py,sha256=eoByr7ZzOJ3b4NWlrEH0jKO1i-Rv3hSJV2s2JLhaOJ0,1081
+saviialib/libs/files_client/clients/csv_client.py,sha256=Rk4QbiKlVKrYxYtxQt-Pmkp9QoB_dNgs5r36JB9hU0o,1478
+saviialib/libs/files_client/files_client.py,sha256=oWfaMLA_Lw91H1_pPirFtFbdJh4abSyZp91qBsAiePs,950
 saviialib/libs/files_client/files_client_contract.py,sha256=fYvd68IMpc1OFkxbzNSmRUoTWVctf3hkNVQ7QZV0m6I,304
-saviialib/libs/files_client/types/files_client_types.py,sha256=8OHm4nvZTW9K3ryeIUvthE_Cj7wF4zVcqG0KCi_vQIU,718
+saviialib/libs/files_client/types/files_client_types.py,sha256=GOTM5fBY304-EX6IiUnWmt2JpsyQ6Nmb_Tp0wingt0M,755
 saviialib/libs/ftp_client/__init__.py,sha256=dW2Yutgc7mJJJzgKLhWKXMgQ6KIWJYfFa1sGpjHH5xU,191
 saviialib/libs/ftp_client/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 saviialib/libs/ftp_client/clients/aioftp_client.py,sha256=5sgr3PMETgaBRlKeu_avxHIh6tr1gb7t2mkxmpRas_k,1925
@@ -39,16 +40,18 @@ saviialib/services/epii/controllers/types/update_thies_data_types.py,sha256=kmt1
 saviialib/services/epii/controllers/types/upload_backup_to_sharepoint_types.py,sha256=5mDMLy3J5grACl1ezBGYZPOyhIkWYeIWWBOklaP2IlQ,471
 saviialib/services/epii/controllers/update_thies_data.py,sha256=8TV420d4VWTvCW18M6sAguHZrX5VODXscgrUfhwO1fs,4884
 saviialib/services/epii/controllers/upload_backup_to_sharepoint.py,sha256=7-ABo89yXhrWv-iOFjCkBGX-y_I62eFaOi1LtO5Gb4k,3817
+saviialib/services/epii/use_cases/components/create_thies_statistics_file.py,sha256=G9K5UpsUYPM_w2sbusiyRWnZwbumSh5rYgil-rt8EB8,5171
+saviialib/services/epii/use_cases/components/thies_bp.py,sha256=1Iq5Wz3kqzJfQNawV_v8ORr_Pl-PoamFp_2Xo8DjvmI,15042
 saviialib/services/epii/use_cases/constants/upload_backup_to_sharepoint_constants.py,sha256=erkn-3E8YwBMFs25o7exXoK7s73NdgP9IYDXeWzALcI,98
 saviialib/services/epii/use_cases/types/__init__.py,sha256=u6fyodOEJE2j6FMqJux40Xf9ccYAi-UUYxqT-Kzc0kE,199
 saviialib/services/epii/use_cases/types/update_thies_data_types.py,sha256=3lJzG1nuZoP1mqFlvQ0-aFJp80SECaeiROlvucVhaSY,539
 saviialib/services/epii/use_cases/types/upload_backup_to_sharepoint_types.py,sha256=J_sGhqSaPoMZA0GIrzCx6pmgSKgIkGi0uyrsltU_H8w,320
-saviialib/services/epii/use_cases/update_thies_data.py,sha256=0dkk45qPkvPpxS8uERPDrPN4woWz4vze3FR0gA-P6wI,9866
+saviialib/services/epii/use_cases/update_thies_data.py,sha256=moB4fz_aOiKBqQixjm5uS0hyvpF9ufLIMWbVxwXqICc,14591
 saviialib/services/epii/use_cases/upload_backup_to_sharepoint.py,sha256=-0MvCzkFM7T2y7XLgFevWAXmqAsthzuY68ReiiL2zDU,11886
 saviialib/services/epii/utils/__init__.py,sha256=cYt2tvq65_OMjFaqb8-CCC7IGCQgFd4ziEUWJV7s1iY,98
 saviialib/services/epii/utils/update_thies_data_utils.py,sha256=EpjYWXqyHxJ-dO3MHhdXp-rGV7WyUckeFko-nnfnNac,555
 saviialib/services/epii/utils/upload_backup_to_sharepoint_utils.py,sha256=hEeV4_kcG8YL6t_3V8AlhgDHtHiUNsdYpilfgTLaQMc,3528
-saviialib-1.0.1.dist-info/LICENSE,sha256=NWpf6b38xgBWPBo5HZsCbdfp9hZSliEbRqWQgm0fkOo,1076
-saviialib-1.0.1.dist-info/METADATA,sha256=qxj1sUazmjA38ue9RFtljIuW4xt9IJGyPSkBpS_LXqQ,4063
-saviialib-1.0.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-saviialib-1.0.1.dist-info/RECORD,,
+saviialib-1.2.0.dist-info/LICENSE,sha256=NWpf6b38xgBWPBo5HZsCbdfp9hZSliEbRqWQgm0fkOo,1076
+saviialib-1.2.0.dist-info/METADATA,sha256=RP2WzDmV8mw8Y9IBsqE3-tXVoBchB2ugGlNjrlnH2oY,4130
+saviialib-1.2.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+saviialib-1.2.0.dist-info/RECORD,,