servify 0.0.43__tar.gz → 0.0.45__tar.gz

{servify-0.0.43/servify.egg-info → servify-0.0.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: servify
-Version: 0.0.43
+Version: 0.0.45
 Summary: A Python library that simplifies data manipulation and workflow development with PySpark in Databricks environments.
 Author: Felipe Pegoraro
 Author-email: felipepegoraro93@gmail.com

{servify-0.0.43 → servify-0.0.45}/servify/__init__.py

@@ -1,6 +1,6 @@
 from pyspark.sql import DataFrame
 
-from .commons.func_read import servify_read
+from .commons.func_read.read import servify_read
 from .settings.log.servify_configs import set_logging, LOG_ENABLED
 
 _reader: servify_read | None = None

servify-0.0.45/servify/commons/__init__.py

@@ -0,0 +1,9 @@
+from .func_read import servify_read
+from .func_read import ConfigError, DataValidationError, IoError
+
+__all__ = [
+    "servify_read",
+    "ConfigError",
+    "DataValidationError",
+    "IoError",
+]

servify-0.0.45/servify/commons/func_read/utils/csv_reader.py

@@ -0,0 +1,91 @@
+import csv
+import re
+from pyspark.sql import DataFrame
+
+
+class CsvReader:
+
+    def __init__(self, spark, log):
+        self.spark = spark
+        self.log = log
+
+    def detect_delimiter(self, file_path: str) -> str:
+        
+        encoding_detectado = self.obter_enconding(file_path)
+
+        self.log.info(f"Starting delimiter detection for file: {file_path}")
+
+        try:
+            with open(
+                file_path, "r", encoding=encoding_detectado, newline=""
+            ) as f:
+                linha = f.readline()
+                if not linha:
+                    self.log.warning(f"File is empty: {file_path}.")
+
+                self.log.debug(
+                    f"first line read for delimiter detection: {linha.rstrip("\n")}"
+                )
+
+        except Exception as e:
+            self.log.error(
+                f"Error reading file {file_path}: {e}", exc_info=True
+            )
+            raise ValueError(f"Error reading file {file_path}: {e}") from e
+
+        delimitadores = [",", ";", "\t", "|"]
+
+        contagem = {d: len(re.findall(re.escape(d), linha)) for d in delimitadores}
+        self.log.debug(f"Delimiter counts: {contagem}")
+
+        if all(c == 0 for c in contagem.values()):
+            self.log.warning(
+                f"No delimiters found in the first line of file: {file_path}. Trying csv.Sniffer...."
+            )
+            try:
+                dialect = csv.Sniffer().sniff(linha, delimiters="," ";|")
+                detected = dialect.delimiter
+                self.log.info(f"Delimiter detected by csv.Sniffer: {detected}")
+                return detected
+            except Exception as e:
+                self.log.error(
+                    f"csv.Sniffer failed to detect delimiter for file {file_path}: {e}",
+                    exc_info=True,
+                )
+                self.log.warning(
+                    f"Using default delimiter ',' for file: {file_path}."
+                )
+                return ","
+
+        delimitador_detectado = max(contagem.items(), key=lambda kv: kv[1])[0]
+        self.log.info(
+            f"Delimiter detected: {delimitador_detectado} for file: {file_path}"
+        )
+        return delimitador_detectado
+
+    def read_csv(self, path: str, delimiter: str | None = None, encoding: str | None = None) -> DataFrame:
+
+        if delimiter or encoding:
+            self.log.info(
+                f"Using provided delimiter: '{delimiter}' and encoding: '{encoding}' for file: {path}."
+            )
+        
+        else:
+            self.log.warning(
+                f"No delimiter or encoding provided. Starting auto-detection for file: {path}."
+            )
+
+            delimiter = ","
+            encoding = "utf-8"
+
+            self.log.info(f"Using default delimiter: '{delimiter}' and encoding: '{encoding}' for file: {path}.")
+
+        return (
+            self.spark.read
+            .option("header", True)
+            .option("inferSchema", "false")
+            .option("samplingRatio", 0.1)
+            .option("delimiter", delimiter)
+            .option("encoding", encoding)
+            .csv(path)
+        )

servify-0.0.45/servify/commons/func_read/utils/encoding.py

@@ -0,0 +1,33 @@
+import chardet
+
+
+class EncodingUtils:
+
+    def detect_encoding(self, file_path: str, *, sample_bytes: int = 4096) -> str:
+
+        self.log.info(f"Initialized encoding detection for file: {file_path}")
+
+        try:
+            with open(file_path, "rb") as f:
+                rawdata = f.read(sample_bytes)
+                result = chardet.detect(rawdata) or {}
+                encoding_detectado: str = result.get("encoding") or "utf-8"
+
+                conf = result.get("confidence")
+                self.log.debug(
+                    f"Encoding detected: {encoding_detectado} with confidence: {conf} and language: {result.get('language')}"
+                )
+        except Exception as e:
+            self.log.error(
+                f"Error detecting encoding for file {file_path}: {e}",
+                exc_info=True,
+            )
+            raise ValueError(
+                f"Error detecting encoding for file {file_path}: {e}"
+            ) from e
+
+        self.log.info(
+            f"Encoding detection completed for file: {file_path} - Encoding: {encoding_detectado}"
+        )
+
+        return encoding_detectado

servify-0.0.45/servify/commons/func_read/utils/excel.py

@@ -0,0 +1,208 @@
+import pandas as pd
+import os
+import math
+import re
+from pyspark.sql import DataFrame
+from pyspark.sql import types as T, functions as F
+import numpy as np
+from typing import List, Optional
+
+
+class ExcelReader:
+
+    def __init__(self, spark, log):
+        self.spark = spark
+        self.log = log
+
+    def saniteze_columns(
+        self, header_cells: List, prefer_from_schema: Optional[T.StructType] = None
+    ) -> List[str]:
+
+        target_names: List[str] = (
+            [f.name for f in prefer_from_schema] if prefer_from_schema else []
+        )
+
+        seen = set()
+        safe_cols = []
+
+        def is_blank(x):
+            if x is None:
+                return True
+            try:
+                if isinstance(x, float) and math.isnan(x):
+                    return True
+            except Exception:
+                pass
+            s = str(x).strip()
+            return s == "" or s.lower() == "nan"
+
+        for i, h in enumerate(header_cells):
+            if is_blank(h):
+                if target_names and i < len(target_names):
+                    base = target_names[i]
+                else:
+                    base = f"c_{i+1}"
+            else:
+                base = str(h).replace("\n", " ").replace("\r", " ").strip()
+
+            base = re.sub(r"\s+", "_", base)
+            base = re.sub(r"[^0-9a-zA-Z_]", "", base)
+
+            if base[0].isdigit():
+                base = f"c_{base}"
+
+            name = base
+            k = 1
+            while name in seen:
+                name = f"{base}__{k}"
+                k += 1
+            seen.add(name)
+            safe_cols.append(name)
+
+        return safe_cols
+
+    def concat_ps_dfs(
+        self, lista_files: List[str], schema: Optional[T.StructType] = None
+    ) -> DataFrame:
+        """
+        Concatena uma lista de DataFrames do Pandas on Spark API.
+        """
+
+        if not lista_files:
+            self.log.warning("No DataFrames provided for concatenation.")
+            raise ValueError("No DataFrames provided for concatenation.")
+
+        if len(lista_files) == 1:
+            self.log.info("Only one DataFrame provided, returning it directly.")
+            return self.read_xlsx_with_pandas(lista_files[0], schema=schema)
+
+        self.log.info(f"Concatenating {len(lista_files)} Pandas on Spark DataFrames")
+
+        df_final = self.read_xlsx_with_pandas(lista_files[0], schema=schema)
+        self.log.info(f"file: {lista_files[0]} read successfully.")
+
+        for file in lista_files[1:]:
+            df = self.read_xlsx_with_pandas(file, schema=schema)
+            df_final = df_final.unionByName(df, allowMissingColumns=True)
+            self.log.info(f"file: {file} read and concatenated successfully.")
+
+        self.log.info("DataFrames concatenated successfully.")
+        return df_final
+
+    def remove_header_rows(self, spark_df: DataFrame) -> DataFrame:
+        """
+        Remove linhas de cabeçalho duplicadas de um DataFrame do Spark.
+        """
+
+        self.log.info("Verify if having heands on lines to remove")
+
+        primeira_coluna = spark_df.columns[0]
+
+        valor_normalizado = F.lower(
+            F.regexp_replace(
+                F.trim(F.regexp_replace(F.col(primeira_coluna), "_", "")),
+                r"\s+",
+                " ",
+            )
+        )
+
+        nome_normalizado = primeira_coluna.lower().replace("_", "").strip()
+
+        existe_header = (
+            spark_df.filter(valor_normalizado == F.lit(nome_normalizado)).count() > 0
+        )
+
+        if existe_header:
+            self.log.info("Header rows detected, removing them")
+            spark_df_limpo = spark_df.filter(
+                valor_normalizado != F.lit(nome_normalizado)
+            )
+            self.log.info("Header rows removed successfully")
+            return spark_df_limpo
+
+        self.log.info("No header rows detected, returning original DataFrame")
+        return spark_df
+
+    def read_xlsx_with_pandas(
+        self, xlsx_path: str, schema: Optional[T.StructType] = None, sheet_name: int = 0
+    ):
+        """
+        Lê um arquivo .xlsx em um DataFrame do Pandas on Spark API.
+        """
+
+        self.log.info(f"Reading .xlsx file: {xlsx_path} by pandas")
+
+        try:
+            df = pd.read_excel(
+                xlsx_path,
+                engine="openpyxl",
+                header=0,
+                sheet_name=sheet_name,
+            )
+
+            valid_mask = df.notna().any(axis=1)
+            if not bool(valid_mask.all()):
+                self.log.warning(f"All rows are empty in .xlsx file: {xlsx_path}")
+                raise ValueError(f"All rows are empty in .xlsx file: {xlsx_path}")
+
+            first_valid_pos = int(np.argmax(valid_mask.to_numpy()))
+
+        except Exception as e:
+            self.log.warning(f"Error reading .xlsx file {xlsx_path} by pandas: {e}")
+            raise
+
+        header_raw = df.iloc[first_valid_pos].tolist()
+        safe_cols = self.saniteze_columns(header_raw, prefer_from_schema=schema)
+
+        df = df.dropna(how="all").reset_index(drop=True)
+
+        for c in range(len(safe_cols)):
+            col = df.columns[c]
+            df[col] = (
+                df[col]
+                .where(~df[col].isna(), None)
+                .map(lambda x: str(x) if x is not None else None)
+            )
+
+        if schema is None:
+            df.columns = safe_cols
+
+            self.log.info(
+                "Using safe conversion mode to avoid internal Serverless Arrow errors."
+            )
+
+            string_schema = T.StructType(
+                [T.StructField(c, T.StringType(), True) for c in df.columns]
+            )
+
+            try:
+                sdf = self.spark.createDataFrame(df, schema=string_schema)
+            except Exception as e:
+                self.log.error(
+                    "Spark Serverless failed to create DataFrame even in safe mode."
+                )
+                self.log.debug(f"Internal Spark error: {e}")
+                raise
+
+            sdf = sdf.select([F.col(c).cast("string").alias(c) for c in sdf.columns])
+            sdf = sdf.withColumn("source_file", F.lit(os.path.basename(xlsx_path)))
+            sdf = self.remove_header_rows(sdf)
+
+            return sdf
+
+        df.columns = safe_cols
+
+        string_schema = T.StructType(
+            [T.StructField(c, T.StringType(), True) for c in df.columns]
+        )
+
+        sdf = self.spark.createDataFrame(df, schema=string_schema)
+
+        sdf = sdf.withColumn("source_file", F.lit(os.path.basename(xlsx_path)))
+
+        sdf = self.commons_shared.aplicar_schema_df(sdf, schema)
+
+        sdf = self.remove_header_rows(sdf)
+
+        return sdf
+

{servify-0.0.43 → servify-0.0.45}/servify/commons/func_read/utils/helpers.py

@@ -22,10 +22,10 @@ class helper_reading_data(HelperBase):
         # composição
         self.paths = PathUtils()
         self.encoding = EncodingUtils()
-        self.excel = ExcelReader(self.spark)
-        self.csv = CsvReader(self.spark)
-        self.json = JsonReader(self.spark)
-        self.parquet = ParquetReader(self.spark)
+        self.excel = ExcelReader(self.spark, self.log)
+        self.csv = CsvReader(self.spark, self.log)
+        self.json = JsonReader(self.spark, self.log)
+        self.parquet = ParquetReader(self.spark, self.log)
         self.schema_utils = SchemaUtils(self.commons_shared)
 
     # métodos de fachada, mantendo a API que o read.py espera

servify-0.0.45/servify/commons/func_read/utils/json_reader.py

@@ -0,0 +1,50 @@
+from pyspark.sql import DataFrame
+
+
+class JsonReader:
+
+    def __init__(self, spark, log):
+        self.spark = spark
+        self.log = log
+
+    def detectar_json_multiline(self, path: str) -> bool:
+
+        self.log.info(f"Starting JSON multiline detection for file: {path}")
+
+        encoding_detectado = self.obter_enconding(path)
+
+        try:
+            with open(path, "r", encoding=encoding_detectado) as f:
+                linhas = f.readlines()
+            self.log.info(f"File {path} read successfully.")
+        except Exception as e:
+            self.log.error(f"Error reading file {path}: {e}")
+            raise ValueError(f"Error reading file {path}: {e}") from e
+
+        primeira_linha = linhas[0].strip()
+        self.log.debug(f"First line for JSON multiline detection: {primeira_linha}")
+
+        if primeira_linha.startswith("{") or (
+            primeira_linha.startswith("[") and len(linhas) >= 1
+        ):
+            self.log.info(f"JSON multiline detected for file: {path}")
+            return True
+        self.log.info(f"JSON single line detected for file: {path}")
+        return False
+
+    def read_json(self, path: str) -> DataFrame:
+
+        arquivo_escolhido = self.resolve_latest_file(path)
+
+        multiline = self.detectar_json_multiline(arquivo_escolhido)
+
+        if multiline is not None:
+
+            return self.spark.read.option("multiline", str(multiline).lower()).json(
+                path
+            )
+        else:
+            self.log.info(
+                f"Using default JSON multiline 'false' for {path}."
+            )
+            return self.spark.read.json(path)

{servify-0.0.43 → servify-0.0.45}/servify/commons/func_read/utils/parquet_reader.py

@@ -3,8 +3,10 @@ from pyspark.sql import DataFrame
 
 class ParquetReader:
 
-    def __init__(self, spark):
+    def __init__(self, spark, log):
         self.spark = spark
+        self.log = log
 
     def read_parquet(self, path: str) -> DataFrame:
+        self.log.info(f"Reading Parquet file: {path}")
         return self.spark.read.parquet(path)

servify-0.0.45/servify/commons/func_read/utils/paths.py

@@ -0,0 +1,110 @@
+import glob
+import os
+from typing import List
+
+
+class PathUtils:
+
+    def list_xlsx_paths(self, directory: str) -> List[str]:
+        """
+        Lista todos os arquivos .xlsx em um diretório especificado.
+        """
+
+        self.log.info(f"Listing .xlsx files in directory: {directory}")
+
+        if os.path.isfile(directory):
+            if directory.lower().endswith(".xlsx"):
+                self.log.info(f"Single .xlsx file found: {directory}")
+                return [directory]
+
+            self.log.error(f"The specified path is a file but not .xlsx: {directory}")
+            raise ValueError(f"The specified path is a file but not .xlsx: {directory}")
+
+        if os.path.isdir(directory):
+            paths = [
+                os.path.join(directory, nome)
+                for nome in os.listdir(directory)
+                if nome.lower().endswith(".xlsx")
+            ]
+            if not paths:
+                self.log.error(f"No .xlsx files found in directory: {directory}")
+                raise FileNotFoundError(
+                    f"No .xlsx files found in directory: {directory}"
+                )
+
+            self.log.info(f"Found {len(paths)} .xlsx files in directory: {directory}")
+            return paths
+
+        self.log.error(
+            f"The specified path is neither a file nor a directory: {directory}"
+        )
+        raise ValueError(
+            f"The specified path is neither a file nor a directory: {directory}"
+        )
+
+    def resolve_accessible_path(self, path: str, dbutils) -> str:
+        """
+        Valida/resolve um path para leitura DBFS ou 'file:'.
+        - Com wildcard: garante que existe ao menos um arquivo, mantém wildcard para leitura.
+        - Sem wildcard: tenta DBFS, se não, tenta 'file:' se nada der certo, lança FileNotFoundError
+        """
+
+        if "*" in path:
+            arquivos = glob.glob(path.replace("file:", ""))
+            if not arquivos:
+                raise FileNotFoundError(f"No file founded in: {path}")
+
+            primeiro = arquivos[0]
+            try:
+                dbutils.fs.ls(primeiro)
+            except Exception:
+                arquivo_file = f"file:{primeiro}"
+                try:
+                    dbutils.fs.ls(arquivo_file)
+                except Exception as exc_file:
+                    raise FileNotFoundError(
+                        f"File '{arquivo_file}' is not accessible by DBFS netheir 'file:'."
+                    ) from exc_file
+
+            return path
+
+        try:
+            dbutils.fs.ls(path)
+            return path
+        except Exception:
+            path_file = f"file:{path}"
+            try:
+                dbutils.fs.ls(path_file)
+                return path_file
+            except Exception as exc_file:
+                raise FileNotFoundError(
+                    f"File '{path_file}' is not accessible by DBFS netheir 'file:'."
+                ) from exc_file
+
+    def resolve_latest_file(self, path: str) -> str:
+
+        self.log.debug(f"Resolving path: {path}")
+
+        try:
+            path_resolvido = path.replace("file:", "")
+        except Exception as e:
+            self.log.error(f"Error resolving path: {e}")
+            raise ValueError(f"Error resolving path: {e}") from e
+
+        if "*" in path_resolvido:
+            arquivos = glob.glob(path_resolvido)
+            self.log.debug(f"Found files with wildcard: {arquivos}")
+
+            if not arquivos:
+                self.log.error(
+                    f"No files found for path with wildcard: {path_resolvido}"
+                )
+                raise FileNotFoundError(f"No files found for path: {path_resolvido}")
+
+            arquivos.sort(key=os.path.getmtime, reverse=True)
+            escolhido = arquivos[0]
+            self.log.info(f"Latest file selected: {escolhido}")
+            return escolhido
+
+        self.log.info(f"Path resolved without wildcard: {path_resolvido}")
+        return path_resolvido

{servify-0.0.43 → servify-0.0.45/servify.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: servify
-Version: 0.0.43
+Version: 0.0.45
 Summary: A Python library that simplifies data manipulation and workflow development with PySpark in Databricks environments.
 Author: Felipe Pegoraro
 Author-email: felipepegoraro93@gmail.com

{servify-0.0.43 → servify-0.0.45}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as arq:
 
 setup(
     name="servify",
-    version="0.0.43",
+    version="0.0.45",
     license="MIT",
     author="Felipe Pegoraro",
     author_email="felipepegoraro93@gmail.com",

servify-0.0.43/servify/commons/func_read/utils/csv_reader.py

@@ -1,23 +0,0 @@
-import csv
-from pyspark.sql import DataFrame
-
-
-class CsvReader:
-
-    def __init__(self, spark):
-        self.spark = spark
-
-    def detect_delimiter(self, file_path: str) -> str:
-        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
-            sample = f.read(2048)
-        return csv.Sniffer().sniff(sample).delimiter
-
-    def read_csv(self, path: str, delimiter: str | None = None) -> DataFrame:
-        if delimiter is None:
-            delimiter = self.detect_delimiter(path)
-        return (
-            self.spark.read
-            .option("header", True)
-            .option("delimiter", delimiter)
-            .csv(path)
-        )

servify-0.0.43/servify/commons/func_read/utils/encoding.py

@@ -1,9 +0,0 @@
-import chardet
-
-
-class EncodingUtils:
-
-    def detect_encoding(self, file_path: str) -> str:
-        with open(file_path, "rb") as f:
-            raw = f.read(50000)
-        return chardet.detect(raw)["encoding"] or "utf-8"

servify-0.0.43/servify/commons/func_read/utils/excel.py

@@ -1,15 +0,0 @@
-import pandas as pd
-from pyspark.sql import DataFrame
-from pyspark.sql import types as T
-
-
-class ExcelReader:
-
-    def __init__(self, spark):
-        self.spark = spark
-
-    def concat_ps_dfs(self, paths, schema: T.StructType | None) -> DataFrame:
-        pdfs = [pd.read_excel(p) for p in paths]
-        pdf = pd.concat(pdfs, ignore_index=True)
-        return self.spark.createDataFrame(pdf, schema=schema)
-

servify-0.0.43/servify/commons/func_read/utils/json_reader.py

@@ -1,10 +0,0 @@
-from pyspark.sql import DataFrame
-
-
-class JsonReader:
-
-    def __init__(self, spark):
-        self.spark = spark
-
-    def read_json(self, path: str) -> DataFrame:
-        return self.spark.read.option("multiLine", True).json(path)

servify-0.0.43/servify/commons/func_read/utils/paths.py

@@ -1,14 +0,0 @@
-import glob
-import os
-from typing import List
-
-
-class PathUtils:
-
-    def list_xlsx_paths(self, directory: str) -> List[str]:
-        return glob.glob(os.path.join(directory, "*.xlsx"))
-
-    def resolve_accessible_path(self, path: str) -> str:
-        if path.startswith("dbfs:"):
-            return path.replace("dbfs:", "/dbfs")
-        return path