FlowerPower 0.11.6__py3-none-any.whl → 0.11.6.2__py3-none-any.whl

flowerpower/fs/ext.py

@@ -193,6 +193,7 @@ def _read_json(
         as_dataframe: (bool, optional) If True, return a DataFrame. Defaults to True.
         concat: (bool, optional) If True, concatenate the DataFrames. Defaults to True.
         verbose: (bool, optional) If True, print verbose output. Defaults to False.
+        opt_dtypes: (bool, optional) If True, optimize DataFrame dtypes. Defaults to False.
         **kwargs: Additional keyword arguments.
 
     Returns:
@@ -247,8 +248,8 @@ def _read_json(
             data = [opt_dtype_pl(df, strict=False) for df in data]
         if concat:
             result = pl.concat(data, how="diagonal_relaxed")
-            if opt_dtypes:
-                result = opt_dtype_pl(result, strict=False)
+            # if opt_dtypes:
+            #   result = opt_dtype_pl(result, strict=False)
             return result
     return data
 
@@ -280,6 +281,7 @@ def _read_json_batches(
         concat: Combine files within each batch
         use_threads: Enable parallel file reading within batches
         verbose: Print progress information
+        opt_dtypes: Optimize DataFrame dtypes
         **kwargs: Additional arguments for DataFrame conversion
 
     Yields:
@@ -354,10 +356,16 @@ def _read_json_batches(
                     ][0]
                     for _data in batch_data
                 ]
-
+            if opt_dtypes:
+                batch_dfs = [opt_dtype_pl(df, strict=False) for df in batch_dfs]
             if concat and len(batch_dfs) > 1:
-                yield pl.concat(batch_dfs, how="diagonal_relaxed")
+                batch_df = pl.concat(batch_dfs, how="diagonal_relaxed")
+                # if opt_dtypes:
+                #    batch_df = opt_dtype_pl(batch_df, strict=False)
+                yield batch_df
             else:
+                # if opt_dtypes:
+                #    batch_dfs = [opt_dtype_pl(df, strict=False) for df in batch_dfs]
                 yield batch_dfs
         else:
             yield batch_data
@@ -403,6 +411,7 @@ def read_json(
         concat: Combine multiple files/batches into single result
         use_threads: Enable parallel file reading
         verbose: Print progress information
+        opt_dtypes: Optimize DataFrame dtypes for performance
         **kwargs: Additional arguments passed to DataFrame conversion
 
     Returns:
@@ -486,6 +495,7 @@ def _read_csv_file(
         path: Path to CSV file
         self: Filesystem instance to use for reading
         include_file_path: Add source filepath as a column
+        opt_dtypes: Optimize DataFrame dtypes
         **kwargs: Additional arguments passed to pl.read_csv()
 
     Returns:
@@ -544,6 +554,7 @@ def _read_csv(
         use_threads: (bool, optional) If True, read files in parallel. Defaults to True.
         concat: (bool, optional) If True, concatenate the DataFrames. Defaults to True.
         verbose: (bool, optional) If True, print verbose output. Defaults to False.
+        opt_dtypes: (bool, optional) If True, optimize DataFrame dtypes. Defaults to False.
         **kwargs: Additional keyword arguments.
 
     Returns:
@@ -587,8 +598,8 @@ def _read_csv(
         )
     if concat:
         result = pl.concat(dfs, how="diagonal_relaxed")
-        if opt_dtypes:
-            result = opt_dtype_pl(result, strict=False)
+        # if opt_dtypes:
+        #    result = opt_dtype_pl(result, strict=False)
         return result
     return dfs
 
@@ -616,6 +627,7 @@ def _read_csv_batches(
         concat: Combine files within each batch
         use_threads: Enable parallel file reading within batches
         verbose: Print progress information
+        opt_dtypes: Optimize DataFrame dtypes
         **kwargs: Additional arguments passed to pl.read_csv()
 
     Yields:
@@ -667,23 +679,28 @@ def _read_csv_batches(
                 n_jobs=-1,
                 backend="threading",
                 verbose=verbose,
+                opt_dtypes=opt_dtypes,
                 **kwargs,
             )
         else:
             batch_dfs = [
                 _read_csv_file(
-                    p, self=self, include_file_path=include_file_path, **kwargs
+                    p,
+                    self=self,
+                    include_file_path=include_file_path,
+                    opt_dtypes=opt_dtypes,
+                    **kwargs,
                 )
                 for p in batch_paths
             ]
 
-        if opt_dtypes:
-            batch_dfs = [opt_dtype_pl(df, strict=False) for df in batch_dfs]
+        # if opt_dtypes:
+        #    batch_dfs = [opt_dtype_pl(df, strict=False) for df in batch_dfs]
 
         if concat and len(batch_dfs) > 1:
             result = pl.concat(batch_dfs, how="diagonal_relaxed")
-            if opt_dtypes:
-                result = opt_dtype_pl(result, strict=False)
+            # if opt_dtypes:
+            #    result = opt_dtype_pl(result, strict=False)
             yield result
         else:
             yield batch_dfs
@@ -766,6 +783,7 @@ def read_csv(
             concat=concat,
             use_threads=use_threads,
             verbose=verbose,
+            opt_dtypes=opt_dtypes,
             **kwargs,
         )
     return _read_csv(
@@ -775,6 +793,7 @@ def read_csv(
         concat=concat,
         use_threads=use_threads,
         verbose=verbose,
+        opt_dtypes=opt_dtypes,
         **kwargs,
     )
 
@@ -858,9 +877,7 @@ def _read_parquet(
     if not include_file_path and concat:
         if isinstance(path, str):
             path = path.replace("**", "").replace("*.parquet", "")
-        table = pq.read_table(path, filesystem=self, **kwargs)
-        if opt_dtypes:
-            table = opt_dtype_pa(table, strict=False)
+        table = _read_parquet_file(path, self=self, opt_dtypes=opt_dtypes, **kwargs)
         return table
     else:
         if isinstance(path, str):
@@ -907,12 +924,12 @@ def _read_parquet(
                 unified_schema = unify_schemas_pa(schemas)
                 tables = [cast_schema(t, unified_schema) for t in tables]
             result = pa.concat_tables(tables, promote_options="permissive")
-            if opt_dtypes:
-                result = opt_dtype_pa(result, strict=False)
+            # if opt_dtypes:
+            #    result = opt_dtype_pa(result, strict=False)
             return result
         elif isinstance(tables, pa.Table):
-            if opt_dtypes:
-                tables = opt_dtype_pa(tables, strict=False)
+            # if opt_dtypes:
+            #    tables = opt_dtype_pa(tables, strict=False)
             return tables
         else:
             return pa.concat_tables(tables, promote_options="permissive")
@@ -981,9 +998,9 @@ def _read_parquet_batches(
     if not include_file_path and concat and batch_size is None:
         if isinstance(path, str):
             path = path.replace("**", "").replace("*.parquet", "")
-        table = pq.read_table(path, filesystem=self, **kwargs)
-        if opt_dtypes:
-            table = opt_dtype_pa(table, strict=False)
+        table = _read_parquet_file(
+            path=path, self=self, opt_dtypes=opt_dtypes, **kwargs
+        )
         yield table
         return
 
@@ -994,7 +1011,11 @@ def _read_parquet_batches(
 
     if not isinstance(path, list):
         yield _read_parquet_file(
-            path=path, self=self, include_file_path=include_file_path, **kwargs
+            path=path,
+            self=self,
+            include_file_path=include_file_path,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
         )
         return
 
@@ -1032,12 +1053,12 @@ def _read_parquet_batches(
                 unified_schema = unify_schemas_pa(schemas)
                 batch_tables = [cast_schema(t, unified_schema) for t in batch_tables]
             result = pa.concat_tables(batch_tables, promote_options="permissive")
-            if opt_dtypes:
-                result = opt_dtype_pa(result, strict=False)
+            # if opt_dtypes:
+            #    result = opt_dtype_pa(result, strict=False)
             yield result
         else:
-            if opt_dtypes and isinstance(batch_tables, list):
-                batch_tables = [opt_dtype_pa(t, strict=False) for t in batch_tables]
+            # if opt_dtypes and isinstance(batch_tables, list):
+            #    batch_tables = [opt_dtype_pa(t, strict=False) for t in batch_tables]
             yield batch_tables
 
 
@@ -1077,6 +1098,7 @@ def read_parquet(
         concat: Combine multiple files/batches into single Table
         use_threads: Enable parallel file reading
         verbose: Print progress information
+        opt_dtypes: Optimize Table dtypes for performance
         **kwargs: Additional arguments passed to pq.read_table()
 
     Returns:
@@ -1119,6 +1141,7 @@ def read_parquet(
             concat=concat,
             use_threads=use_threads,
             verbose=verbose,
+            opt_dtypes=opt_dtypes,
             **kwargs,
         )
     return _read_parquet(
@@ -1128,6 +1151,7 @@ def read_parquet(
         use_threads=use_threads,
         concat=concat,
         verbose=verbose,
+        opt_dtypes=opt_dtypes,
         **kwargs,
     )
 
@@ -1142,6 +1166,7 @@ def read_files(
     jsonlines: bool = False,
     use_threads: bool = True,
     verbose: bool = False,
+    opt_dtypes: bool = False,
     **kwargs: Any,
 ) -> (
     pl.DataFrame
@@ -1175,6 +1200,7 @@ def read_files(
         jsonlines: For JSON format, whether to read as JSON Lines
         use_threads: Enable parallel file reading
         verbose: Print progress information
+        opt_dtypes: Optimize DataFrame/Arrow Table dtypes for performance
         **kwargs: Additional format-specific arguments
 
     Returns:
@@ -1224,6 +1250,7 @@ def read_files(
                 concat=concat,
                 use_threads=use_threads,
                 verbose=verbose,
+                opt_dtypes=opt_dtypes,
                 **kwargs,
             )
         return read_json(
@@ -1234,6 +1261,7 @@ def read_files(
             concat=concat,
             use_threads=use_threads,
             verbose=verbose,
+            opt_dtypes=opt_dtypes,
             **kwargs,
         )
     elif format == "csv":
@@ -1246,6 +1274,7 @@ def read_files(
                 concat=concat,
                 use_threads=use_threads,
                 verbose=verbose,
+                opt_dtypes=opt_dtypes,
                 **kwargs,
             )
         return read_csv(
@@ -1255,6 +1284,7 @@ def read_files(
             use_threads=use_threads,
             concat=concat,
             verbose=verbose,
+            opt_dtypes=opt_dtypes,
             **kwargs,
         )
     elif format == "parquet":
@@ -1267,6 +1297,7 @@ def read_files(
                 concat=concat,
                 use_threads=use_threads,
                 verbose=verbose,
+                opt_dtypes=opt_dtypes,
                 **kwargs,
             )
         return read_parquet(
@@ -1276,6 +1307,7 @@ def read_files(
             use_threads=use_threads,
             concat=concat,
             verbose=verbose,
+            opt_dtypes=opt_dtypes,
             **kwargs,
         )
 

flowerpower/plugins/io/base.py

@@ -185,74 +185,135 @@ class BaseFileReader(BaseFileIO, gc=False):
     include_file_path: bool = field(default=False)
     concat: bool = field(default=True)
     batch_size: int | None = field(default=None)
+    opt_dtypes: bool = field(default=True)
+    use_threads: bool = field(default=True)
     conn: duckdb.DuckDBPyConnection | None = field(default=None)
     ctx: datafusion.SessionContext | None = field(default=None)
     jsonlines: bool | None = field(default=None)
     partitioning: str | list[str] | pds.Partitioning | None = field(default=None)
+    verbose: bool | None = field(default=None)
     _data: Any | None = field(default=None)
 
-    def _load(self, reload: bool = False, **kwargs):
-        if "include_file_path" in kwargs:
-            if self.include_file_path != kwargs["include_file_path"]:
+    def _load(
+        self,
+        metadata: bool = False,
+        reload: bool = False,
+        batch_size: int | None = None,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
+    ):
+        if batch_size is not None:
+            if self.batch_size != batch_size:
                 reload = True
-                self.include_file_path = kwargs.pop("include_file_path")
-            else:
-                kwargs.pop("include_file_path")
+            self.batch_size = batch_size
 
-        if "concat" in kwargs:
-            if self.concat != kwargs["concat"]:
+        if include_file_path is not None:
+            if self.include_file_path != include_file_path:
                 reload = True
-                self.concat = kwargs.pop("concat")
-            else:
-                kwargs.pop("concat")
+                self.include_file_path = include_file_path
 
-        if "batch_size" in kwargs:
-            if self.batch_size != kwargs["batch_size"]:
+        if concat is not None:
+            if self.concat != concat:
                 reload = True
-                self.batch_size = kwargs.pop("batch_size")
-            else:
-                kwargs.pop("batch_size")
+                self.concat = concat
+
+        if use_threads is not None:
+            if self.use_threads != use_threads:
+                reload = True
+                self.use_threads = use_threads
+
+        if verbose is not None:
+            if self.verbose != verbose:
+                reload = True
+                self.verbose = verbose
+
+        if opt_dtypes is not None:
+            if self.opt_dtypes != opt_dtypes:
+                reload = True
+                self.opt_dtypes = opt_dtypes
 
         if "partitioning" in kwargs:
             if self.partitioning != kwargs["partitioning"]:
                 reload = True
                 self.partitioning = kwargs.pop("partitioning")
-            else:
-                kwargs.pop("partitioning")
 
         if not hasattr(self, "_data") or self._data is None or reload:
             self._data = self.fs.read_files(
                 path=self._glob_path,
                 format=self.format,
-                include_file_path=True,
+                include_file_path=True if metadata or self.include_file_path else False,
                 concat=self.concat,
                 jsonlines=self.jsonlines or None,
                 batch_size=self.batch_size,
                 partitioning=self.partitioning,
+                opt_dtypes=self.opt_dtypes,
+                verbose=self.verbose,
+                use_threads=self.use_threads,
                 **kwargs,
             )
-            if not isinstance(self._data, Generator):
-                self._metadata = get_dataframe_metadata(
-                    df=self._data,
-                    path=self.path,
-                    format=self.format,
-                    # num_files=pl.from_arrow(self._data.select(["file_path"])).select(
-                    #    pl.n_unique("file_path")
-                    # )[0, 0],
-                )
-                if not self.include_file_path:
-                    if isinstance(self._data, pa.Table):
+            if metadata:
+                if isinstance(self._data, tuple | list):
+                    self._metadata = [
+                        get_dataframe_metadata(
+                            df=df,
+                            path=self.path,
+                            format=self.format,
+                            num_files=pl.from_arrow(df.select(["file_path"])).select(
+                                pl.n_unique("file_path")
+                            )[0, 0]
+                            if isinstance(df, pa.Table)
+                            else df.select(pl.n_unique("file_path"))[0, 0],
+                        )
+                        for df in self._data
+                    ]
+                    if not self.include_file_path:
+                        self._data = [df.drop("file_path") for df in self._data]
+
+                elif isinstance(self._data, pa.Table):
+                    self._metadata = get_dataframe_metadata(
+                        df=self._data,
+                        path=self.path,
+                        format=self.format,
+                        num_files=pl.from_arrow(
+                            self._data.select(pl.n_unique("file_path"))
+                        )[0, 0],
+                    )
+                    if not self.include_file_path:
+                        self._data = self._data.drop("file_path")
+
+                elif isinstance(self._data, pl.DataFrame | pl.LazyFrame):
+                    self._metadata = get_dataframe_metadata(
+                        df=self._data,
+                        path=self.path,
+                        format=self.format,
+                        num_files=self._data.select(pl.n_unique("file_path"))[0, 0]
+                        if isinstance(self._data, pl.DataFrame)
+                        else self._data.select(pl.n_unique("file_path")).collect()[
+                            0, 0
+                        ],
+                    )
+
+                    if not self.include_file_path:
                         self._data = self._data.drop("file_path")
-                    elif isinstance(self._data, list | tuple):
-                        self._data = [
-                            df.drop("file_path") if isinstance(df, pa.Table) else df
-                            for df in self._data
-                        ]
+                else:
+                    metadata = {}
             else:
                 self._metadata = {}
 
     def to_pandas(
-        self, metadata: bool = False, reload: bool = False, **kwargs
+        self,
+        metadata: bool = False,
+        reload: bool = False,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> (
         tuple[pd.DataFrame | list[pd.DataFrame], dict[str, Any]]
         | pd.DataFrame
@@ -263,12 +324,28 @@ class BaseFileReader(BaseFileIO, gc=False):
         Args:
             metadata (bool, optional): Include metadata in the output. Default is False.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             tuple[pd.DataFrame | list[pd.DataFrame], dict[str, Any]] | pd.DataFrame | list[pd.DataFrame]: Pandas
                 DataFrame or list of DataFrames and optional metadata.
         """
-        self._load(reload=reload, **kwargs)
+        self._load(
+            reload=reload,
+            metadata=metadata,
+            batch_size=None,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if isinstance(self._data, list):
             df = [
                 df if isinstance(df, pd.DataFrame) else df.to_pandas()
@@ -282,26 +359,49 @@ class BaseFileReader(BaseFileIO, gc=False):
                 else self._data.to_pandas()
             )
         if metadata:
-            metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
-            return df, metadata
+            # metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
+            return df, self._metadata
         return df
 
     def iter_pandas(
-        self, reload: bool = False, **kwargs
+        self,
+        reload: bool = False,
+        batch_size: int | None = None,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> Generator[pd.DataFrame, None, None]:
         """Iterate over Pandas DataFrames.
 
         Args:
             batch_size (int, optional): Batch size for iteration. Default is 1.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             Generator[pd.DataFrame, None, None]: Generator of Pandas DataFrames.
         """
-        if self.batch_size is None and "batch_size" not in kwargs:
-            self.batch_size = 1
+        batch_size = batch_size or self.batch_size or 1
+
+        self._load(
+            reload=reload,
+            batch_size=batch_size,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
 
-        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df if isinstance(df, pd.DataFrame) else df.to_pandas()
@@ -313,13 +413,47 @@ class BaseFileReader(BaseFileIO, gc=False):
             )
 
     def _to_polars_dataframe(
-        self, metadata: bool = False, reload: bool = False, **kwargs
+        self,
+        metadata: bool = False,
+        reload: bool = False,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> (
         tuple[pl.DataFrame | list[pl.DataFrame], dict[str, Any]]
         | pl.DataFrame
         | list[pl.DataFrame]
     ):
-        self._load(reload=reload, **kwargs)
+        """Convert data to Polars DataFrame(s).
+
+        Args:
+            metadata (bool, optional): Include metadata in the output. Default is False.
+            reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
+
+        Returns:
+            tuple[pl.DataFrame | list[pl.DataFrame], dict[str, Any]] | pl.DataFrame | list[pl.DataFrame]: Polars
+                DataFrame or list of DataFrames and optional metadata.
+        """
+        self._load(
+            metadata=metadata,
+            reload=reload,
+            batch_size=None,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if isinstance(self._data, list):
             df = [
                 df if isinstance(self._data, pl.DataFrame) else pl.from_arrow(df)
@@ -333,22 +467,48 @@ class BaseFileReader(BaseFileIO, gc=False):
                 else pl.from_arrow(self._data)
             )
         if metadata:
-            metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
-            return df, metadata
+            # metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
+            return df, self._metadata
         return df
 
     def _iter_polars_dataframe(
-        self, reload: bool = False, **kwargs
+        self,
+        reload: bool = False,
+        batch_size: int | None = None,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> Generator[pl.DataFrame, None, None]:
         """Iterate over Polars DataFrames.
 
+        Args:
+            batch_size (int, optional): Batch size for iteration. Default is 1.
+            reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
+
         Returns:
             Generator[pl.DataFrame, None, None]: Generator of Polars DataFrames.
         """
-        if self.batch_size is None and "batch_size" not in kwargs:
-            self.batch_size = 1
-
-        self._load(reload=reload, **kwargs)
+        batch_size = batch_size or self.batch_size or 1
+
+        self._load(
+            reload=reload,
+            batch_size=batch_size,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df if isinstance(df, pl.DataFrame) else pl.from_arrow(df)
@@ -360,38 +520,95 @@ class BaseFileReader(BaseFileIO, gc=False):
             )
 
     def _to_polars_lazyframe(
-        self, metadata: bool = False, reload: bool = False, **kwargs
+        self,
+        metadata: bool = False,
+        reload: bool = False,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> (
         tuple[pl.LazyFrame | list[pl.LazyFrame], dict[str, Any]]
         | pl.LazyFrame
         | list[pl.LazyFrame]
     ):
-        self._load(reload=reload, **kwargs)
+        """Convert data to Polars LazyFrame(s).
+
+        Args:
+            metadata (bool, optional): Include metadata in the output. Default is False.
+            reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
+
+        Returns:
+            tuple[pl.LazyFrame | list[pl.LazyFrame], dict[str, Any]] | pl.LazyFrame | list[pl.LazyFrame]: Polars
+                LazyFrame or list of LazyFrames and optional metadata.
+        """
+        self._load(
+            metadata=metadata,
+            reload=reload,
+            batch_size=None,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if not self.concat:
             df = [df.lazy() for df in self._to_polars_dataframe()]
 
         else:
             df = self._to_polars_dataframe().lazy()
         if metadata:
-            metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
-            return df, metadata
+            # metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
+            return df, self._metadata
         return df
 
     def _iter_polars_lazyframe(
-        self, reload: bool = False, **kwargs
+        self,
+        reload: bool = False,
+        batch_size: int | None = None,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> Generator[pl.LazyFrame, None, None]:
         """Iterate over Polars LazyFrames.
 
         Args:
             batch_size (int, optional): Batch size for iteration. Default is 1.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             Generator[pl.LazyFrame, None, None]: Generator of Polars LazyFrames.
         """
-        if self.batch_size is None and "batch_size" not in kwargs:
-            self.batch_size = 1
-        self._load(reload=reload, **kwargs)
+        batch_size = batch_size or self.batch_size or 1
+
+        self._load(
+            reload=reload,
+            batch_size=batch_size,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield (
@@ -410,6 +627,12 @@ class BaseFileReader(BaseFileIO, gc=False):
         self,
         lazy: bool = False,
         metadata: bool = False,
+        reload: bool = False,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
         **kwargs,
     ) -> (
         pl.DataFrame
@@ -426,6 +649,14 @@ class BaseFileReader(BaseFileIO, gc=False):
         Args:
             lazy (bool, optional): Return a LazyFrame if True, else a DataFrame.
             metadata (bool, optional): Include metadata in the output. Default is False.
+            reload (bool, optional): Reload data if True. Default is False.
+            batch_size (int, optional): Batch size for iteration. Default is 1.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             pl.DataFrame | pl.LazyFrame | list[pl.DataFrame] | list[pl.LazyFrame] | tuple[pl.DataFrame | pl.LazyFrame
@@ -433,32 +664,115 @@ class BaseFileReader(BaseFileIO, gc=False):
                 metadata.
         """
         if lazy:
-            return self._to_polars_lazyframe(**kwargs)
-        return self._to_polars_dataframe(**kwargs)
+            return self._to_polars_lazyframe(
+                metadata=metadata,
+                reload=reload,
+                batch_size=None,
+                include_file_path=include_file_path,
+                concat=concat,
+                use_threads=use_threads,
+                verbose=verbose,
+                opt_dtypes=opt_dtypes,
+                **kwargs,
+            )
+        return self._to_polars_dataframe(
+            metadata=metadata,
+            reload=reload,
+            batch_size=None,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
 
     def iter_polars(
         self,
         lazy: bool = False,
+        reload: bool = False,
+        batch_size: int | None = None,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
         **kwargs,
     ) -> Generator[pl.DataFrame | pl.LazyFrame, None, None]:
+        """Iterate over Polars DataFrames or LazyFrames.
+
+        Args:
+            lazy (bool, optional): Return a LazyFrame if True, else a DataFrame. Default is False.
+            reload (bool, optional): Reload data if True. Default is False.
+            batch_size (int, optional): Batch size for iteration. Default is 1.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single DataFrame. Default is True.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
+
+        Returns:
+            Generator[pl.DataFrame | pl.LazyFrame, None, None]: Generator of Polars DataFrames or LazyFrames.
+        """
         if lazy:
-            yield from self._iter_polars_lazyframe(**kwargs)
-        yield from self._iter_polars_dataframe(**kwargs)
+            yield from self._iter_polars_lazyframe(
+                reload=reload,
+                batch_size=batch_size,
+                include_file_path=include_file_path,
+                concat=concat,
+                use_threads=use_threads,
+                verbose=verbose,
+                opt_dtypes=opt_dtypes,
+                **kwargs,
+            )
+        yield from self._iter_polars_dataframe(
+            reload=reload,
+            batch_size=batch_size,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
 
     def to_pyarrow_table(
-        self, metadata: bool = False, reload: bool = False, **kwargs
+        self,
+        metadata: bool = False,
+        reload: bool = False,
+        include_file_path: bool = False,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> pa.Table | list[pa.Table] | tuple[pa.Table | list[pa.Table], dict[str, Any]]:
         """Convert data to PyArrow Table(s).
 
         Args:
             metadata (bool, optional): Include metadata in the output. Default is False.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             pa.Table | list[pa.Table] | tuple[pa.Table | list[pa.Table], dict[str, Any]]: PyArrow Table or list of
                 Tables and optional metadata.
         """
-        self._load(reload=reload, **kwargs)
+        self._load(
+            reload=reload,
+            metadata=metadata,
+            batch_size=None,
+            include_file_path=include_file_path,
+            concat=None,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if isinstance(self._data, list):
             df = [
                 df.to_arrow(**kwargs) if isinstance(df, pl.DataFrame) else df
@@ -472,22 +786,48 @@ class BaseFileReader(BaseFileIO, gc=False):
                 else self._data
             )
         if metadata:
-            metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
-            return df, metadata
+            # metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
+            return df, self._metadata
         return df
 
     def iter_pyarrow_table(
-        self, reload: bool = False, **kwargs
+        self,
+        reload: bool = False,
+        batch_size: int | None = None,
+        include_file_path: bool = False,
+        concat: bool | None = None,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
+        **kwargs,
     ) -> Generator[pa.Table, None, None]:
         """Iterate over PyArrow Tables.
 
+        Args:
+            reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            concat (bool, optional): Concatenate multiple files into a single Table. Default is True.
+            batch_size (int, optional): Batch size for iteration. Default is 1.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
+
         Returns:
             Generator[pa.Table, None, None]: Generator of PyArrow Tables.
         """
-        if self.batch_size is None and "batch_size" not in kwargs:
-            self.batch_size = 1
-
-        self._load(reload=reload, **kwargs)
+        batch_size = batch_size or self.batch_size or 1
+
+        self._load(
+            reload=reload,
+            batch_size=batch_size,
+            include_file_path=include_file_path,
+            concat=concat,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
+        )
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df.to_arrow(**kwargs) if isinstance(df, pl.DataFrame) else df
@@ -503,6 +843,10 @@ class BaseFileReader(BaseFileIO, gc=False):
         conn: duckdb.DuckDBPyConnection | None = None,
         metadata: bool = False,
         reload: bool = False,
+        include_file_path: bool = False,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
         **kwargs,
     ) -> duckdb.DuckDBPyRelation | tuple[duckdb.DuckDBPyRelation, dict[str, Any]]:
         """Convert data to DuckDB relation.
@@ -511,6 +855,11 @@ class BaseFileReader(BaseFileIO, gc=False):
             conn (duckdb.DuckDBPyConnection, optional): DuckDB connection instance.
             metadata (bool, optional): Include metadata in the output. Default is False.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             duckdb.DuckDBPyRelation | tuple[duckdb.DuckDBPyRelation, dict[str, Any]]: DuckDB relation and optional
@@ -523,10 +872,27 @@ class BaseFileReader(BaseFileIO, gc=False):
 
         if metadata:
             return self._conn.from_arrow(
-                self.to_pyarrow_table(concat=True, reload=reload, **kwargs),
+                self.to_pyarrow_table(
+                    metadata=metadata,
+                    reload=reload,
+                    batch_size=None,
+                    include_file_path=include_file_path,
+                    se_threads=use_threads,
+                    verbose=verbose,
+                    opt_dtypes=opt_dtypes,
+                    **kwargs,
+                ),
             ), self._metadata
         return self._conn.from_arrow(
-            self.to_pyarrow_table(concat=True, reload=reload, **kwargs)
+            self.to_pyarrow_table(
+                reload=reload,
+                batch_size=None,
+                include_file_path=include_file_path,
+                use_threads=use_threads,
+                verbose=verbose,
+                opt_dtypes=opt_dtypes,
+                **kwargs,
+            )
         )
 
     def register_in_duckdb(
@@ -535,6 +901,10 @@ class BaseFileReader(BaseFileIO, gc=False):
         name: str | None = None,
         metadata: bool = False,
         reload: bool = False,
+        include_file_path: bool = False,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
         **kwargs,
     ) -> duckdb.DuckDBPyConnection | tuple[duckdb.DuckDBPyConnection, dict[str, Any]]:
         """Register data in DuckDB.
@@ -544,6 +914,11 @@ class BaseFileReader(BaseFileIO, gc=False):
             name (str, optional): Name for the DuckDB table.
             metadata (bool, optional): Include metadata in the output. Default is False.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
+            kwargs: Additional keyword arguments.
 
         Returns:
             duckdb.DuckDBPyConnection | tuple[duckdb.DuckDBPyConnection, dict[str, Any]]: DuckDB connection instance
@@ -558,7 +933,16 @@ class BaseFileReader(BaseFileIO, gc=False):
             self._conn = conn
 
         self._conn.register(
-            name, self.to_pyarrow_table(concat=True, reload=reload, **kwargs)
+            name,
+            self.to_pyarrow_table(
+                metadata=metadata,
+                reload=reload,
+                include_file_path=include_file_path,
+                use_threads=use_threads,
+                verbose=verbose,
+                opt_dtypes=opt_dtypes,
+                **kwargs,
+            ),
         )
         if metadata:
             return self._conn, self._metadata
@@ -571,6 +955,10 @@ class BaseFileReader(BaseFileIO, gc=False):
         name: str | None = None,
         metadata: bool = False,
         reload: bool = False,
+        include_file_path: bool = False,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
         **kwargs,
     ) -> (
         duckdb.DuckDBPyRelation
@@ -586,6 +974,10 @@ class BaseFileReader(BaseFileIO, gc=False):
             name (str, optional): Name for the DuckDB table.
             metadata (bool, optional): Include metadata in the output. Default is False.
             reload (bool, optional): Reload data if True. Default is False.
+            include_file_path (bool, optional): Include file path in the output. Default is False.
+            use_threads (bool, optional): Use threads for reading data. Default is True.
+            verbose (bool, optional): Verbose output. Default is None.
+            opt_dtypes (bool, optional): Optimize data types. Default is True.
             **kwargs: Additional keyword arguments.
 
         Returns:
@@ -596,10 +988,25 @@ class BaseFileReader(BaseFileIO, gc=False):
         """
         if as_relation:
             return self.to_duckdb_relation(
-                conn=conn, metadata=metadata, reload=reload, **kwargs
+                conn=conn,
+                metadata=metadata,
+                reload=reload,
+                include_file_path=include_file_path,
+                use_threads=use_threads,
+                verbose=verbose,
+                opt_dtypes=opt_dtypes,
+                **kwargs,
             )
         return self.register_in_duckdb(
-            conn=conn, name=name, metadata=metadata, reload=reload, **kwargs
+            conn=conn,
+            name=name,
+            metadata=metadata,
+            reload=reload,
+            include_file_path=include_file_path,
+            use_threads=use_threads,
+            verbose=verbose,
+            opt_dtypes=opt_dtypes,
+            **kwargs,
         )
 
     def register_in_datafusion(
@@ -608,6 +1015,10 @@ class BaseFileReader(BaseFileIO, gc=False):
         name: str | None = None,
         metadata: bool = False,
         reload: bool = False,
+        include_file_path: bool = False,
+        use_threads: bool | None = None,
+        verbose: bool | None = None,
+        opt_dtypes: bool | None = None,
         **kwargs,
     ) -> datafusion.SessionContext | tuple[datafusion.SessionContext, dict[str, Any]]:
         """Register data in DataFusion.
@@ -632,11 +1043,18 @@ class BaseFileReader(BaseFileIO, gc=False):
 
         self._ctx.register_record_batches(
             name,
-            [self.to_pyarrow_table(concat=True, reload=reload, **kwargs).to_batches()],
+            [
+                self.to_pyarrow_table(
+                    reload=reload,
+                    include_file_path=include_file_path,
+                    use_threads=use_threads,
+                    opt_dtypes=opt_dtypes**kwargs,
+                ).to_batches()
+            ],
         )
         if metadata:
             return self._ctx, self._metadata
-        return ctx
+        return self._ctx
 
     def filter(
         self, filter_expr: str | pl.Expr | pa.compute.Expression

{flowerpower-0.11.6.dist-info → flowerpower-0.11.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.6
+Version: 0.11.6.2
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.6.dist-info → flowerpower-0.11.6.2.dist-info}/RECORD

@@ -18,7 +18,7 @@ flowerpower/cli/pipeline.py,sha256=60P6u_QOSgp0jJXEMxazEEo5Sh7-SWFo-Kkuaz21YuI,3
 flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
 flowerpower/fs/__init__.py,sha256=uZaPXErEfQqQRbKRIjkB9yiygd45X5_psYn9-VVrBTQ,910
 flowerpower/fs/base.py,sha256=TqgqBsaFj13O1NpAr8kHuGJ9CTlaSWViMB8Ai_iuCjs,22761
-flowerpower/fs/ext.py,sha256=zTZO-j__O6Om7gbOpXJL7uDo2Cki6hOdlx_GDJ-Xujw,67625
+flowerpower/fs/ext.py,sha256=2NmhSbCIL0qnONMRNPHcPUuR39bGjWpxJE4hNHU5Rvw,69044
 flowerpower/fs/storage_options.py,sha256=msq5TpxAU8tcE_Bxjw6SyxaFa75UjdYnR4-O9U2wmbk,48034
 flowerpower/job_queue/__init__.py,sha256=a25hIqv2xoFKb4JZlyUukS0ppZ9-2sJKH3XAvbk3rlk,10788
 flowerpower/job_queue/base.py,sha256=YwLunDQSyqkSU_vJ69C5SSybJeJP1bAiZ3teUtOchxA,13640
@@ -44,7 +44,7 @@ flowerpower/pipeline/manager.py,sha256=KVpOclUEUAETUNJamJJGuKt3oxCaLitQgxWxkE1q0
 flowerpower/pipeline/registry.py,sha256=6ngmHyKyQsxvIO4qRYxljedY0BE1wE3lpfksEGOzjNs,18963
 flowerpower/pipeline/runner.py,sha256=dsSVYixFXqlxFk8EJfT4wV_7IwgkXq0ErwH_yf_NGS8,25654
 flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
-flowerpower/plugins/io/base.py,sha256=tyFbvx8Ij8gTKP8p8GfwpP5dpIWNncGJfcuK_hPCPN0,79383
+flowerpower/plugins/io/base.py,sha256=d3U5L--SpmowOpXyLTpnvbpaVCeyzoxyiqBbSk2h_K4,96685
 flowerpower/plugins/io/metadata.py,sha256=PCrepLilXRWKDsB5BKFF_-OFs712s1zBeitW-84lDLQ,7005
 flowerpower/plugins/io/helpers/datetime.py,sha256=1WBUg2ywcsodJQwoF6JiIGc9yhVobvE2IErWp4i95m4,10649
 flowerpower/plugins/io/helpers/polars.py,sha256=346DBHG-HvoGZWF-DWxgz7H3KlZu8bFylKIqMOnVJSk,27031
@@ -94,9 +94,9 @@ flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.11.6.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.11.6.dist-info/METADATA,sha256=5Lg0RYLDvqAzJw05z2Qp-OmldA5Cy_FA7XTXlOa2Oos,21610
-flowerpower-0.11.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.11.6.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.11.6.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.11.6.dist-info/RECORD,,
+flowerpower-0.11.6.2.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.11.6.2.dist-info/METADATA,sha256=ftcXBLIRI60sqhug1BnV6KRZY66H0_a65hlamW3COz0,21612
+flowerpower-0.11.6.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.11.6.2.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.11.6.2.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.11.6.2.dist-info/RECORD,,