feilian 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

feilian/_dist_ver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 # file generated by setuptools_scm
 # don't change, don't track in version control
-VERSION = (1, 2, 2)
-__version__ = '1.2.2'
+VERSION = (1, 2, 3)
+__version__ = '1.2.3'

feilian/dataframe.py

@@ -25,9 +25,18 @@ if pd_version[0] < 1 or (pd_version[0] == 1 and pd_version[1] < 5):
 FILE_FORMAT = Literal['csv', 'tsv', 'json', 'xlsx', 'parquet']
 COMPRESSION_FORMAT = Literal[None, 'infer', 'snappy', 'gzip', 'brotli', 'bz2', 'zip', 'xz']
 
+def _drop_na_values(data: Union[pd.DataFrame, Dict[str, pd.DataFrame]], axis: Literal['columns', 'rows']):
+    if isinstance(data, pd.DataFrame):
+        data.dropna(axis=axis, how='all', inplace=True)
+    else:
+        assert isinstance(data, dict)
+        for df in data.values():
+            df.dropna(axis=axis, how='all', inplace=True)
+
 def read_dataframe(file: str, *args, sheet_name=0,
                    file_format: FILE_FORMAT = None,
                    jsonl=False, dtype: type = None,
+                   drop_na_columns=False, drop_na_rows=False,
                    **kwargs) -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]:
     """
     read file as pandas `DataFrame`
@@ -37,6 +46,8 @@ def read_dataframe(file: str, *args, sheet_name=0,
     :param file_format:     csv, tsv, json ,xlsx, parquet
     :param jsonl:       jsonl format or not, only used in json format
     :param dtype:       `dtype` for `pd.read_xx()`
+    :param drop_na_columns:     drop column if all values of the column is na
+    :param drop_na_rows:        drop row if all values of the row is na
     :param kwargs:      extra kwargs for `pd.read_xx()`
     """
     # decide the file format
@@ -61,16 +72,23 @@ def read_dataframe(file: str, *args, sheet_name=0,
         jsonl = True
 
     if file_format == 'csv':
-        return pd.read_csv(file, *args, dtype=dtype, **kwargs)
+        df = pd.read_csv(file, *args, dtype=dtype, **kwargs)
     elif file_format == 'xlsx':
-        return pd.read_excel(file, *args, sheet_name=sheet_name, dtype=dtype, **kwargs)
+        df = pd.read_excel(file, *args, sheet_name=sheet_name, dtype=dtype, **kwargs)
     elif file_format == 'json':
-        return pd.read_json(file, *args, lines=jsonl, dtype=dtype, **kwargs)
+        df = pd.read_json(file, *args, lines=jsonl, dtype=dtype, **kwargs)
     elif file_format == 'parquet':
-        return pd.read_parquet(file, *args, **kwargs)
+        df = pd.read_parquet(file, *args, **kwargs)
     else:
         raise IOError(f"Unknown file format: {file}")
 
+    if drop_na_columns:
+        _drop_na_values(df, axis='columns')
+    if drop_na_rows:
+        _drop_na_values(df, axis='rows')
+
+    return df
+
 def save_dataframe(file: Union[str, 'pd.WriteBuffer[bytes]',  'pd.WriteBuffer[str]'],
                    df: Union[pd.DataFrame, Iterable[Union[pd.Series, Dict[str, Any]]]],
                    *args, sheet_name='Sheet1',

feilian/process.py

@@ -1,8 +1,14 @@
 import abc
 import tqdm
 import pandas as pd
-from typing import Any, Dict, Hashable, List, Tuple, Union, Iterable
-from .dataframe import read_dataframe, save_dataframe
+from typing import (
+    Any, Dict, Hashable, List,
+    Tuple, Union, Iterable, Optional,
+)
+from .dataframe import (
+    read_dataframe,
+    save_dataframe,
+)
 
 class BaseProcessor(abc.ABC):
     """
@@ -74,9 +80,10 @@ class DataframeProcessor(BaseProcessor, abc.ABC):
         save_dataframe(filepath, result)
 
     @abc.abstractmethod
-    def process_row(self, i: Hashable, row: pd.Series) -> Dict[str, Any]:
+    def process_row(self, i: Hashable, row: pd.Series) -> Optional[Dict[str, Any]]:
         """
         Process a single row of data.
+        :return:    if `None`, ignore this row
         """
 
     def process(self, data: pd.DataFrame) -> pd.DataFrame:
@@ -84,6 +91,7 @@ class DataframeProcessor(BaseProcessor, abc.ABC):
         if self.progress:
             desc = "process" if self.progress is True else self.progress
             bar = tqdm.tqdm(bar, total=len(data), desc=desc)
-        res = [self.process_row(i, row) for i, row in bar]
+        res = (self.process_row(i, row) for i, row in bar)
+        res = (x for x in res if x is not None)
         return pd.DataFrame(res)
 

{feilian-1.2.2.dist-info → feilian-1.2.3.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: feilian
-Version: 1.2.2
+Version: 1.2.3
 Summary: General data processing tool.
 Author-email: darkpeath <darkpeath@gmail.com>
 Project-URL: Homepage, https://github.com/darkpeath/feilian
 Description-Content-Type: text/markdown
 Requires-Dist: pandas
 Provides-Extra: extra
-Requires-Dist: tqdm ; extra == 'extra'
+Requires-Dist: tqdm; extra == "extra"
 
 # feilian
 

{feilian-1.2.2.dist-info → feilian-1.2.3.dist-info}/RECORD

@@ -1,15 +1,15 @@
 feilian/__init__.py,sha256=97Rvz2O6LknvCWInAjINH4GeqG4-QPPcep9n-V7KD8k,911
-feilian/_dist_ver.py,sha256=SiCl_bfFc54zvOKPEIeqL_n-ebAmqzSF_PdLPL0lPlU,148
+feilian/_dist_ver.py,sha256=WDsMjivFi6Plqpo4JtGV48YaZNQnJ56AduVgKWA5Klo,148
 feilian/arg.py,sha256=n2nIcmC_3rb9A6BOzm9C5z3-T4lnubaGzH2sFhtqwZQ,8402
-feilian/dataframe.py,sha256=DsUiNRuVAe2H6WwqtZ6TGu6WHiLxFEN_Xjl208xvvnw,10698
+feilian/dataframe.py,sha256=BtW6IQrXTF9hQCeccIXkedSjrpfuSKkzWGxd0VxiVHU,11390
 feilian/datetime.py,sha256=IONvWhLeGEy9IVe6GWKEW3FhrfRrShyhGP8-RTf9r3c,763
 feilian/io.py,sha256=aYN3QwWcLoRKzhGMNutqdkmxArVcXfeWXzxCB07LcFc,155
 feilian/json.py,sha256=1FkQ6e4JmbccpxhMobXpsGg-f7uVrTtUmu6jDCXCFTQ,1406
-feilian/process.py,sha256=x3HKXakLriAkQ2jgbQ848wdQLYHQ1WBrOfDu_aUUHRA,3012
+feilian/process.py,sha256=3uig0s_X_espK26tD1cKkapi1gatT1M3zoiFwAws6PU,3153
 feilian/string.py,sha256=G_X3dnR0Oxmi4hXF-6E5jm5M7GPjGoMYrSMyI1dj6Z4,370
 feilian/utils.py,sha256=pzzGEgngidVkYvuNJWY7KWjkdgGRuH5_ENaLS6kxBtk,2648
 feilian/version.py,sha256=oH_DvE7jRCWlCCX9SSadwxwRJXFas_rIisYLBGPYZn4,350
-feilian-1.2.2.dist-info/METADATA,sha256=Pqgl793zKpYn9z0z01MbF84tVra4avtYDP6MIhlgHdw,3723
-feilian-1.2.2.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-feilian-1.2.2.dist-info/top_level.txt,sha256=1Q2-B6KJrcTr7drW_kik35PTVEUJLPP4wVrn0kYKwGw,8
-feilian-1.2.2.dist-info/RECORD,,
+feilian-1.2.3.dist-info/METADATA,sha256=1yull6iP_bypv8etP-9wxoc0zOMdYgXwvlixByFN4xs,3722
+feilian-1.2.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+feilian-1.2.3.dist-info/top_level.txt,sha256=1Q2-B6KJrcTr7drW_kik35PTVEUJLPP4wVrn0kYKwGw,8
+feilian-1.2.3.dist-info/RECORD,,