feilian 1.2.0__py3-none-any.whl → 1.2.2__py3-none-any.whl

feilian/__init__.py

@@ -7,7 +7,7 @@ from .datetime import format_time, format_date
 from .arg import ArgValueParser
 from .json import read_json, save_json
 from .process import DataframeProcessor
-from .utils import flatten_dict
+from .utils import flatten_dict, flatten_list
 from .version import __version__
 
 __all__ = [
@@ -18,6 +18,6 @@ __all__ = [
     'ArgValueParser',
     'read_json', 'save_json',
     'DataframeProcessor',
-    'flatten_dict',
+    'flatten_dict', 'flatten_list',
     '__version__',
 ]

feilian/_dist_ver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 # file generated by setuptools_scm
 # don't change, don't track in version control
-VERSION = (1, 2, 0)
-__version__ = '1.2.0'
+VERSION = (1, 2, 2)
+__version__ = '1.2.2'

feilian/process.py

@@ -1,7 +1,7 @@
 import abc
 import tqdm
 import pandas as pd
-from typing import Any, Dict, Hashable
+from typing import Any, Dict, Hashable, List, Tuple, Union, Iterable
 from .dataframe import read_dataframe, save_dataframe
 
 class BaseProcessor(abc.ABC):
@@ -10,10 +10,25 @@ class BaseProcessor(abc.ABC):
     """
 
     @abc.abstractmethod
-    def read_data(self, filepath: str) -> Any:
+    def read_single_file(self, filepath: str) -> Any:
+        """
+        Actual method to read data from a single file.
+        """
+
+    def merge_input_data(self, data: Iterable[Any]) -> Any:
+        """
+        Merge data read from multi files.
+        """
+        return data
+
+    def read_data(self, filepath: Union[str, List[str], Tuple[str]]) -> Any:
         """
         Read data from input file.
         """
+        if isinstance(filepath, (list, tuple)):
+            return self.merge_input_data(self.read_single_file(x) for x in filepath)
+        else:
+            return self.read_single_file(filepath)
 
     @abc.abstractmethod
     def save_result(self, filepath: str, result: Any):
@@ -27,7 +42,7 @@ class BaseProcessor(abc.ABC):
         Process data and return result.
         """
 
-    def run(self, input_path: str, output_path: str = None, write_output=True):
+    def run(self, input_path: Union[str, List[str], Tuple[str]], output_path: str = None, write_output=True):
         """
         Read from a file, and save result to another file.
         :param input_path:      file with the data
@@ -40,12 +55,20 @@ class BaseProcessor(abc.ABC):
             self.save_result(output_path or input_path, result)
 
 class DataframeProcessor(BaseProcessor, abc.ABC):
-    def __init__(self, input_dtype=None, progress=False):
-        self.input_dtype = input_dtype
+    def __init__(self, input_dtype=None, progress=False, read_args: Dict[str, Any] = None):
         self.progress = progress
+        self.read_args = read_args or {}
+        if input_dtype is not None:
+            self.read_args['dtype'] = input_dtype
+
+    def read_single_file(self, filepath: str) -> pd.DataFrame:
+        return read_dataframe(filepath, **self.read_args)
+
+    def merge_input_data(self, data: Iterable[pd.DataFrame]) -> pd.DataFrame:
+        return pd.concat(data)
 
-    def read_data(self, filepath: str) -> pd.DataFrame:
-        return read_dataframe(filepath, dtype=self.input_dtype)
+    def read_data(self, filepath: Union[str, List[str], Tuple[str]]) -> pd.DataFrame:
+        return super().read_data(filepath)
 
     def save_result(self, filepath: str, result: pd.DataFrame):
         save_dataframe(filepath, result)

feilian/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from typing import Dict, Any, Union, Collection
+from typing import Dict, Any, Union, Collection, List
 
 def flatten_dict(data: Dict[str, Any], prefix="", joiner=".",
                  exclude: Union[None, str, Collection[str]] = None,
@@ -62,3 +62,21 @@ def flatten_dict(data: Dict[str, Any], prefix="", joiner=".",
 
     return res
 
+
+def flatten_list(data: List[Any], res: List[Any] = None) -> List[Any]:
+    """
+    Flatten nested list as a flat on layer list.
+    :param data:    a nested list
+    :param res:     the result flat layer list, create a new one if not given.
+    """
+    if res is None:
+        res = []
+
+    for x in data:
+        if isinstance(x, list):
+            # flatten recursively
+            flatten_list(x, res)
+        else:
+            res.append(x)
+
+    return res

{feilian-1.2.0.dist-info → feilian-1.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: feilian
-Version: 1.2.0
+Version: 1.2.2
 Summary: General data processing tool.
 Author-email: darkpeath <darkpeath@gmail.com>
 Project-URL: Homepage, https://github.com/darkpeath/feilian

feilian-1.2.2.dist-info/RECORD

@@ -0,0 +1,15 @@
+feilian/__init__.py,sha256=97Rvz2O6LknvCWInAjINH4GeqG4-QPPcep9n-V7KD8k,911
+feilian/_dist_ver.py,sha256=SiCl_bfFc54zvOKPEIeqL_n-ebAmqzSF_PdLPL0lPlU,148
+feilian/arg.py,sha256=n2nIcmC_3rb9A6BOzm9C5z3-T4lnubaGzH2sFhtqwZQ,8402
+feilian/dataframe.py,sha256=DsUiNRuVAe2H6WwqtZ6TGu6WHiLxFEN_Xjl208xvvnw,10698
+feilian/datetime.py,sha256=IONvWhLeGEy9IVe6GWKEW3FhrfRrShyhGP8-RTf9r3c,763
+feilian/io.py,sha256=aYN3QwWcLoRKzhGMNutqdkmxArVcXfeWXzxCB07LcFc,155
+feilian/json.py,sha256=1FkQ6e4JmbccpxhMobXpsGg-f7uVrTtUmu6jDCXCFTQ,1406
+feilian/process.py,sha256=x3HKXakLriAkQ2jgbQ848wdQLYHQ1WBrOfDu_aUUHRA,3012
+feilian/string.py,sha256=G_X3dnR0Oxmi4hXF-6E5jm5M7GPjGoMYrSMyI1dj6Z4,370
+feilian/utils.py,sha256=pzzGEgngidVkYvuNJWY7KWjkdgGRuH5_ENaLS6kxBtk,2648
+feilian/version.py,sha256=oH_DvE7jRCWlCCX9SSadwxwRJXFas_rIisYLBGPYZn4,350
+feilian-1.2.2.dist-info/METADATA,sha256=Pqgl793zKpYn9z0z01MbF84tVra4avtYDP6MIhlgHdw,3723
+feilian-1.2.2.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+feilian-1.2.2.dist-info/top_level.txt,sha256=1Q2-B6KJrcTr7drW_kik35PTVEUJLPP4wVrn0kYKwGw,8
+feilian-1.2.2.dist-info/RECORD,,

{feilian-1.2.0.dist-info → feilian-1.2.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.2.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

feilian-1.2.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-feilian/__init__.py,sha256=G5SKKbmcX8Um6BJ1-nZvMrXhiJnWJbDPMyBjhKkWWAc,881
-feilian/_dist_ver.py,sha256=U8gcehSDq7YjTNBnP2g7zZAOcoHw9_WLrgCWQX4QAUY,148
-feilian/arg.py,sha256=n2nIcmC_3rb9A6BOzm9C5z3-T4lnubaGzH2sFhtqwZQ,8402
-feilian/dataframe.py,sha256=DsUiNRuVAe2H6WwqtZ6TGu6WHiLxFEN_Xjl208xvvnw,10698
-feilian/datetime.py,sha256=IONvWhLeGEy9IVe6GWKEW3FhrfRrShyhGP8-RTf9r3c,763
-feilian/io.py,sha256=aYN3QwWcLoRKzhGMNutqdkmxArVcXfeWXzxCB07LcFc,155
-feilian/json.py,sha256=1FkQ6e4JmbccpxhMobXpsGg-f7uVrTtUmu6jDCXCFTQ,1406
-feilian/process.py,sha256=GLkmogYnhkxi6qf-JX-FwIlJAHmTynmgB7zSAggEe1E,2080
-feilian/string.py,sha256=G_X3dnR0Oxmi4hXF-6E5jm5M7GPjGoMYrSMyI1dj6Z4,370
-feilian/utils.py,sha256=DqBKjpRBbSNipRDau9sYnoCfSDR_Of-xTOCoNxGWUJk,2180
-feilian/version.py,sha256=oH_DvE7jRCWlCCX9SSadwxwRJXFas_rIisYLBGPYZn4,350
-feilian-1.2.0.dist-info/METADATA,sha256=P-6B1-dzIzMm-4tQU_Ew7azWeMOo9X8pKTUrAF9rWwI,3723
-feilian-1.2.0.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-feilian-1.2.0.dist-info/top_level.txt,sha256=1Q2-B6KJrcTr7drW_kik35PTVEUJLPP4wVrn0kYKwGw,8
-feilian-1.2.0.dist-info/RECORD,,