feilian 1.1.8__py3-none-any.whl → 1.2.0__py3-none-any.whl

feilian/__init__.py

@@ -6,6 +6,8 @@ from .dataframe import is_empty_text, is_nonempty_text, is_blank_text, is_non_bl
 from .datetime import format_time, format_date
 from .arg import ArgValueParser
 from .json import read_json, save_json
+from .process import DataframeProcessor
+from .utils import flatten_dict
 from .version import __version__
 
 __all__ = [
@@ -15,5 +17,7 @@ __all__ = [
     'format_time', 'format_date',
     'ArgValueParser',
     'read_json', 'save_json',
+    'DataframeProcessor',
+    'flatten_dict',
     '__version__',
 ]

feilian/_dist_ver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 # file generated by setuptools_scm
 # don't change, don't track in version control
-VERSION = (1, 1, 8)
-__version__ = '1.1.8'
+VERSION = (1, 2, 0)
+__version__ = '1.2.0'

feilian/dataframe.py

@@ -158,6 +158,7 @@ def save_dataframe(file: Union[str, 'pd.WriteBuffer[bytes]',  'pd.WriteBuffer[st
     elif file_format == 'json':
         if jsonl:
             orient = 'records'
+        if orient not in ['split', 'table']:
             index = True
         df.to_json(file, *args, compression=compression, index=index,
                    force_ascii=force_ascii, orient=orient, lines=jsonl,

feilian/json.py

@@ -4,10 +4,16 @@ from typing import Dict, List, Union, Any
 import json
 from .io import ensure_parent_dir_exist
 
-def read_json(filepath: str, jsonl=False, encoding='utf-8', **kwargs):
+def _is_jsonl(filepath: str, jsonl=None) -> bool:
+    if jsonl is None:
+        jsonl = filepath.lower().endswith('.jsonl')
+    return jsonl
+
+def read_json(filepath: str, jsonl=None, encoding='utf-8', **kwargs):
     """
     An agent for `json.load()` with some default value.
     """
+    jsonl = _is_jsonl(filepath, jsonl)
     with open(filepath, encoding=encoding) as f:
         if jsonl:
             return [json.loads(x) for x in f]
@@ -19,6 +25,7 @@ def save_json(filepath: str, data: Union[Dict[str, Any], List[Any]], jsonl=False
     """
     An agent for `json.dump()` with some default value.
     """
+    jsonl = _is_jsonl(filepath, jsonl)
     if jsonl and not isinstance(data, list):
         # data should be a list
         raise ValueError("data should be a list when save as jsonl format")

feilian/process.py

@@ -0,0 +1,66 @@
+import abc
+import tqdm
+import pandas as pd
+from typing import Any, Dict, Hashable
+from .dataframe import read_dataframe, save_dataframe
+
+class BaseProcessor(abc.ABC):
+    """
+    Base class for processing data.
+    """
+
+    @abc.abstractmethod
+    def read_data(self, filepath: str) -> Any:
+        """
+        Read data from input file.
+        """
+
+    @abc.abstractmethod
+    def save_result(self, filepath: str, result: Any):
+        """
+        Save result to output file.
+        """
+
+    @abc.abstractmethod
+    def process(self, data: Any) -> Any:
+        """
+        Process data and return result.
+        """
+
+    def run(self, input_path: str, output_path: str = None, write_output=True):
+        """
+        Read from a file, and save result to another file.
+        :param input_path:      file with the data
+        :param output_path:     where to save the result, if not given, use input_path
+        :param write_output:    whether to write the result to the output_file
+        """
+        data = self.read_data(input_path)
+        result = self.process(data)
+        if write_output:
+            self.save_result(output_path or input_path, result)
+
+class DataframeProcessor(BaseProcessor, abc.ABC):
+    def __init__(self, input_dtype=None, progress=False):
+        self.input_dtype = input_dtype
+        self.progress = progress
+
+    def read_data(self, filepath: str) -> pd.DataFrame:
+        return read_dataframe(filepath, dtype=self.input_dtype)
+
+    def save_result(self, filepath: str, result: pd.DataFrame):
+        save_dataframe(filepath, result)
+
+    @abc.abstractmethod
+    def process_row(self, i: Hashable, row: pd.Series) -> Dict[str, Any]:
+        """
+        Process a single row of data.
+        """
+
+    def process(self, data: pd.DataFrame) -> pd.DataFrame:
+        bar = data.iterrows()
+        if self.progress:
+            desc = "process" if self.progress is True else self.progress
+            bar = tqdm.tqdm(bar, total=len(data), desc=desc)
+        res = [self.process_row(i, row) for i, row in bar]
+        return pd.DataFrame(res)
+

feilian/utils.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from typing import Dict, Any, Union, Collection
+
+def flatten_dict(data: Dict[str, Any], prefix="", joiner=".",
+                 exclude: Union[None, str, Collection[str]] = None,
+                 frozen: Union[None, str, Collection[str]] = None,
+                 empty_as_default=False, empty_value=None,
+                 res: Dict[str, Any] = None) -> Dict[str, Any]:
+    """
+    flatten dict as a flat one layer dict
+    :param data:      origin dict
+    :param prefix:    prefix for key in the dict
+    :param joiner:    join symbol for different layer key
+    :param exclude:   prefix to be excluded from result
+    :param frozen:    keys not to be flattened
+    :param empty_as_default:  should set a default value if value is an empty dict
+    :param empty_value:       if `empty_as_default` is `True`, used as the default value for empty dict
+    :param res:       the result flat layer dict, create a new one if not given.
+    """
+    if res is None:
+        res = {}
+    if isinstance(exclude, str):
+        exclude = {exclude}
+    if isinstance(frozen, str):
+        frozen = {frozen}
+
+    # all keys are start with the prefix, ignore data
+    if exclude and prefix in exclude:
+        return res
+
+    # all keys in data should be frozen
+    if frozen and prefix in frozen:
+        for k, v in data.items():
+            res[prefix+k] = v
+        return res
+
+    for k, v in data.items():
+        k = prefix + k
+
+        if exclude and k in exclude:
+            # only the key should be excluded
+            continue
+
+        if frozen and k in frozen:
+            # frozen key, keep it as original value
+            res[k] = v
+            continue
+
+        if isinstance(v, dict):
+            if len(v) == 0:
+                # empty dict, set as default value if set
+                if empty_as_default:
+                    res[k] = empty_value
+            else:
+                # value is a dict, flatten recursively
+                flatten_dict(v, prefix=k+joiner, joiner=joiner, exclude=exclude, frozen=frozen, res=res)
+        else:
+            # normal value, keep it as original value
+            res[k] = v
+
+    return res
+

feilian-1.2.0.dist-info/METADATA

@@ -0,0 +1,210 @@
+Metadata-Version: 2.1
+Name: feilian
+Version: 1.2.0
+Summary: General data processing tool.
+Author-email: darkpeath <darkpeath@gmail.com>
+Project-URL: Homepage, https://github.com/darkpeath/feilian
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Provides-Extra: extra
+Requires-Dist: tqdm ; extra == 'extra'
+
+# feilian
+
+General data processing tool.
+
+## Features
+
+- More default values, less necessary arg.
+- Encapsulation of panda dataframe for simple usage.
+
+## Usage
+
+### Process data with pandas dataframe
+
+#### Read a file as dataframe
+
+```python
+import feilian
+
+input_file = ''     # file can be any csv, json, parquet or xlsx format
+df = feilian.read_dataframe(input_file)
+```
+
+#### Write dataframe to a file
+
+```python
+import feilian
+import pandas as pd
+
+df = pd.DataFrame(dict(a=[1, 2, 3], b=[4, 5, 6]))
+output_file = ''  # file can be any csv, json, parquet or xlsx format
+feilian.save_dataframe(output_file, df)
+```
+
+#### Iter a dataframe with a progress bar
+
+```python
+import feilian
+import pandas as pd
+
+df = pd.DataFrame(dict(a=[1, 2, 3], b=[4, 5, 6]))
+feilian.iter_dataframe(data=df, progress_bar="process")
+```
+
+#### Extract sample from a dataframe
+
+```python
+import feilian
+import pandas as pd
+
+df = pd.DataFrame(dict(a=[1, 2, 3], b=[4, 5, 6]))
+sample = feilian.extract_dataframe_sample(size=2, shuffle=True)
+```
+
+#### Test text value in dataframe
+
+```python
+import feilian
+
+s = ''
+
+# test if s is na or empty string
+feilian.is_empty_text(s)
+
+# test if s is not na and non-empty string
+feilian.is_nonempty_text(s)
+
+# test if s is na or blank string
+feilian.is_blank_text(s)
+
+# test if s is not na and non-blank string
+feilian.is_non_blank_text(s)
+```
+
+#### Merge same id rows to one row
+
+```python
+import feilian
+import pandas as pd
+
+df = pd.DataFrame([
+    {"a": "1", "b": "2", "c": "5"},
+    {"a": "2", "b": 6, "c": "8"},
+    {"a": "1", "b": 8, "c": "9"},
+])
+
+res = feilian.merge_dataframe_rows(df, col_id="a", join_sep=",")
+```
+
+### IO for json file
+
+#### Read a json file
+
+```python
+import feilian
+
+input_file = ''
+data = feilian.read_json(input_file)
+```
+
+#### Write a json file
+
+```python
+import feilian
+
+data = [
+    {"a": "1", "b": "2", "c": "5"},
+    {"a": "2", "b": 6, "c": "8"},
+    {"a": "1", "b": 8, "c": "9"},
+]
+output_file = ''
+feilian.save_dataframe(output_file, data)
+```
+
+### Datetime format
+
+```python
+import feilian
+import datetime
+
+d = datetime.datetime.now()
+
+# format a date string
+feilian.format_date(d, sep='-')
+
+# format a time string
+feilian.format_time(d, fmt='%H:%M:%S')
+```
+
+### Process dict
+
+#### Flatten dict value
+
+```python
+import feilian
+
+data = {
+    "a": 12,
+    "b": ["4", "s"],
+    "c": {
+        "l": 0,
+        "j": {
+            "se": "we",
+            "t": 5,
+        }
+    },
+    "f": 7,
+    "g": {
+        "ts": "9w",
+        "j2": 8,
+    },
+    "w": {
+        "s": {
+            "ge": 89,
+            "00": "ej",
+        },
+        "r": {
+            "le": 33,
+            "03": "ef",
+        }
+    },
+    "sk": {
+        "a": "23",
+        "b": {
+            "s": 9,
+            "g": 0,
+            "p": 4,
+        },
+        "c": {
+            "s": 8,
+            "t": "w",
+            "j": "23",
+        }
+    },
+}
+res = feilian.flatten_dict(data, frozen={"g", "w.s", "sk."}, exclude="f")
+```
+
+### Process args 
+
+```python
+from feilian import ArgValueParser
+
+value = ''
+
+# split value
+ArgValueParser.split_strs_to_list(value)
+ArgValueParser.split_strs_to_set(value)
+
+# bound value
+ArgValueParser.bound_set_if_singleton(value)
+ArgValueParser.bound_tuple_if_singleton(value)
+ArgValueParser.bound_list_if_singleton(value)
+
+# force value type
+ArgValueParser.ensure_set(value)
+ArgValueParser.ensure_list(value)
+ArgValueParser.ensure_tuple(value)
+```
+

feilian-1.2.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+feilian/__init__.py,sha256=G5SKKbmcX8Um6BJ1-nZvMrXhiJnWJbDPMyBjhKkWWAc,881
+feilian/_dist_ver.py,sha256=U8gcehSDq7YjTNBnP2g7zZAOcoHw9_WLrgCWQX4QAUY,148
+feilian/arg.py,sha256=n2nIcmC_3rb9A6BOzm9C5z3-T4lnubaGzH2sFhtqwZQ,8402
+feilian/dataframe.py,sha256=DsUiNRuVAe2H6WwqtZ6TGu6WHiLxFEN_Xjl208xvvnw,10698
+feilian/datetime.py,sha256=IONvWhLeGEy9IVe6GWKEW3FhrfRrShyhGP8-RTf9r3c,763
+feilian/io.py,sha256=aYN3QwWcLoRKzhGMNutqdkmxArVcXfeWXzxCB07LcFc,155
+feilian/json.py,sha256=1FkQ6e4JmbccpxhMobXpsGg-f7uVrTtUmu6jDCXCFTQ,1406
+feilian/process.py,sha256=GLkmogYnhkxi6qf-JX-FwIlJAHmTynmgB7zSAggEe1E,2080
+feilian/string.py,sha256=G_X3dnR0Oxmi4hXF-6E5jm5M7GPjGoMYrSMyI1dj6Z4,370
+feilian/utils.py,sha256=DqBKjpRBbSNipRDau9sYnoCfSDR_Of-xTOCoNxGWUJk,2180
+feilian/version.py,sha256=oH_DvE7jRCWlCCX9SSadwxwRJXFas_rIisYLBGPYZn4,350
+feilian-1.2.0.dist-info/METADATA,sha256=P-6B1-dzIzMm-4tQU_Ew7azWeMOo9X8pKTUrAF9rWwI,3723
+feilian-1.2.0.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+feilian-1.2.0.dist-info/top_level.txt,sha256=1Q2-B6KJrcTr7drW_kik35PTVEUJLPP4wVrn0kYKwGw,8
+feilian-1.2.0.dist-info/RECORD,,

{feilian-1.1.8.dist-info → feilian-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

feilian-1.1.8.dist-info/METADATA

@@ -1,41 +0,0 @@
-Metadata-Version: 2.1
-Name: feilian
-Version: 1.1.8
-Summary: General data processing tool.
-Author-email: darkpeath <darkpeath@gmail.com>
-Project-URL: Homepage, https://github.com/darkpeath/feilian
-Description-Content-Type: text/markdown
-Requires-Dist: pandas
-Provides-Extra: extra
-Requires-Dist: tqdm ; extra == 'extra'
-
-# feilian
-
-General data processing tool.
-
-## Features
-
-- More default values, less necessary arg.
-- Encapsulation of panda dataframe for simple usage.
-
-## Usage
-
-### Read a file as dataframe
-
-```python
-from feilian import read_dataframe
-
-input_file = ''     # file can be any csv, json or xlsx format
-df = read_dataframe(input_file)
-```
-
-### Write dataframe to a file
-
-```python
-import pandas as pd
-from feilian import save_dataframe
-
-df = pd.DataFrame(dict(a=[1, 2, 3], b=[4, 5, 6]))
-output_file = ''  # file can be any csv, json or xlsx format
-save_dataframe(output_file, df)
-```

feilian-1.1.8.dist-info/RECORD

@@ -1,13 +0,0 @@
-feilian/__init__.py,sha256=D_OYSAfDALQnRsbaAXMnHyC0LtykaIXLTHq7XNcsYjU,763
-feilian/_dist_ver.py,sha256=nHzOoqCTSUUNWuG27m4tgLUVSw_cZ9edFvXao_UemJA,148
-feilian/arg.py,sha256=n2nIcmC_3rb9A6BOzm9C5z3-T4lnubaGzH2sFhtqwZQ,8402
-feilian/dataframe.py,sha256=G7Ai_JsMS7kNfqRNptqGOOxGWjyYwcJrT73IttDO1vo,10653
-feilian/datetime.py,sha256=IONvWhLeGEy9IVe6GWKEW3FhrfRrShyhGP8-RTf9r3c,763
-feilian/io.py,sha256=aYN3QwWcLoRKzhGMNutqdkmxArVcXfeWXzxCB07LcFc,155
-feilian/json.py,sha256=PSjDJ3MCdolKwfAOmT9DuS8KnJZo9oGABKgJDduCliU,1187
-feilian/string.py,sha256=G_X3dnR0Oxmi4hXF-6E5jm5M7GPjGoMYrSMyI1dj6Z4,370
-feilian/version.py,sha256=oH_DvE7jRCWlCCX9SSadwxwRJXFas_rIisYLBGPYZn4,350
-feilian-1.1.8.dist-info/METADATA,sha256=VIBIvtc9CJxD39A5m6be_sQqYD5zRXuWOftRPBjgvDo,902
-feilian-1.1.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-feilian-1.1.8.dist-info/top_level.txt,sha256=1Q2-B6KJrcTr7drW_kik35PTVEUJLPP4wVrn0kYKwGw,8
-feilian-1.1.8.dist-info/RECORD,,