easymlops 0.0.8__tar.gz → 0.1__tar.gz

{easymlops-0.0.8 → easymlops-0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: easymlops
-Version: 0.0.8
+Version: 0.1
 Summary: MLOps Toolkit In Pipeline
 Home-page: https://github.com/zhulei227/EasyMLOps
 Author: zhulei227

easymlops-0.1/README.rst

@@ -0,0 +1,178 @@
+EasyMLOps
+=========
+
+介绍
+----
+
+``EasyMLOps``\ 包以\ ``Pipline``\ 的方式构建建模任务，可直接进行模型训练、预测(离线，在线)，测试(离线在线预测一致性、预测性能)、特征存储、监控、分析等功能，通过外套一层Flask或FastApi即可直接部署生产，更多内容请访问\ `使用文档1>> <http://www.easymlops.com>`__
+或 `使用文档2>> <http://43.153.61.105>`__
+
+安装
+----
+
+.. code:: bash
+
+   pip install easymlops
+
+使用
+----
+
+.. code:: ipython3
+
+    import pandas as pd
+    data=pd.read_csv("./data/demo.csv")
+    print(data.head(5).to_markdown())
+
+
+.. parsed-literal::
+
+    |    |   PassengerId |   Survived |   Pclass | Name                                                | Sex    |   Age |   SibSp |   Parch | Ticket           |    Fare | Cabin   | Embarked   |
+    |---:|--------------:|-----------:|---------:|:----------------------------------------------------|:-------|------:|--------:|--------:|:-----------------|--------:|:--------|:-----------|
+    |  0 |             1 |          0 |        3 | Braund, Mr. Owen Harris                             | male   |    22 |       1 |       0 | A/5 21171        |  7.25   | nan     | S          |
+    |  1 |             2 |          1 |        1 | Cumings, Mrs. John Bradley (Florence Briggs Thayer) | female |    38 |       1 |       0 | PC 17599         | 71.2833 | C85     | C          |
+    |  2 |             3 |          1 |        3 | Heikkinen, Miss. Laina                              | female |    26 |       0 |       0 | STON/O2. 3101282 |  7.925  | nan     | S          |
+    |  3 |             4 |          1 |        1 | Futrelle, Mrs. Jacques Heath (Lily May Peel)        | female |    35 |       1 |       0 | 113803           | 53.1    | C123    | S          |
+    |  4 |             5 |          0 |        3 | Allen, Mr. William Henry                            | male   |    35 |       0 |       0 | 373450           |  8.05   | nan     | S          |
+    
+
+.. code:: ipython3
+
+    # 拆分训练测试
+    x_train=data[:500]
+    x_test=data[500:]
+    y_train=x_train["Survived"]
+    y_test=x_test["Survived"]
+    del x_train["Survived"]
+    del x_test["Survived"]
+
+表格型任务
+~~~~~~~~~~
+
+.. code:: ipython3
+
+    from easymlops import TablePipeLine
+    from easymlops.table.preprocessing import *
+    from easymlops.table.encoding import *
+    from easymlops.table.decomposition import *
+    from easymlops.table.classification import *
+    from easymlops.table.ensemble import *
+
+.. code:: ipython3
+
+    table = TablePipeLine()
+    table.pipe(FixInput()) \
+      .pipe(FillNa()) \
+      .pipe(Parallel([OneHotEncoding(cols=["Pclass", "Sex"]), LabelEncoding(cols=["Sex", "Pclass"]),
+                        TargetEncoding(cols=["Name", "Ticket", "Embarked", "Cabin", "Sex"], y=y_train)])) \
+      .pipe(Parallel([PCADecomposition(n_components=2, prefix="pca"), NMFDecomposition(n_components=2, prefix="nmf")]))\
+      .pipe(Parallel([LGBMClassification(y=y_train, prefix="lgbm"), LogisticRegressionClassification(y=y_train, prefix="lr")]))
+    
+    table.fit(x_train)
+
+
+
+
+.. parsed-literal::
+
+    <easymlops.table.core.pipeline_object.TablePipeLine at 0x213de1f9988>
+
+
+
+.. code:: ipython3
+
+    print(table.transform(x_test).head(5).to_markdown())
+
+
+.. parsed-literal::
+
+    |     |   lgbm_0 |    lgbm_1 |     lr_0 |     lr_1 |
+    |----:|---------:|----------:|---------:|---------:|
+    | 500 | 0.965218 | 0.0347825 | 0.651417 | 0.348583 |
+    | 501 | 0.98153  | 0.0184698 | 0.65506  | 0.34494  |
+    | 502 | 0.979139 | 0.0208607 | 0.647266 | 0.352734 |
+    | 503 | 0.808796 | 0.191204  | 0.656613 | 0.343387 |
+    | 504 | 0.184484 | 0.815516  | 0.449149 | 0.550851 |
+    
+
+.. code:: ipython3
+
+    table.transform_single(x_test.to_dict("record")[0])
+
+
+
+
+.. parsed-literal::
+
+    {'lgbm_0': 0.9652175316373408,
+     'lgbm_1': 0.03478246836265923,
+     'lr_0': 0.6514166695229407,
+     'lr_1': 0.3485833304770593}
+
+
+
+文本特征提取
+~~~~~~~~~~~~
+
+.. code:: ipython3
+
+    from easymlops import NLPPipeline
+    from easymlops.nlp.preprocessing import *
+    from easymlops.nlp.representation import *
+
+.. code:: ipython3
+
+    nlp = NLPPipeline()
+    nlp.pipe(FixInput()) \
+      .pipe(FillNa()) \
+      .pipe(SelectCols(cols=["Name"]))\
+      .pipe(ReplaceDigits())\
+      .pipe(RemovePunctuation())\
+      .pipe(Parallel([Word2VecModel(embedding_size=4), FastTextModel(embedding_size=4)]))
+    
+    nlp.fit(x_train)
+
+
+
+
+.. parsed-literal::
+
+    <easymlops.nlp.core.pipeline_object.NLPPipeline at 0x213e69dda88>
+
+
+
+.. code:: ipython3
+
+    print(nlp.transform(x_test).head(5).to_markdown())
+
+
+.. parsed-literal::
+
+    |     |   w2v_Name_0 |   w2v_Name_1 |   w2v_Name_2 |   w2v_Name_3 |   fasttext_Name_0 |   fasttext_Name_1 |   fasttext_Name_2 |   fasttext_Name_3 |
+    |----:|-------------:|-------------:|-------------:|-------------:|------------------:|------------------:|------------------:|------------------:|
+    | 500 |   -0.0160304 |   0.00341222 |     0.131879 |    0.227311  |        0.0470462  |       -0.00434371 |        0.0662056  |        0.0268283  |
+    | 501 |   -0.168437  |  -0.0802843  |     0.188844 |    0.0576391 |        0.00569692 |       -0.00443027 |        0.00559943 |        0.0177013  |
+    | 502 |   -0.168437  |  -0.0802843  |     0.188844 |    0.0576391 |       -0.00195689 |       -0.0279951  |        0.00245144 |        0.0229158  |
+    | 503 |   -0.233626  |  -0.181061   |     0.165193 |    0.226518  |        0.00192385 |       -0.0182485  |       -0.005737   |        0.032779   |
+    | 504 |   -0.233626  |  -0.181061   |     0.165193 |    0.226518  |        0.010348   |       -0.0144759  |        0.028474   |        0.00600368 |
+    
+
+.. code:: ipython3
+
+    nlp.transform_single(x_test.to_dict("record")[0])
+
+
+
+
+.. parsed-literal::
+
+    {'w2v_Name_0': -0.01603037677705288,
+     'w2v_Name_1': 0.003412220859900117,
+     'w2v_Name_2': 0.13187919557094574,
+     'w2v_Name_3': 0.227310910820961,
+     'fasttext_Name_0': 0.04704619571566582,
+     'fasttext_Name_1': -0.004343707114458084,
+     'fasttext_Name_2': 0.06620561331510544,
+     'fasttext_Name_3': 0.026828348636627197}
+
+
+

easymlops-0.1/easymlops/__init__.py

@@ -0,0 +1,8 @@
+__version__ = "0.1"
+
+from .table.core import TablePipeLine
+from .nlp.core import NLPPipeline
+
+import warnings
+
+warnings.filterwarnings("ignore")

easymlops-0.1/easymlops/core/__init__.py

@@ -0,0 +1,2 @@
+from .pipe_object import *
+from .pipeline_object import *

easymlops-0.1/easymlops/core/pipe_object.py

@@ -0,0 +1,279 @@
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class PipeObjectBase(object):
+    """
+     所有Pipe以及PipeLine的父类
+    """
+
+    def __init__(self, name=None, **kwargs):
+        """
+        :param name: 该pipe模块名称，默认self.__class__
+        :param kwargs:
+        """
+        super().__init__(**kwargs)
+        if name is None:
+            name = self.__class__
+        self.name = name
+
+    def fit(self, x, **kwargs):
+        """
+        fit:依次调用before_fit,_fit,after_fit
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        self.udf_fit(self.before_fit(x, **kwargs), **kwargs)
+        return self.after_fit(x, **kwargs)
+
+    def before_fit(self, x, **kwargs):
+        """
+        :param x: 入参x
+
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def udf_fit(self, x, **kwargs):
+        """
+        用户自己实现的fit模块
+
+        :param x: 输入x
+        :param kwargs:
+        :return:
+        """
+        return self
+
+    def after_fit(self, x=None, **kwargs):
+        """
+        在udf_fit之后被调用
+
+        :param x: 入参x
+        :param kwargs:
+        :return:
+        """
+        return self
+
+    def transform(self, x, **kwargs):
+        """
+        用于批量数据的transform:依次调用before_transform,udf_transform,after_transform
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return self.after_transform(self.udf_transform(self.before_transform(x, **kwargs), **kwargs), **kwargs)
+
+    def before_transform(self, x, **kwargs):
+        """
+        在udf_transform之前被调用，并将return的结果，作为udf_transform的x
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def udf_transform(self, x, **kwargs):
+        """
+        用户自定义transform
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def after_transform(self, x, **kwargs):
+        """
+        在udf_transform之后被调用
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def transform_single(self, x, **kwargs):
+        """
+        用于单条数据的transform函数:调用顺序before_transform_single,_transform_single,after_transform_single
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return self.after_transform_single(
+            self.udf_transform_single(self.before_transform_single(x, **kwargs), **kwargs), **kwargs)
+
+    def before_transform_single(self, x, **kwargs):
+        """
+        在udf_transform_single之前被调用
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def udf_transform_single(self, x, **kwargs):
+        """
+        用户自定义单条数据预测实现
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def after_transform_single(self, x, **kwargs):
+        """
+        在udf_transform_single之后被调用
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        return x
+
+    def get_params(self):
+        """
+        return 当前pipe需要被保存的所有参数
+
+        :return:
+        """
+        return self.udf_get_params()
+
+    def udf_get_params(self):
+        """
+        return 前pipe需要被保存的参数（不需要考虑父类）
+
+        :return:
+        """
+        return {"name": self.name}
+
+    def set_params(self, params):
+        """
+        恢复所有参数
+
+        :param params:
+        :return:
+        """
+        self.udf_set_params(params)
+
+    def udf_set_params(self, params):
+        """
+        恢复参数（不需要考虑父类）  \n
+
+        :param params:
+        :return:
+        """
+        self.name = params["name"]
+
+    def callback(self, callback_func, data, return_callback_result=False, *args, **kwargs):
+        """
+        回调函数接口  \n
+
+        :param callback_func: 回调函数
+        :param data: 回调入参x
+        :param return_callback_result: 是否返回回调函数结果
+        :param args:
+        :param kwargs:
+        :return:
+        """
+        result = callback_func(self, data, *args, **kwargs)
+        if return_callback_result:
+            return result
+
+    def set_branch_pipe(self, pipe_obj):
+        """
+        挂载支线的pipe模块，不影响主线的执行任务(支线的输出结果不会并入到主线中)，比如存储监控模块等; \n
+        branch_pipe的fit/transform/transform_single运行均在当前挂载对象之后  \n
+
+        :param pipe_obj:
+        :return:
+        """
+        pass
+
+    def get_branch_pipe(self, index):
+        """
+        获取到对应的模块，注意branch pipe可以有多个，所以通过index索引 \n
+
+        :param index:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def remove_branch_pipe(self, index):
+        """
+        移除指定的branch pipe
+
+        :param index:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def set_master_pipe(self, master_pipe):
+        """
+        当前pipe作为branch时所绑定的master的pipe
+
+        :param master_pipe:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def get_master_pipe(self):
+        """
+        当前pipe作为branch时所绑定的master的pipe
+
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def set_parent_pipe(self, parent_pipe=None):
+        """
+        设置父类pipe模块，有时候需要在内部回溯之前的transform操作
+
+        :param parent_pipe:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def get_parent_pipe(self):
+        """
+        返回父类pipe模块
+
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def get_all_parent_pipes(self):
+        """
+        返回所有父类（从头到尾的顺序）
+
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def transform_all_parent(self, x, **kwargs):
+        """
+        顺序运行当前pipe之前父类的transform
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def transform_single_all_parent(self, x, **kwargs):
+        """
+        顺序运行当前pipe之前父类的transform_single
+
+        :param x:
+        :param kwargs:
+        :return:
+        """
+        raise Exception("need to implement!")

easymlops-0.1/easymlops/core/pipeline_object.py

@@ -0,0 +1,46 @@
+from .pipe_object import *
+
+
+class PipeLineBase(PipeObjectBase):
+    """
+    PipeLine设计为Pipe的子类，目的是为了方便嵌套使用;这个类名称虽为PipeLine的Base，但更多是起一个规范作用
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def auto_test(self, x):
+        """
+        测试功能
+
+        :param x:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def save(self, path):
+        """
+        保存模型
+
+        :param path:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def load(self, path):
+        """
+        加载模型
+
+        :param path:
+        :return:
+        """
+        raise Exception("need to implement!")
+
+    def pipe(self, module):
+        """
+        加入pipe模块
+
+        :param module:
+        :return:
+        """
+        raise Exception("need to implement!")

easymlops-0.1/easymlops/nlp/core/__init__.py

@@ -0,0 +1,2 @@
+from .pipe_object import *
+from .pipeline_object import *

easymlops-0.1/easymlops/nlp/core/pipe_object.py

@@ -0,0 +1,10 @@
+from easymlops.table.core import TablePipeObjectBase
+
+
+class NLPPipeObjectBase(TablePipeObjectBase):
+    """
+    由于NLP任务也可以存放于表格数据中，这里借用pandas dataframe做NLP任务
+    """
+
+    def udf_get_params(self):
+        return {}

easymlops-0.1/easymlops/nlp/core/pipeline_object.py

@@ -0,0 +1,10 @@
+from easymlops.table.core import TablePipeLine
+
+
+class NLPPipeline(TablePipeLine):
+    """
+    借用PipeTable做相关任务
+    """
+
+    def udf_get_params(self):
+        return {}