ddeutil-workflow 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

{ddeutil_workflow-0.0.1.dist-info → ddeutil_workflow-0.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.1
+Version: 0.0.3
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -23,15 +23,14 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
-Requires-Dist: ddeutil-model
 Requires-Dist: python-dotenv
 Provides-Extra: test
 Requires-Dist: sqlalchemy ==2.0.30 ; extra == 'test'
 Requires-Dist: paramiko ==3.4.0 ; extra == 'test'
 Requires-Dist: sshtunnel ==0.4.0 ; extra == 'test'
-Requires-Dist: boto3 ==1.34.109 ; extra == 'test'
+Requires-Dist: boto3 ==1.34.117 ; extra == 'test'
 Requires-Dist: fsspec ==2024.5.0 ; extra == 'test'
-Requires-Dist: polars ==0.20.26 ; extra == 'test'
+Requires-Dist: polars ==0.20.31 ; extra == 'test'
 Requires-Dist: pyarrow ==16.1.0 ; extra == 'test'
 
 # Data Utility: _Workflow_
@@ -83,6 +82,15 @@ The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
+The main feature of this project is the `Pipeline` object that can call any
+registries function. The pipeline can handle everything that you want to do, it
+will passing parameters and catching the output for re-use it to next step.
+
+> [!IMPORTANT]
+> In the future of this project, I will drop the connection and dataset to
+> dynamic registries instead of main features because it have a lot of maintain
+> vendor codes and deps. (I do not have time to handle this features)
+
 ### Connection
 
 The connection for worker able to do any thing.
@@ -102,7 +110,9 @@ assert conn.ping()
 
 ### Dataset
 
-The dataset is define any objects on the connection.
+The dataset is define any objects on the connection. This feature was implemented
+on `/vendors` because it has a lot of tools that can interact with any data systems
+in the data tool stacks.
 
 ```yaml
 ds_postgres_customer_tbl:
@@ -114,7 +124,7 @@ ds_postgres_customer_tbl:
 ```
 
 ```python
-from ddeutil.workflow.dataset import PostgresTbl
+from ddeutil.workflow.vendors.pg import PostgresTbl
 
 dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
 assert dataset.exists()
@@ -124,14 +134,14 @@ assert dataset.exists()
 
 ```yaml
 schd_for_node:
-  type: schedule.Scdl
+  type: schedule.Schedule
   cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.schedule import Scdl
+from ddeutil.workflow.schedule import Schedule
 
-scdl = Scdl.from_loader(name='schd_for_node', externals={})
+scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
 
 cron_iterate = scdl.generate('2022-01-01 00:00:00')
@@ -155,8 +165,10 @@ The state of doing lists that worker should to do. It be collection of the stage
 run_py_local:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    author-run: utils.receive.string
-    run-date: utils.receive.datetime
+    author-run:
+      type: str
+    run-date:
+      type: datetime
   jobs:
     first-job:
       stages:
@@ -203,13 +215,15 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 pipe_el_pg_to_lake:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run-date: utils.receive.datetime
-    author-email: utils.receive.string
+    run-date:
+      type: datetime
+    author-email:
+      type: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
-          id: extract
+          id: extract-load
           task: tasks/postgres-to-delta@polars
           with:
             source:
@@ -222,21 +236,22 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Hooks (Transform)
+### Tasks (Transform)
 
 ```yaml
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date: utils.receive.datetime
-    sp_name: utils.receive.string
-    source_name: utils.receive.string
-    target_name: utils.receive.string
+    run_date: datetime
+    sp_name: str
+    source_name: str
+    target_name: str
   jobs:
     transform:
       stages:
         - name: "Transform Data in MS SQL Server"
-          hook: hooks/mssql-proc@odbc
+          id: transform
+          task: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:

ddeutil_workflow-0.0.3.dist-info/RECORD

@@ -0,0 +1,29 @@
+ddeutil/workflow/__about__.py,sha256=smA9c0CTLewINRoxj2VBHoiYDESoFGtXYFDvRT31dgs,27
+ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
+ddeutil/workflow/__types.py,sha256=AkpQq6QlrclpurCZZVY9RMxoyS9z2WGzhaz_ikeTaCU,453
+ddeutil/workflow/conn.py,sha256=POtNcyqFNGxZnkg5J_H1OIvQVnnqG-ajmBBzjoHl9sg,7238
+ddeutil/workflow/exceptions.py,sha256=XAq82VHSMLNb4UjGatp7hYfjxFtMiKFtBqJyAhwTl-s,434
+ddeutil/workflow/loader.py,sha256=TXS4k2dqNycBYSTYcJ80WIsPMKNZbHNeBbcufX6lrJc,5483
+ddeutil/workflow/pipeline.py,sha256=fG6ta-SNx4OWS6n8w7YpYDadfnbqayj8A1uY03TvLUA,16942
+ddeutil/workflow/schedule.py,sha256=RMbTC7L32D3fJ5gYxJDCn-vPr2RYEBMSD0G2kj1Qows,2712
+ddeutil/workflow/utils.py,sha256=z7evB9kOsgTr30uVuL994bmOMDNZB5xDY2KjO7gL1dc,5379
+ddeutil/workflow/tasks/__init__.py,sha256=TIcw9JinrdepWgyazSMLk_QflUFms99ILI4GvLHUGD0,338
+ddeutil/workflow/tasks/_pandas.py,sha256=rqz5_VMSqkEdirk7i3EElZoqnRYFyyK_Z8_Zt8FyeTg,1693
+ddeutil/workflow/tasks/_polars.py,sha256=SYEBx-0I9tbY046QGSMokVugK8Fqjhiw4dzpL6y6Hww,2917
+ddeutil/workflow/vendors/__dataset.py,sha256=n9EwTIoVlgCKyCQQgQrijeoQgauOoKtdKiWCzSIErns,4065
+ddeutil/workflow/vendors/__dict.py,sha256=ETwkeA0qzKNgedfeRgAz1qShNXTIXIS4DXzJB4lM4jo,9962
+ddeutil/workflow/vendors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/vendors/__schedule.py,sha256=cUIwtTli9G0klrNFxAIbG3VTiv6umRuNAZiKA-kOtpE,20690
+ddeutil/workflow/vendors/aws.py,sha256=zjq_LCu3ffVBRrxS2vqss9X24yrtuAEt9ouy2_WvS0o,5980
+ddeutil/workflow/vendors/az.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/vendors/minio.py,sha256=pScLy38Du9moOrGaSBSFsoQRhiQ686FQyloOeLA0OQk,261
+ddeutil/workflow/vendors/pd.py,sha256=J6Nkb4RqUnz3NMfo3cHX-Udw3HPjqjUimojS86rR4os,116
+ddeutil/workflow/vendors/pg.py,sha256=TGwkV6nsarGLbiRTT_wB4uAy3xCR89EPPCMWqlWhFe8,422
+ddeutil/workflow/vendors/pl.py,sha256=B-l9zcZ9vATAKVMLv5tjKiWo5Qt8ZIv_aQzuVFinKbY,5087
+ddeutil/workflow/vendors/sftp.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
+ddeutil_workflow-0.0.3.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.3.dist-info/METADATA,sha256=KcsTd-FjufMK-4fhiIq27yeQUuA7NeB8TCkbXADQ1Dc,7992
+ddeutil_workflow-0.0.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+ddeutil_workflow-0.0.3.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.3.dist-info/RECORD,,

ddeutil/workflow/hooks/__init__.py

@@ -1,9 +0,0 @@
-from typing import Any
-
-from ddeutil.core import lazy
-
-registries: dict[str, Any] = {
-    "postgres-proc": {
-        "pysycopg": lazy("ddeutil.workflow.tasks._postgres.postgres_procedure"),
-    },
-}

ddeutil/workflow/hooks/_postgres.py

@@ -1,2 +0,0 @@
-def postgres_procedure():
-    return

ddeutil/workflow/utils/receive.py

@@ -1,33 +0,0 @@
-import datetime as dt
-from typing import Any
-
-
-def datetime(value: Any) -> dt.datetime:
-    if isinstance(value, dt.datetime):
-        return value
-    elif isinstance(value, dt.date):
-        return dt.datetime(value.year, value.month, value.day)
-    if value is None:
-        return dt.datetime.now(dt.timezone.utc)
-    elif not isinstance(value, str):
-        raise ValueError(
-            f"Value that want to convert to datetime does not support for "
-            f"type: {type(value)}"
-        )
-    return dt.datetime.fromisoformat(value)
-
-
-def string(value: Any) -> str:
-    return str(value)
-
-
-def integer(value: Any) -> int:
-    if not isinstance(value, int):
-        try:
-            return int(str(value))
-        except TypeError as err:
-            raise ValueError(
-                f"Value that want to convert to integer does not support for "
-                f"type: {type(value)}"
-            ) from err
-    return value

ddeutil/workflow/utils/selection.py

@@ -1,2 +0,0 @@
-def conn(value):
-    return value

ddeutil_workflow-0.0.1.dist-info/RECORD

@@ -1,28 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=YZoDMFgikTmgL5EqRKa4mX-zXFr2c20J7dSZJ4dqSd0,27
-ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
-ddeutil/workflow/__types.py,sha256=7t-i5_-eif9zUBwb7jnv5RbSjzOihyl3yFDqXzaeyxk,428
-ddeutil/workflow/conn.py,sha256=5Nf6IQf2PXC6eLn469tGNvtnXAx-eOm3jJ_ltj90VbQ,6831
-ddeutil/workflow/dataset.py,sha256=SNp58WwTvgDSFWll9aizJeiHX_fd68f687cRME7vBo4,8599
-ddeutil/workflow/exceptions.py,sha256=aV_LN4_zmAk2R2p5ZYJcSMrsdBsRo6cMwm0lwZv_400,2399
-ddeutil/workflow/loader.py,sha256=nJffdJ8kchhwYsEGnWoG_leR16dzU0evo_p5Wbi4rp4,9710
-ddeutil/workflow/pipeline.py,sha256=kN6k2ht4C2OPHZYPljpnUQqfY3K54kMSOS1azGt7R68,10755
-ddeutil/workflow/schedule.py,sha256=RSvT0pi1Gb1un3OseRUW9AngRgVuAlXFT23QrSjtxYk,2828
-ddeutil/workflow/hooks/__init__.py,sha256=x5pY43037-cE0j3ofU_f7oEylzx-sN7pQwhiZubBtvY,199
-ddeutil/workflow/hooks/_postgres.py,sha256=8bAAlxHe_K1sAqwE1Pz6y6cKuytimmPnaEeudEPag2A,37
-ddeutil/workflow/tasks/__init__.py,sha256=NgGzbxWBV6lRgIEvp4VuREbKzo_ncTLgx00W2BAR-uk,276
-ddeutil/workflow/tasks/_polars.py,sha256=OT6kt7mYHx88DEJsO8sfnIfcjdXZ02XCAAcQBKr6wXM,1429
-ddeutil/workflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/utils/receive.py,sha256=lWB8k-_hPmdkq2bjPSL_2IX1X19I8f017u8-7jNRH68,939
-ddeutil/workflow/utils/selection.py,sha256=0qJu4mgVfNLLlnBbrsQtKK__LmG_DZf2Gs8CBplP4c0,34
-ddeutil/workflow/vendors/__dict.py,sha256=ETwkeA0qzKNgedfeRgAz1qShNXTIXIS4DXzJB4lM4jo,9962
-ddeutil/workflow/vendors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/vendors/__schedule.py,sha256=cUIwtTli9G0klrNFxAIbG3VTiv6umRuNAZiKA-kOtpE,20690
-ddeutil/workflow/vendors/aws_warpped.py,sha256=zjq_LCu3ffVBRrxS2vqss9X24yrtuAEt9ouy2_WvS0o,5980
-ddeutil/workflow/vendors/minio_warpped.py,sha256=pScLy38Du9moOrGaSBSFsoQRhiQ686FQyloOeLA0OQk,261
-ddeutil/workflow/vendors/sftp_wrapped.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
-ddeutil_workflow-0.0.1.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.1.dist-info/METADATA,sha256=uFH3MTuZk6UNPjLIQqaQSebszywROMUMzjLMftY0OM0,7444
-ddeutil_workflow-0.0.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ddeutil_workflow-0.0.1.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.1.dist-info/RECORD,,