xfmr-zem 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

xfmr_zem/zenml_wrapper.py

@@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, List
 from zenml import step
 from mcp import ClientSession, StdioServerParameters
 from mcp.client.stdio import stdio_client
+from loguru import logger
 import json
 import os
 
@@ -23,6 +24,10 @@ def run_mcp_tool(
     """
     cmd = [command] + args
     
+    # Forward stderr to sys.stderr for real-time logging in verbose mode
+    import sys
+    import threading
+    
     process = subprocess.Popen(
         cmd,
         stdin=subprocess.PIPE,
@@ -32,6 +37,16 @@ def run_mcp_tool(
         text=True,
         bufsize=0 
     )
+    
+    # Start a thread to stream stderr
+    def stream_stderr():
+        for line in process.stderr:
+            sys.stderr.write(line)
+            sys.stderr.flush()
+    
+    stderr_thread = threading.Thread(target=stream_stderr, daemon=True)
+    stderr_thread.start()
+
 
     try:
         # 1. Initialize
@@ -52,8 +67,7 @@ def run_mcp_tool(
         while True:
             line = process.stdout.readline()
             if not line:
-                 err = process.stderr.read()
-                 raise RuntimeError(f"Server closed connection during init. Stderr: {err}")
+                 raise RuntimeError(f"Server closed connection during init. Check logs above for details.")
             
             if line.strip().startswith("{"):
                 try:
@@ -77,8 +91,7 @@ def run_mcp_tool(
         while True:
             line = process.stdout.readline()
             if not line:
-                 err = process.stderr.read()
-                 raise RuntimeError(f"Server closed connection during {method}. Stderr: {err}")
+                 raise RuntimeError(f"Server closed connection during {method}. Check logs above for details.")
             
             if line.strip().startswith("{"):
                 try:
@@ -122,7 +135,7 @@ def list_mcp_tools(
         result = run_mcp_tool(command, args, env, "tools/list", {})
         return result.get("tools", [])
     except Exception as e:
-        print(f"Error listing tools: {e}")
+        logger.error(f"Error listing tools: {e}")
         return []
 
 
@@ -162,7 +175,7 @@ def mcp_generic_step(
     args = server_config.get("args", [])
     env = server_config.get("env", os.environ.copy())
     
-    print(f"[{server_name}] Executing tool '{tool_name}'")
+    logger.info(f"[{server_name}] Executing tool '{tool_name}'")
     start_time = time.time()
     
     try:
@@ -172,7 +185,7 @@ def mcp_generic_step(
         }
         result_data = run_mcp_tool(command, args, env, "tools/call", params)
         execution_time = time.time() - start_time
-        print(f"[{server_name}] Tool '{tool_name}' finished in {execution_time:.2f}s")
+        logger.info(f"[{server_name}] Tool '{tool_name}' finished in {execution_time:.2f}s")
         
         output_data = {}
         

{xfmr_zem-0.2.4.dist-info → xfmr_zem-0.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xfmr-zem
-Version: 0.2.4
+Version: 0.2.6
 Summary: Zem: Unified Data Pipeline Framework (ZenML + NeMo Curator + DataJuicer) for multi-domain processing
 Project-URL: Homepage, https://github.com/OAI-Labs/xfmr-zem
 Project-URL: Repository, https://github.com/OAI-Labs/xfmr-zem
@@ -22,8 +22,10 @@ Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyarrow>=15.0.0
 Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-magic>=0.4.27
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=13.0.0
+Requires-Dist: unstructured[all-docs]>=0.16.0
 Requires-Dist: zenml[local,server]>=0.75.0
 Provides-Extra: all
 Requires-Dist: nemo-curator>=0.6.0; extra == 'all'
@@ -39,6 +41,23 @@ Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: ruff>=0.1.0; extra == 'dev'
 Provides-Extra: nemo
 Requires-Dist: nemo-curator>=0.6.0; extra == 'nemo'
+Provides-Extra: ocr
+Requires-Dist: cachetools>=5.0.0; extra == 'ocr'
+Requires-Dist: einops; extra == 'ocr'
+Requires-Dist: onnxruntime>=1.16.0; extra == 'ocr'
+Requires-Dist: opencv-python>=4.8.0; extra == 'ocr'
+Requires-Dist: paddleocr>=2.7.0; extra == 'ocr'
+Requires-Dist: paddlepaddle>=2.6.0; extra == 'ocr'
+Requires-Dist: pdfplumber>=0.11.0; extra == 'ocr'
+Requires-Dist: pillow>=10.0.0; extra == 'ocr'
+Requires-Dist: pyclipper; extra == 'ocr'
+Requires-Dist: pymupdf>=1.23.0; extra == 'ocr'
+Requires-Dist: pytesseract>=0.3.10; extra == 'ocr'
+Requires-Dist: ruamel-yaml>=0.17.0; extra == 'ocr'
+Requires-Dist: shapely; extra == 'ocr'
+Requires-Dist: torch==2.5.1; extra == 'ocr'
+Requires-Dist: torchvision==0.20.1; extra == 'ocr'
+Requires-Dist: transformers>=4.40.0; extra == 'ocr'
 Provides-Extra: zenml
 Requires-Dist: zenml>=0.75.0; extra == 'zenml'
 Description-Content-Type: text/markdown

xfmr_zem-0.2.6.dist-info/RECORD

@@ -0,0 +1,58 @@
+xfmr_zem/__init__.py,sha256=Abx2BepsZu-e7E93N2lOgu9w0b4TBZLN6MEzCzDCn_A,1138
+xfmr_zem/cli.py,sha256=5oz4qxXthU4mXu7bSbfKreVkAvCqrieXpGoKhJBXBvk,12538
+xfmr_zem/client.py,sha256=wf9N_fILDBvWd-08TnNq3B1PqKQPhR0pvVuJq0vidk0,11435
+xfmr_zem/schemas.py,sha256=0tHM0ftOWTWxNiqmAZn_MyIYJwF2p9brHK0MHlOMlKY,494
+xfmr_zem/server.py,sha256=EeohfqhUiCm0cGnV85H2ODZ4FLXjcTjbkdHrHuGHW4I,8363
+xfmr_zem/zenml_wrapper.py,sha256=LHgDewuPBjCl4EiU6JZVU-_lyEi-ATURDSG9Vf7PbEY,6739
+xfmr_zem/orchestrators/parallel_local.py,sha256=_ve7UBmDM3yoLFljKBu0cS6TcZsyo6pgDs554YmTWiQ,3037
+xfmr_zem/servers/data_juicer/parameters.yml,sha256=dl7YdcDlCCAjF_upLmuI8YwD5gti5gLR3SWHcqE8L2c,299
+xfmr_zem/servers/data_juicer/server.py,sha256=AGWnpy-17NqmXOOQda30zi0dyY4_60HRIco7Qkps2kQ,3242
+xfmr_zem/servers/instruction_gen/parameters.yml,sha256=q5cnper2ufdH1ceYxo95aHJ5nXtOHbd_tc75VzRt2rc,505
+xfmr_zem/servers/instruction_gen/server.py,sha256=XRxnNIUWJ8RdquukrKo5OGhK6_o4a5zILILpuVnsXhI,3129
+xfmr_zem/servers/io/parameters.yml,sha256=CDyETx0Mbo85BUmrQ_okGVhcbKNfkFj-63VXvd_989k,182
+xfmr_zem/servers/io/server.py,sha256=af48i4v-1n97N0Dsb1gENM-E8_TswhF6Bg7-AtHMiMc,3272
+xfmr_zem/servers/llm/parameters.yml,sha256=IQqAHqPHQS8_ffA3CfNGsprRi8OqF-N8cBouTDDnhjU,157
+xfmr_zem/servers/llm/server.py,sha256=ugCQ7bIuZmc-j_DCjo5GDI5AmC2fbFPx7SXAvwj1VAo,1930
+xfmr_zem/servers/nemo_curator/parameters.yml,sha256=EGEzo0heI-ajkwFFy3xxq_YD7cXUO4n4bjl73XoFZpI,357
+xfmr_zem/servers/nemo_curator/server.py,sha256=zcHoSwxxoK_rMaDIAbEy1s8qfdp68Ue4B-XBcjGxQak,3848
+xfmr_zem/servers/ocr/engines.py,sha256=zScn4Qjxbpl2nB8UXEf3kd9l8z84TEwGs6bV5ka8Lks,10295
+xfmr_zem/servers/ocr/install_models.py,sha256=t02zpoy8djVhITOLEaRJ2mjiMrFfA9H6fpeHD3hXuio,2135
+xfmr_zem/servers/ocr/parameters.yml,sha256=04v59-6QXwN6XEpnHLc5pz6iTgNBDhloHtCCjHr8YRA,89
+xfmr_zem/servers/ocr/server.py,sha256=lhT5rfpz2vPbdbM7RZAtsJKfZEqoth2ijEifMA_vWSA,3590
+xfmr_zem/servers/ocr/deepdoc_vietocr/__init__.py,sha256=XJE7RnOu5oo5p902HPWPDBd7FhVQXetmnr2-kWEG0nI,2419
+xfmr_zem/servers/ocr/deepdoc_vietocr/implementations.py,sha256=79fYr76fx8yZda3HaFcK1d5G-4sDVf1JFHNW_OBQAk8,47348
+xfmr_zem/servers/ocr/deepdoc_vietocr/layout_recognizer.py,sha256=7BeLHzf9FQUkkHMb5jDpggruJmfXVMU78MF_EeZ9PG4,10462
+xfmr_zem/servers/ocr/deepdoc_vietocr/ocr.py,sha256=Iqiqof4LtMjIbU93INYgtWRtkvftAKFk6__gFvV0TUQ,18948
+xfmr_zem/servers/ocr/deepdoc_vietocr/operators.py,sha256=HT2VHEnk1cgGEMlaPRaYeHzgDyYgvNivm4w1savPhUA,23999
+xfmr_zem/servers/ocr/deepdoc_vietocr/phases.py,sha256=3gt__xNlKRrHso7qRRFK_BdWQMYMNunn-V_PfKZBTyI,6605
+xfmr_zem/servers/ocr/deepdoc_vietocr/pipeline.py,sha256=3nTfvbIianpIMwxKkcGiMKho_E844qAhONvk1awbZs8,21992
+xfmr_zem/servers/ocr/deepdoc_vietocr/postprocess.py,sha256=8Ik32oIdRVOalp3qQS-IdWp7VCZkKqRqnqMNSnenGsU,13210
+xfmr_zem/servers/ocr/deepdoc_vietocr/recognizer.py,sha256=dvU9SCid_vqL-hdkBKpPuAhKedGtIcIMQf384KBeZPU,16076
+xfmr_zem/servers/ocr/deepdoc_vietocr/table_structure_recognizer.py,sha256=QFQtPUXQgx4kRhK7EXIdkMkpUzOJLkbzLXIgxMDYliU,21933
+xfmr_zem/servers/ocr/deepdoc_vietocr/onnx/.gitattributes,sha256=Ea1--iSXXuSww8OjjtGHN_Blil91oKlnh7V2p4oCM2E,1519
+xfmr_zem/servers/ocr/deepdoc_vietocr/onnx/README.md,sha256=oIP5wxeqC9ps7KWh3sQFUGbi0_mMdvIL6RQrRSI_zFA,119
+xfmr_zem/servers/ocr/deepdoc_vietocr/onnx/ocr.res,sha256=KLI2KtSrLcOHaapy_rU146nds_0qdYWgWSDmOTsdx_c,26249
+xfmr_zem/servers/ocr/deepdoc_vietocr/utils/__init__.py,sha256=Eoefn8qLo4YhhlhAd5R4lNhKCL-HumlywFhicfeodmw,2488
+xfmr_zem/servers/ocr/deepdoc_vietocr/utils/file_utils.py,sha256=sjw2t8WkjZpIW_eZJEjh60Cp3VBsZIsbYDEaMXmSXSw,7780
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/config/base.yml,sha256=8ujp9qH6FK3Lvaztiv0-scDxbXigtbAgMCfGdzYEkPQ,1808
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/config/vgg-seq2seq.yml,sha256=NeNBuoHlksCYmwpa3lDgakv2lOwbTIPSG6Y9HfXDmIQ,748
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/model/transformerocr.py,sha256=_qhwMR2YIQgQuw7NfBEj0gN5D88Q2sXinudJV2BnwMw,803
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/model/vocab.py,sha256=se7YdFVh0bXe2A1ytUh2yL-GFVckQj9L283jylvN7Zw,944
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/model/backbone/cnn.py,sha256=ag4VGrDGDv9CPDy_09smqx0GJ_mjd2zL5GJv49V90zo,673
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/model/backbone/vgg.py,sha256=_9nhphrAvib2v8xXFQdrh_3HDAGJn3FKcZfX34pWQOA,1584
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/model/seqmodel/seq2seq.py,sha256=EMSXVGmo5ZD8ygpvbnzeZdIvIn2ifry0JMW5fSUOXVg,5731
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/tool/config.py,sha256=UGc4hsSWbPzMJMrVgwyxcOMmtNducAi2MJ_dWXHjSSA,1096
+xfmr_zem/servers/ocr/deepdoc_vietocr/vietocr/tool/translate.py,sha256=nWMcewxW6mBp568eyfRGx1frNaTf7SSG6QpFNfDZI6s,3377
+xfmr_zem/servers/profiler/parameters.yml,sha256=2ruRnZ47GkiJoyAyGpDxq9IAFR_HvizFCP2CFfXlCxM,99
+xfmr_zem/servers/profiler/server.py,sha256=GcBzroxHIQ9SwMgdgHSwaoqvFrKeGfUu9Y6Dk_OaTwM,2397
+xfmr_zem/servers/sinks/parameters.yml,sha256=9HAnv84Utw2qWsVZH8uOjVE62lnAKBkzv4PZcPtDo6w,108
+xfmr_zem/servers/sinks/server.py,sha256=jI_r4sq_U_avNwF1PiE0alpaDrYpzOI-qPeLU7hgHP0,1589
+xfmr_zem/servers/unstructured/parameters.yml,sha256=N31cmc56GTr3rkVhbni4yOpbnHISReN8f-KnRZTDbBc,118
+xfmr_zem/servers/unstructured/server.py,sha256=0XmXWMAUNEJboX-J4bn_8EBUfMHIqu_ylNC_s9YOZdk,1996
+xfmr_zem-0.2.6.dist-info/METADATA,sha256=E2bD-td6hx7ntkKGH3XIgwH8BTUADbTE5T-cFi21Qds,6379
+xfmr_zem-0.2.6.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+xfmr_zem-0.2.6.dist-info/entry_points.txt,sha256=uxs-IXFxpSakHivpFN3mEr13cz-z-0vkeSF_4dEBMa4,65
+xfmr_zem-0.2.6.dist-info/licenses/LICENSE,sha256=kf_ILr0zLkSy5-EBu0VF2PGaOykYo83z3UijI-bZeAE,11342
+xfmr_zem-0.2.6.dist-info/RECORD,,

xfmr_zem-0.2.4.dist-info/RECORD

@@ -1,23 +0,0 @@
-xfmr_zem/__init__.py,sha256=Abx2BepsZu-e7E93N2lOgu9w0b4TBZLN6MEzCzDCn_A,1138
-xfmr_zem/cli.py,sha256=u3qzzoxPIBSgBy7f80X_pr8SyjACHP7R8uHwRxwjMWk,11367
-xfmr_zem/client.py,sha256=sAMhIB_N-JjmaUh9g0fSyxhbXvqctugsCOzf_0ctv8w,9027
-xfmr_zem/schemas.py,sha256=0tHM0ftOWTWxNiqmAZn_MyIYJwF2p9brHK0MHlOMlKY,494
-xfmr_zem/server.py,sha256=8ayF-v6P_YO60akD0SRjHBnsB3ZBsJ1ZY_BaHf3qR3I,7517
-xfmr_zem/zenml_wrapper.py,sha256=p6FbvIHFvakKAekzRGiauKi5AbWL0kJMw69iPrHJ8C0,6364
-xfmr_zem/orchestrators/parallel_local.py,sha256=_ve7UBmDM3yoLFljKBu0cS6TcZsyo6pgDs554YmTWiQ,3037
-xfmr_zem/servers/data_juicer/parameter.yaml,sha256=dl7YdcDlCCAjF_upLmuI8YwD5gti5gLR3SWHcqE8L2c,299
-xfmr_zem/servers/data_juicer/server.py,sha256=qmH6SeYa9OL6kMYIO3tTroKJUwoyefqI8SmuY08D_pk,3242
-xfmr_zem/servers/instruction_gen/parameter.yaml,sha256=q5cnper2ufdH1ceYxo95aHJ5nXtOHbd_tc75VzRt2rc,505
-xfmr_zem/servers/instruction_gen/server.py,sha256=orM1QSNjc37APgOHdDTa5joZEOvfM5KlNrBrNuX51Sw,3129
-xfmr_zem/servers/io/parameter.yaml,sha256=CDyETx0Mbo85BUmrQ_okGVhcbKNfkFj-63VXvd_989k,182
-xfmr_zem/servers/io/server.py,sha256=dQ3yWDeKXn7A8Fkwty3-6Yy-FmA0BpEDjzejHref7G0,3272
-xfmr_zem/servers/llm/server.py,sha256=ugCQ7bIuZmc-j_DCjo5GDI5AmC2fbFPx7SXAvwj1VAo,1930
-xfmr_zem/servers/nemo_curator/parameter.yaml,sha256=EGEzo0heI-ajkwFFy3xxq_YD7cXUO4n4bjl73XoFZpI,357
-xfmr_zem/servers/nemo_curator/server.py,sha256=lqN8I4uYhAOKyDyVV6BOewdijfvKTjksuwdr7JLKnkg,3848
-xfmr_zem/servers/profiler/server.py,sha256=GcBzroxHIQ9SwMgdgHSwaoqvFrKeGfUu9Y6Dk_OaTwM,2397
-xfmr_zem/servers/sinks/server.py,sha256=jI_r4sq_U_avNwF1PiE0alpaDrYpzOI-qPeLU7hgHP0,1589
-xfmr_zem-0.2.4.dist-info/METADATA,sha256=4UU8wN-sHU5_FF5C_sP2gHenE4FtqweqzhQJsU_RuKw,5516
-xfmr_zem-0.2.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-xfmr_zem-0.2.4.dist-info/entry_points.txt,sha256=uxs-IXFxpSakHivpFN3mEr13cz-z-0vkeSF_4dEBMa4,65
-xfmr_zem-0.2.4.dist-info/licenses/LICENSE,sha256=kf_ILr0zLkSy5-EBu0VF2PGaOykYo83z3UijI-bZeAE,11342
-xfmr_zem-0.2.4.dist-info/RECORD,,