dtflow 0.5.2__py3-none-any.whl → 0.5.3__py3-none-any.whl

dtflow/__init__.py

@@ -26,6 +26,12 @@ from .converters import (  # LLaMA-Factory 扩展; ms-swift
     to_swift_vlm,
 )
 from .core import DataTransformer, DictWrapper, TransformError, TransformErrors
+from .framework import (
+    CompatibilityResult,
+    check_compatibility,
+    detect_format,
+    export_for,
+)
 from .presets import get_preset, list_presets
 from .schema import (
     Field,
@@ -38,12 +44,6 @@ from .schema import (
     sharegpt_schema,
     validate_data,
 )
-from .framework import (
-    CompatibilityResult,
-    check_compatibility,
-    detect_format,
-    export_for,
-)
 from .storage import load_data, sample_file, save_data
 from .streaming import StreamingTransformer, load_sharded, load_stream, process_shards
 from .tokenizers import (
@@ -60,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.5.2"
+__version__ = "0.5.3"
 
 __all__ = [
     # core

dtflow/converters.py

@@ -4,7 +4,7 @@
 提供与 HuggingFace datasets 等常用格式的互转功能。
 """
 
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional
 
 
 def to_hf_dataset(data: List[Dict[str, Any]]):
@@ -143,14 +143,16 @@ def to_openai_batch(
         >>> batch_input = dt.to(to_openai_batch(model="gpt-4o"))
     """
 
-    def transform(item, idx=[0]) -> dict:
+    counter = {"idx": 0}
+
+    def transform(item) -> dict:
         messages = item.get(messages_field, []) if hasattr(item, "get") else item[messages_field]
 
         if custom_id_field:
             custom_id = item.get(custom_id_field) if hasattr(item, "get") else item[custom_id_field]
         else:
-            custom_id = f"request-{idx[0]}"
-            idx[0] += 1
+            custom_id = f"request-{counter['idx']}"
+            counter["idx"] += 1
 
         return {
             "custom_id": str(custom_id),
@@ -196,7 +198,7 @@ def to_llama_factory(
     """
 
     def transform(item) -> dict:
-        get = lambda f: (item.get(f, "") if hasattr(item, "get") else item.get(f, ""))
+        get = lambda f: item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
 
         result = {
             "instruction": get(instruction_field),
@@ -248,7 +250,7 @@ def to_axolotl(
         conversations = (
             item.get(conversations_field, [])
             if hasattr(item, "get")
-            else item.get(conversations_field, [])
+            else getattr(item, conversations_field, [])
         )
 
         # 如果已经是正确格式，直接返回
@@ -257,7 +259,9 @@ def to_axolotl(
                 return {"conversations": conversations}
 
         # 尝试从 messages 格式转换
-        messages = item.get("messages", []) if hasattr(item, "get") else item.get("messages", [])
+        messages = (
+            item.get("messages", []) if hasattr(item, "get") else getattr(item, "messages", [])
+        )
         if messages:
             role_map = {"user": "human", "assistant": "gpt", "system": "system"}
             conversations = [
@@ -312,7 +316,7 @@ def to_llama_factory_sharegpt(
     }
 
     def transform(item) -> dict:
-        get = lambda f: (item.get(f, "") if hasattr(item, "get") else item.get(f, ""))
+        get = lambda f: item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
         messages = get(messages_field) or []
 
         conversations = []
@@ -385,7 +389,7 @@ def to_llama_factory_vlm(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
         messages = get(messages_field) or []
 
         instruction = ""
@@ -467,7 +471,7 @@ def to_llama_factory_vlm_sharegpt(
     role_map = {"user": "human", "assistant": "gpt", "system": "system"}
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
         messages = get(messages_field) or []
 
         conversations = []
@@ -541,7 +545,7 @@ def to_swift_messages(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
         messages = get(messages_field) or []
 
         # 复制 messages，避免修改原数据
@@ -600,7 +604,7 @@ def to_swift_query_response(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
 
         query = get(query_field)
         response = get(response_field)
@@ -693,7 +697,7 @@ def to_swift_vlm(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
         messages = get(messages_field) or []
 
         result_messages = []

dtflow/presets.py

@@ -6,6 +6,8 @@
 
 from typing import Any, Callable
 
+from dtflow.utils.helpers import get_field_value
+
 
 def openai_chat(
     user_field: str = "q", assistant_field: str = "a", system_prompt: str = None
@@ -33,8 +35,8 @@ def openai_chat(
         if system_prompt:
             messages.append({"role": "system", "content": system_prompt})
 
-        user_content = getattr(item, user_field, None) or item.get(user_field, "")
-        assistant_content = getattr(item, assistant_field, None) or item.get(assistant_field, "")
+        user_content = get_field_value(item, user_field)
+        assistant_content = get_field_value(item, assistant_field)
 
         messages.append({"role": "user", "content": user_content})
         messages.append({"role": "assistant", "content": assistant_content})
@@ -60,10 +62,9 @@ def alpaca(
 
     def transform(item: Any) -> dict:
         return {
-            "instruction": getattr(item, instruction_field, None)
-            or item.get(instruction_field, ""),
-            "input": getattr(item, input_field, None) or item.get(input_field, ""),
-            "output": getattr(item, output_field, None) or item.get(output_field, ""),
+            "instruction": get_field_value(item, instruction_field),
+            "input": get_field_value(item, input_field),
+            "output": get_field_value(item, output_field),
         }
 
     return transform
@@ -84,9 +85,7 @@ def sharegpt(conversations_field: str = "conversations", role_mapping: dict = No
     role_mapping = role_mapping or {"user": "human", "assistant": "gpt"}
 
     def transform(item: Any) -> dict:
-        conversations = getattr(item, conversations_field, None) or item.get(
-            conversations_field, []
-        )
+        conversations = get_field_value(item, conversations_field, [])
 
         # 如果已经是对话格式，直接返回
         if conversations:
@@ -102,7 +101,7 @@ def sharegpt(conversations_field: str = "conversations", role_mapping: dict = No
             ("answer", "gpt"),
             ("output", "gpt"),
         ]:
-            value = getattr(item, field, None) or item.get(field, None)
+            value = get_field_value(item, field, None)
             if value:
                 result.append({"from": role, "value": value})
 
@@ -127,9 +126,9 @@ def dpo_pair(
 
     def transform(item: Any) -> dict:
         return {
-            "prompt": getattr(item, prompt_field, None) or item.get(prompt_field, ""),
-            "chosen": getattr(item, chosen_field, None) or item.get(chosen_field, ""),
-            "rejected": getattr(item, rejected_field, None) or item.get(rejected_field, ""),
+            "prompt": get_field_value(item, prompt_field),
+            "chosen": get_field_value(item, chosen_field),
+            "rejected": get_field_value(item, rejected_field),
         }
 
     return transform
@@ -148,8 +147,8 @@ def simple_qa(question_field: str = "q", answer_field: str = "a") -> Callable:
 
     def transform(item: Any) -> dict:
         return {
-            "question": getattr(item, question_field, None) or item.get(question_field, ""),
-            "answer": getattr(item, answer_field, None) or item.get(answer_field, ""),
+            "question": get_field_value(item, question_field),
+            "answer": get_field_value(item, answer_field),
         }
 
     return transform

dtflow/utils/__init__.py

@@ -9,6 +9,7 @@ from .field_path import (
     get_field_with_spec,
     parse_field_spec,
 )
+from .helpers import get_field_value
 
 __all__ = [
     "display_data",
@@ -20,4 +21,6 @@ __all__ = [
     "extract",
     "extract_with_spec",
     "ExpandMode",
+    # helpers
+    "get_field_value",
 ]

dtflow/utils/helpers.py

@@ -0,0 +1,30 @@
+"""公共辅助函数"""
+
+from typing import Any
+
+
+def get_field_value(item: Any, field: str, default: Any = "") -> Any:
+    """
+    获取字段值，支持 DictWrapper 和普通 dict。
+
+    优先尝试 dict.get()，如果没有 get 方法则使用 getattr()。
+
+    Args:
+        item: 数据对象（dict 或 DictWrapper）
+        field: 字段名
+        default: 默认值
+
+    Returns:
+        字段值或默认值
+
+    Examples:
+        >>> get_field_value({"name": "test"}, "name")
+        'test'
+        >>> get_field_value({"name": ""}, "name", "default")
+        'default'
+    """
+    if hasattr(item, "get"):
+        value = item.get(field, default)
+    else:
+        value = getattr(item, field, default)
+    return value if value else default

{dtflow-0.5.2.dist-info → dtflow-0.5.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.2
+Version: 0.5.3
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme

{dtflow-0.5.2.dist-info → dtflow-0.5.3.dist-info}/RECORD

@@ -1,11 +1,11 @@
-dtflow/__init__.py,sha256=PTqh_6-F6eEwg1RxQ0ueP6CYnZauMuqYhlZe2BJphr0,3031
+dtflow/__init__.py,sha256=RJql_KmINJNbq2FEqU7jD9Z0c5ETkxQJPvUUPKiFt74,3031
 dtflow/__main__.py,sha256=ySpqvEn7k-vsrYFPx-8O6p-yx_24KccgnOSPd2XybhM,12572
-dtflow/converters.py,sha256=gyy-K15zjzGBawFnZa8D9JX37JZ47rey2GhjKa2pxFo,22081
+dtflow/converters.py,sha256=yXafSDeRC7DB2MMj8fD1NWjAG8HoAGh5Ay2A5Z7s6xA,22206
 dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
 dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
 dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
-dtflow/presets.py,sha256=OP1nnM5NFk5Kli9FsXK0xAot48E5OQ6-VOIJT9ffXPg,5023
+dtflow/presets.py,sha256=qa8WQJhbNMuGxqqgA9BFadEBwDB9s0zWNxxhzF3q1K8,4701
 dtflow/schema.py,sha256=IFcij22_UFKcgKT1YWwRg2QJO0vcAvCb1arZmsGByts,16824
 dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
 dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
@@ -27,10 +27,11 @@ dtflow/mcp/docs.py,sha256=DI2Vf-eFo4chRP_bDLsv4Uc3kJt8_1emz8N-NBSVirM,8834
 dtflow/mcp/server.py,sha256=Nf0UlqDGhV55ndGuEglfr7VRjDWAC_9rRsNhdr0-ssM,4275
 dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,362
 dtflow/storage/io.py,sha256=ZH2aSE-S89gpy3z4oTqhcqWf4u10OdkDoyul7o_YBDI,23374
-dtflow/utils/__init__.py,sha256=f8v9HJZMWRI5AL64Vjr76Pf2Na_whOF9nJBKgPbXXYg,429
+dtflow/utils/__init__.py,sha256=Pn-ltwV04fBQmeZG7FxInDQmzH29LYOi90LgeLMEuQk,506
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
-dtflow-0.5.2.dist-info/METADATA,sha256=RlpGaySrAIgTviom_Wyn6o2LWzQQVihff12Jpazy10o,22544
-dtflow-0.5.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.5.2.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.5.2.dist-info/RECORD,,
+dtflow/utils/helpers.py,sha256=JXN176_B2pm53GLVyZ1wj3wrmBJG52Tkw6AMQSdj7M8,791
+dtflow-0.5.3.dist-info/METADATA,sha256=5joXihL8gkmnNEaUTqRpe0_U-y8osaIfdX0v91WVtK8,22544
+dtflow-0.5.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.3.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.3.dist-info/RECORD,,