dtflow 0.5.8__py3-none-any.whl → 0.5.9__py3-none-any.whl

dtflow/tokenizers.py

@@ -122,8 +122,8 @@ def _get_tiktoken_encoder(model: str):
                 _tokenizer_cache[model] = tiktoken.get_encoding(model)
             else:
                 _tokenizer_cache[model] = tiktoken.encoding_for_model(model)
-        except ImportError as e:
-            raise ImportError("需要安装 tiktoken: pip install tiktoken") from e
+        except ImportError:
+            raise ImportError("需要安装 tiktoken: pip install tiktoken")
     return _tokenizer_cache[model]
 
 
@@ -149,12 +149,12 @@ def _get_hf_tokenizer(model: str):
 
                 tokenizer = AutoTokenizer.from_pretrained(resolved, trust_remote_code=True)
                 _tokenizer_cache[resolved] = ("transformers", tokenizer)
-            except ImportError as e:
+            except ImportError:
                 raise ImportError(
                     "需要安装 tokenizers 或 transformers:\n"
                     "  pip install tokenizers huggingface_hub  (推荐，更轻量)\n"
                     "  pip install transformers"
-                ) from e
+                )
     return _tokenizer_cache[resolved]
 
 
@@ -309,29 +309,12 @@ def _std(counts: List[int], avg: float) -> float:
     return variance**0.5
 
 
-def _count_item_tokens(args: tuple) -> int:
-    """
-    计算单条数据的 token 数（用于多进程）。
-
-    Args:
-        args: (item, fields, model, backend) 元组
-    """
-    item, fields, model, backend = args
-    total = 0
-    for field in fields:
-        value = get_field_with_spec(item, field, default="")
-        if value:
-            total += count_tokens(str(value), model=model, backend=backend)
-    return total
-
-
 def token_stats(
     data: List[Dict[str, Any]],
     fields: Union[str, List[str]],
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
     progress_callback: Optional[Callable[[int, int], None]] = None,
-    workers: Optional[int] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集的 token 信息。
@@ -342,7 +325,6 @@ def token_stats(
         model: 模型名称或别名，如 "qwen2.5", "gpt-4" 等
         backend: 后端选择，None 则自动检测
         progress_callback: 进度回调函数，接收 (current, total) 两个参数
-        workers: 进程数，None 自动检测，1 表示禁用并行
 
     Returns:
         统计信息字典，包含:
@@ -360,42 +342,17 @@ def token_stats(
     if not data:
         return {"total_tokens": 0, "count": 0}
 
+    counts = []
     total_items = len(data)
-    _backend = backend or _auto_backend(model)
-
-    # 判断是否使用多进程
-    use_parallel = workers != 1 and total_items >= 1000
-
-    if use_parallel:
-        from .parallel import get_optimal_workers, parallel_imap
-
-        actual_workers = get_optimal_workers(total_items, workers)
-        # 准备参数
-        args_list = [(item, fields, model, _backend) for item in data]
-        counts = []
-        for i, result in enumerate(
-            parallel_imap(
-                _count_item_tokens,
-                args_list,
-                workers=actual_workers,
-                threshold=1000,
-            )
-        ):
-            counts.append(result)
-            if progress_callback:
-                progress_callback(i + 1, total_items)
-    else:
-        # 串行处理
-        counts = []
-        for i, item in enumerate(data):
-            total = 0
-            for field in fields:
-                value = get_field_with_spec(item, field, default="")
-                if value:
-                    total += count_tokens(str(value), model=model, backend=_backend)
-            counts.append(total)
-            if progress_callback:
-                progress_callback(i + 1, total_items)
+    for i, item in enumerate(data):
+        total = 0
+        for field in fields:
+            value = get_field_with_spec(item, field, default="")
+            if value:
+                total += count_tokens(str(value), model=model, backend=backend)
+        counts.append(total)
+        if progress_callback:
+            progress_callback(i + 1, total_items)
 
     sorted_counts = sorted(counts)
     avg = sum(counts) / len(counts)
@@ -591,27 +548,12 @@ def messages_token_filter(
     return filter_func
 
 
-def _count_messages_tokens_wrapper(args: tuple) -> Optional[Dict[str, int]]:
-    """
-    计算单条 messages 的 token 数（用于多进程）。
-
-    Args:
-        args: (item, messages_field, model, backend) 元组
-    """
-    item, messages_field, model, backend = args
-    messages = get_field_with_spec(item, messages_field, default=[])
-    if messages:
-        return _count_messages_tokens(messages, model=model, backend=backend)
-    return None
-
-
 def messages_token_stats(
     data: List[Dict[str, Any]],
     messages_field: str = "messages",
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
     progress_callback: Optional[Callable[[int, int], None]] = None,
-    workers: Optional[int] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集中 messages 的 token 信息。
@@ -622,7 +564,6 @@ def messages_token_stats(
         model: 模型名称或别名
         backend: 后端，None 则自动检测
         progress_callback: 进度回调函数，接收 (current, total) 两个参数
-        workers: 进程数，None 自动检测，1 表示禁用并行
 
     Returns:
         统计信息字典，包含:
@@ -640,38 +581,14 @@ def messages_token_stats(
     if not data:
         return {"count": 0, "total_tokens": 0}
 
-    total_items = len(data)
-
-    # 判断是否使用多进程
-    use_parallel = workers != 1 and total_items >= 1000
-
     all_stats = []
-    if use_parallel:
-        from .parallel import get_optimal_workers, parallel_imap
-
-        actual_workers = get_optimal_workers(total_items, workers)
-        args_list = [(item, messages_field, model, _backend) for item in data]
-
-        for i, result in enumerate(
-            parallel_imap(
-                _count_messages_tokens_wrapper,
-                args_list,
-                workers=actual_workers,
-                threshold=1000,
-            )
-        ):
-            if result is not None:
-                all_stats.append(result)
-            if progress_callback:
-                progress_callback(i + 1, total_items)
-    else:
-        # 串行处理
-        for i, item in enumerate(data):
-            messages = get_field_with_spec(item, messages_field, default=[])
-            if messages:
-                all_stats.append(_count_messages_tokens(messages, model=model, backend=_backend))
-            if progress_callback:
-                progress_callback(i + 1, total_items)
+    total_items = len(data)
+    for i, item in enumerate(data):
+        messages = get_field_with_spec(item, messages_field, default=[])
+        if messages:
+            all_stats.append(_count_messages_tokens(messages, model=model, backend=_backend))
+        if progress_callback:
+            progress_callback(i + 1, total_items)
 
     if not all_stats:
         return {"count": 0, "total_tokens": 0}

dtflow/utils/text_parser.py

@@ -0,0 +1,124 @@
+"""
+文本清洗工具
+
+提供 LLM 输出的常见清洗函数：
+- strip_think_tags: 去除 <think>...</think> 思考链内容
+- extract_code_snippets: 提取 ``` 代码块
+- parse_generic_tags: 解析 <tag>content</tag> 格式标签
+"""
+
+import re
+from typing import Dict, List
+
+
+def strip_think_tags(text: str) -> str:
+    """去除 <think>...</think> 包裹的内容
+
+    Args:
+        text: 输入文本
+
+    Returns:
+        去除思考链后的文本
+
+    Examples:
+        >>> strip_think_tags("<think>让我想想...</think>答案是42")
+        '答案是42'
+    """
+    if not text:
+        return text
+    return re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
+
+
+def extract_code_snippets(text: str, strict: bool = True) -> List[Dict[str, str]]:
+    """提取 ``` 代码块
+
+    Args:
+        text: 输入文本
+        strict: True 仅匹配 ```lang...``` 格式，False 额外匹配 {...} 格式
+
+    Returns:
+        代码片段列表，每项为 {"language": ..., "code": ...}
+
+    Examples:
+        >>> extract_code_snippets("```json\\n{\"a\": 1}\\n```")
+        [{'language': 'json', 'code': '{"a": 1}'}]
+    """
+    pattern = r"```(\w+)?\s*([\s\S]*?)```"
+    matches = re.findall(pattern, text)
+
+    code_snippets = []
+    for lang, code in matches:
+        code_snippets.append(
+            {
+                "language": lang.strip() if lang else "unknown",
+                "code": code.strip(),
+            }
+        )
+
+    if not strict:
+        # 移除已匹配的 ``` 块，在剩余文本中匹配 { ... }
+        text = re.sub(pattern, "", text)
+        brace_matches = re.findall(r"\{[\s\S]*?\}", text)
+        for code in brace_matches:
+            code_snippets.append(
+                {
+                    "language": "unknown",
+                    "code": code.strip(),
+                }
+            )
+
+    return code_snippets
+
+
+def parse_generic_tags(text: str, strict: bool = False) -> Dict[str, str]:
+    """解析 XML 风格标签
+
+    支持两种模式：
+    - strict=True: 仅匹配闭合标签 <label>content</label>
+    - strict=False: 同时匹配开放式标签 <label>content，闭合标签优先
+
+    Args:
+        text: 输入文本
+        strict: 是否严格模式
+
+    Returns:
+        {标签名: 内容} 字典
+
+    Examples:
+        >>> parse_generic_tags("<标签>内容</标签>")
+        {'标签': '内容'}
+        >>> parse_generic_tags("<a>hello<b>world", strict=False)
+        {'a': 'hello', 'b': 'world'}
+    """
+    if not text:
+        return {}
+
+    result = {}
+
+    if strict:
+        pattern_closed = r"<([^>]+)>\s*(.*?)\s*</\1>"
+        matches = re.findall(pattern_closed, text, re.DOTALL)
+        for label, content in matches:
+            result[label.strip()] = content.strip()
+    else:
+        remaining_text = str(text)
+
+        # 1. 优先处理闭合标签
+        def process_closed_tag(match_obj):
+            label = match_obj.group(1).strip()
+            content = match_obj.group(2).strip()
+            result[label] = content
+            return ""
+
+        pattern_closed = r"<([^>]+)>\s*(.*?)\s*</\1>"
+        remaining_text = re.sub(pattern_closed, process_closed_tag, remaining_text, flags=re.DOTALL)
+
+        # 2. 在剩余文本中处理开放式标签
+        pattern_open = r"<([^>]+)>\s*(.*?)(?=<[^>]+>|$)"
+        matches_open = re.findall(pattern_open, remaining_text, re.DOTALL)
+        for label, content in matches_open:
+            label_stripped = label.strip()
+            if label_stripped not in result:
+                result[label_stripped] = content.strip()
+
+    return result

{dtflow-0.5.8.dist-info → dtflow-0.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.8
+Version: 0.5.9
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
@@ -44,6 +44,7 @@ Requires-Dist: flake8>=3.9.0; extra == 'dev'
 Requires-Dist: huggingface-hub>=0.20.0; extra == 'dev'
 Requires-Dist: isort>=5.9.0; extra == 'dev'
 Requires-Dist: mypy>=0.910; extra == 'dev'
+Requires-Dist: pandas>=1.3.0; extra == 'dev'
 Requires-Dist: pyarrow; extra == 'dev'
 Requires-Dist: pytest-cov>=2.12.0; extra == 'dev'
 Requires-Dist: pytest>=6.0.0; extra == 'dev'
@@ -57,10 +58,14 @@ Provides-Extra: docs
 Requires-Dist: myst-parser>=0.15.0; extra == 'docs'
 Requires-Dist: sphinx-rtd-theme>=0.5.0; extra == 'docs'
 Requires-Dist: sphinx>=4.0.0; extra == 'docs'
+Provides-Extra: eval
+Requires-Dist: pandas>=1.3.0; extra == 'eval'
+Requires-Dist: scikit-learn>=0.24.0; extra == 'eval'
 Provides-Extra: full
 Requires-Dist: datasets>=2.0.0; extra == 'full'
 Requires-Dist: datasketch>=1.5.0; extra == 'full'
 Requires-Dist: huggingface-hub>=0.20.0; extra == 'full'
+Requires-Dist: pandas>=1.3.0; extra == 'full'
 Requires-Dist: pyarrow; extra == 'full'
 Requires-Dist: rich>=10.0.0; extra == 'full'
 Requires-Dist: scikit-learn>=0.24.0; extra == 'full'
@@ -435,6 +440,13 @@ dt sample data.jsonl 1000 --by=messages.#         # 按消息数量分层采样
 dt sample data.jsonl --where="category=tech"      # 筛选后采样
 dt sample data.jsonl --where="messages.#>=2"      # 多条件筛选
 
+# 按行范围查看（Python 切片语法）
+dt slice data.jsonl 10:20                          # 第 10-19 行（0-based，左闭右开）
+dt slice data.jsonl :100                           # 前 100 行
+dt slice data.jsonl 100:                           # 第 100 行到末尾
+dt slice data.jsonl 10:20 -o sliced.jsonl          # 保存到文件
+dt slice data.jsonl 10:20 -f question,answer       # 只显示指定字段
+
 # 数据转换 - 预设模式
 dt transform data.jsonl --preset=openai_chat
 dt transform data.jsonl --preset=alpaca
@@ -452,7 +464,6 @@ dt run pipeline.yaml --input=new_data.jsonl --output=result.jsonl
 dt token-stats data.jsonl --field=messages --model=gpt-4
 dt token-stats data.jsonl --field=messages[-1].content   # 统计最后一条消息
 dt token-stats data.jsonl --field=text --detailed
-dt token-stats data.jsonl --workers=4                    # 多进程加速（数据量大时自动启用）
 
 # 数据对比
 dt diff v1/train.jsonl v2/train.jsonl
@@ -469,6 +480,9 @@ dt clean data.jsonl --max-len=messages[-1].content:500  # 最后一条消息最
 dt clean data.jsonl --keep=question,answer          # 只保留这些字段
 dt clean data.jsonl --drop=metadata                 # 删除指定字段
 dt clean data.jsonl --strip                         # 去除字符串首尾空白
+dt clean data.jsonl --min-tokens=content:10          # 最少 10 tokens
+dt clean data.jsonl --max-tokens=content:1000        # 最多 1000 tokens
+dt clean data.jsonl --min-tokens=text:50 -m gpt-4    # 指定分词器
 
 # 数据去重
 dt dedupe data.jsonl                            # 全量精确去重
@@ -477,6 +491,17 @@ dt dedupe data.jsonl --key=meta.id              # 按嵌套字段去重
 dt dedupe data.jsonl --key=messages[0].content  # 按第一条消息内容去重
 dt dedupe data.jsonl --key=text --similar=0.8   # 相似度去重
 
+# 数据集切分
+dt split data.jsonl --ratio=0.8 --seed=42           # 二分: train/test
+dt split data.jsonl --ratio=0.7,0.15,0.15           # 三分: train/val/test
+dt split data.jsonl --ratio=0.8 -o /tmp/output      # 指定输出目录
+
+# 训练框架导出
+dt export data.jsonl --framework=llama-factory       # 导出到 LLaMA-Factory
+dt export data.jsonl -f swift -o ./swift_out         # 导出到 ms-swift
+dt export data.jsonl -f axolotl                      # 导出到 Axolotl
+dt export data.jsonl -f llama-factory --check        # 仅检查兼容性
+
 # 文件拼接
 dt concat a.jsonl b.jsonl -o merged.jsonl
 
@@ -496,7 +521,6 @@ dt validate data.jsonl --preset=openai_chat           # 使用预设 schema 验
 dt validate data.jsonl --preset=alpaca --verbose      # 详细输出
 dt validate data.jsonl --preset=sharegpt --filter-invalid -o valid.jsonl  # 过滤出有效数据
 dt validate data.jsonl --preset=dpo --max-errors=100  # 限制错误输出数量
-dt validate data.jsonl --preset=openai_chat --workers=4  # 多进程加速
 ```
 
 ### 字段路径语法
@@ -522,6 +546,8 @@ CLI 命令中的字段参数支持嵌套路径语法，可访问深层嵌套的
 | `clean` | `--drop-empty=` | `--drop-empty=meta.source` |
 | `clean` | `--min-len=` | `--min-len=messages.#:2` |
 | `clean` | `--max-len=` | `--max-len=messages[-1].content:500` |
+| `clean` | `--min-tokens=` | `--min-tokens=content:10` |
+| `clean` | `--max-tokens=` | `--max-tokens=content:1000` |
 | `token-stats` | `--field=` | `--field=messages[-1].content` |
 | `diff` | `--key=` | `--key=meta.uuid` |
 

{dtflow-0.5.8.dist-info → dtflow-0.5.9.dist-info}/RECORD

@@ -1,35 +1,39 @@
-dtflow/SKILL.md,sha256=nh12TTq_eRzl5O2CTgsiS809BBVR49kmpZ8n7UprMHI,9552
-dtflow/__init__.py,sha256=tofhUr_PMnsONnB3Hu-mwUrD4Q3bV7Kw_0S6dQw6ig8,3031
-dtflow/__main__.py,sha256=p8oZKQhwq04shCB3y_pkXjf-SZ4PZvg5PXdyUP-5rYA,13497
+dtflow/SKILL.md,sha256=hPxJhroGmNbBv8MLZUkOA2yW1TDdUKEUYYlz9tW2mao,10393
+dtflow/__init__.py,sha256=9ZqhqD8qQM9w2dfHKyUWIaqSX-X4elWtbaQN4CNBhgg,3031
+dtflow/__main__.py,sha256=gg3v7u-Ot7AicgKrP1fuyKtMJXVduNuLmhy7L1LUPDg,17710
 dtflow/converters.py,sha256=X3qeFD7FCOMnfiP3MicL5MXimOm4XUYBs5pczIkudU0,22331
 dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
+dtflow/eval.py,sha256=_c-XP2zsOBznYltSyKEScOqvmPVX2orqepg5cNhXXB0,9836
 dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
 dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
-dtflow/parallel.py,sha256=EnIdGEGMrZUNT2-CBIV93UFfpqr_jU_heqqvdGXcP-Y,3046
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
 dtflow/presets.py,sha256=qa8WQJhbNMuGxqqgA9BFadEBwDB9s0zWNxxhzF3q1K8,4701
-dtflow/schema.py,sha256=zCZNEAqTMT1BS_p2t0CYczR5S9rqyDREa7ZsYI5pFGA,19885
+dtflow/schema.py,sha256=IFcij22_UFKcgKT1YWwRg2QJO0vcAvCb1arZmsGByts,16824
 dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
-dtflow/tokenizers.py,sha256=GFQsuLSLn2GHn2kaXhJkP8G85lgsdLzYtJNbppQhYPE,23408
+dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
-dtflow/cli/clean.py,sha256=KuE9ODjD9gSZUIHaD2mQLTDO-1PDwN7EqUpj8EQfVCs,25663
-dtflow/cli/commands.py,sha256=zKUG-B9Az-spqyqM00cR8Sgc2UgeOPQDThJFHWDNO_w,1336
+dtflow/cli/clean.py,sha256=BEQQlH2q6luCbx51M3oxxOwcnwlOA8vo9WX3Fp7I6AY,29498
+dtflow/cli/commands.py,sha256=LvyDQ_nWUM7UlPDEFQadRdw5O2ZKDLgF41_xAJRhYxI,1583
 dtflow/cli/common.py,sha256=gCwnF5Sw2ploqfZJO_z3Ms9mR1HNT7Lj6ydHn0uVaIw,13817
+dtflow/cli/eval.py,sha256=c53kCRH86k2Q_6vESKFlcepcNnTpO9O68agWK4_oJj8,9582
+dtflow/cli/export.py,sha256=loRfVPwEVsDw3ZMKEYGp0Hy38kYZG2QT8JCMbz1dRzU,2156
 dtflow/cli/io_ops.py,sha256=BMDisP6dxzzmSjYwmeFwaHmpHHPqirmXAWeNTD-9MQM,13254
 dtflow/cli/lineage.py,sha256=_lNh35nF9AA0Zy6FyZ4g8IzrXH2ZQnp3inF-o2Hs1pw,1383
 dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
-dtflow/cli/sample.py,sha256=pubpx4AIzsarBEalD150MC2apYQSt4bal70IZkTfFO0,15475
+dtflow/cli/sample.py,sha256=etbro5I0pyNgn0Qfhp1M6Bh-95JN-AntDa5AwVe_oKY,18269
 dtflow/cli/skill.py,sha256=opiTEBejA7JHKrEMftMOPDQlOgZ4n59rwaHXGU1Nukk,2022
-dtflow/cli/stats.py,sha256=HkTZD80h4tzYXTtMnfpjLUMP6kl_es6ifcmExxzGdMU,31813
+dtflow/cli/split.py,sha256=96bhWnxHnjIqifoliLgciApkLbwQU8bWHovK8bcMk9g,3667
+dtflow/cli/stats.py,sha256=Jx3d4X0ftgpzU5q5RAWZEVJWwXviQTF4EAwBmz1IliA,31366
 dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
-dtflow/cli/validate.py,sha256=Frs-jKcDHmYozpmIYZueDSX5o2i1Xn-WW81FGUyUrng,5796
+dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
 dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,362
 dtflow/storage/io.py,sha256=ZH2aSE-S89gpy3z4oTqhcqWf4u10OdkDoyul7o_YBDI,23374
 dtflow/utils/__init__.py,sha256=Pn-ltwV04fBQmeZG7FxInDQmzH29LYOi90LgeLMEuQk,506
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
 dtflow/utils/helpers.py,sha256=JXN176_B2pm53GLVyZ1wj3wrmBJG52Tkw6AMQSdj7M8,791
-dtflow-0.5.8.dist-info/METADATA,sha256=Tm_dfdQfGlShyDt95fNQ87JXiBRnf6mfDgx827h3Rnc,24487
-dtflow-0.5.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.5.8.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.5.8.dist-info/RECORD,,
+dtflow/utils/text_parser.py,sha256=0t2TMOSha4dTiDu9H4ygdb67cI20zhtBH1XavDspL_g,3727
+dtflow-0.5.9.dist-info/METADATA,sha256=Pu92Dz2vj7U_dki4A0e5xgka36BTT9K2PnN1LIeEhN0,25839
+dtflow-0.5.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.9.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.9.dist-info/RECORD,,

dtflow/parallel.py

@@ -1,115 +0,0 @@
-"""
-并行处理模块
-
-提供多进程并行处理工具，用于加速大数据集的 token 统计和 schema 验证。
-"""
-
-from multiprocessing import Pool, cpu_count
-from typing import Callable, List, Optional, TypeVar
-
-T = TypeVar("T")
-R = TypeVar("R")
-
-
-def parallel_map(
-    func: Callable[[T], R],
-    data: List[T],
-    workers: Optional[int] = None,
-    threshold: int = 1000,
-    chunksize: Optional[int] = None,
-) -> List[R]:
-    """
-    并行 map 操作。
-
-    Args:
-        func: 处理函数（必须可 pickle，不能是 lambda 或闭包）
-        data: 数据列表
-        workers: 进程数，None 则使用 CPU 核数
-        threshold: 数据量阈值，低于此值使用串行
-        chunksize: 每个进程的任务块大小，None 则自动计算
-
-    Returns:
-        处理结果列表（保持顺序）
-    """
-    n = len(data)
-
-    # 数据量小或指定单进程，使用串行
-    if n < threshold or workers == 1:
-        return [func(item) for item in data]
-
-    workers = workers or cpu_count()
-    workers = min(workers, n)  # 进程数不超过数据量
-
-    # 自动计算 chunksize
-    if chunksize is None:
-        chunksize = max(1, n // (workers * 4))
-
-    with Pool(processes=workers) as pool:
-        return pool.map(func, data, chunksize=chunksize)
-
-
-def parallel_imap(
-    func: Callable[[T], R],
-    data: List[T],
-    workers: Optional[int] = None,
-    threshold: int = 1000,
-    chunksize: Optional[int] = None,
-):
-    """
-    并行 imap 操作（惰性迭代器版本，支持进度回调）。
-
-    Args:
-        func: 处理函数（必须可 pickle）
-        data: 数据列表
-        workers: 进程数，None 则使用 CPU 核数
-        threshold: 数据量阈值，低于此值使用串行
-        chunksize: 每个进程的任务块大小
-
-    Yields:
-        处理结果（按顺序）
-    """
-    n = len(data)
-
-    # 数据量小或指定单进程，使用串行
-    if n < threshold or workers == 1:
-        for item in data:
-            yield func(item)
-        return
-
-    workers = workers or cpu_count()
-    workers = min(workers, n)
-
-    if chunksize is None:
-        chunksize = max(1, n // (workers * 4))
-
-    with Pool(processes=workers) as pool:
-        for result in pool.imap(func, data, chunksize=chunksize):
-            yield result
-
-
-def get_optimal_workers(data_size: int, default: Optional[int] = None) -> int:
-    """
-    根据数据量计算最优进程数。
-
-    Args:
-        data_size: 数据量
-        default: 用户指定的进程数，None 则自动计算
-
-    Returns:
-        最优进程数
-    """
-    if default is not None:
-        return default
-
-    cpu_cores = cpu_count()
-
-    # 数据量小于阈值，单进程
-    if data_size < 1000:
-        return 1
-
-    # 数据量适中，使用一半 CPU
-    if data_size < 10000:
-        return max(1, cpu_cores // 2)
-
-    # 大数据量，使用全部 CPU
-    return cpu_cores