DeepFabric 4.4.1__py3-none-any.whl → 4.5.1__py3-none-any.whl

deepfabric/training/dataset_utils.py

@@ -0,0 +1,223 @@
+"""Dataset preparation utilities for training.
+
+This module provides utilities for preparing DeepFabric datasets for training,
+including tool filtering to reduce sequence lengths and memory usage.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from datasets import Dataset
+
+logger = logging.getLogger(__name__)
+
+ToolInclusionStrategy = Literal["all", "used_only", "used_plus_related"]
+
+
+def get_used_tool_names(messages: list[dict[str, Any]]) -> set[str]:
+    """Extract tool names that are actually called in a conversation.
+
+    Args:
+        messages: List of message dicts from the conversation
+
+    Returns:
+        Set of tool names that were called
+    """
+    used_tools: set[str] = set()
+
+    for msg in messages:
+        if msg.get("role") == "assistant":
+            tool_calls = msg.get("tool_calls", [])
+            if tool_calls:
+                for tc in tool_calls:
+                    if isinstance(tc, dict):
+                        # OpenAI format: {"function": {"name": "..."}}
+                        func = tc.get("function", {})
+                        if isinstance(func, dict) and func.get("name"):
+                            used_tools.add(func["name"])
+                        # Alternative format: {"name": "..."}
+                        elif tc.get("name"):
+                            used_tools.add(tc["name"])
+
+    return used_tools
+
+
+def clean_tool_schema(tool: dict[str, Any]) -> dict[str, Any]:
+    """Remove null/None values from tool schema to reduce size.
+
+    Args:
+        tool: Tool definition in OpenAI format
+
+    Returns:
+        Cleaned tool definition with nulls removed
+    """
+    if not isinstance(tool, dict):
+        return tool
+
+    cleaned: dict[str, Any] = {}
+
+    for key, value in tool.items():
+        if value is None:
+            continue
+        if isinstance(value, dict):
+            cleaned_value = clean_tool_schema(value)
+            # Only include if dict is not empty after cleaning
+            if cleaned_value:
+                cleaned[key] = cleaned_value
+        elif isinstance(value, list):
+            cleaned_list = []
+            for item in value:
+                if isinstance(item, dict):
+                    cleaned_item = clean_tool_schema(item)
+                    if cleaned_item:
+                        cleaned_list.append(cleaned_item)
+                elif item is not None:
+                    cleaned_list.append(item)
+            if cleaned_list:
+                cleaned[key] = cleaned_list
+        else:
+            cleaned[key] = value
+
+    return cleaned
+
+
+def filter_tools_for_sample(
+    sample: dict[str, Any],
+    strategy: ToolInclusionStrategy = "used_only",
+    min_tools: int = 1,
+    clean_schemas: bool = True,
+) -> dict[str, Any]:
+    """Filter tools in a sample to only include relevant ones.
+
+    Args:
+        sample: Dataset sample with 'messages' and 'tools' fields
+        strategy: Tool inclusion strategy:
+            - "all": Keep all tools (no filtering)
+            - "used_only": Only include tools that are called in the conversation
+            - "used_plus_related": Include used tools plus related ones (not implemented)
+        min_tools: Minimum number of tools to include (fallback if filtering
+            removes all tools)
+        clean_schemas: Whether to remove null values from tool schemas
+
+    Returns:
+        Modified sample with filtered tools
+    """
+    if strategy == "all" and not clean_schemas:
+        return sample
+
+    messages = sample.get("messages", [])
+    all_tools = sample.get("tools", [])
+
+    if not all_tools:
+        return sample
+
+    # Clean schemas if requested
+    if clean_schemas:
+        all_tools = [clean_tool_schema(tool) for tool in all_tools]
+
+    if strategy == "all":
+        sample["tools"] = all_tools
+        return sample
+
+    # Get tools actually used
+    used_names = get_used_tool_names(messages)
+
+    if not used_names:
+        # No tools used - keep minimum number of tools
+        sample["tools"] = all_tools[:min_tools] if min_tools > 0 else []
+        return sample
+
+    # Filter to used tools
+    filtered_tools = []
+    for tool in all_tools:
+        func = tool.get("function", {})
+        if isinstance(func, dict) and func.get("name") in used_names:
+            filtered_tools.append(tool)
+
+    # Ensure minimum tools
+    if len(filtered_tools) < min_tools:
+        # Add more tools from the original list
+        for tool in all_tools:
+            if tool not in filtered_tools:
+                filtered_tools.append(tool)
+                if len(filtered_tools) >= min_tools:
+                    break
+
+    sample["tools"] = filtered_tools
+    return sample
+
+
+def prepare_dataset_for_training(
+    dataset: Dataset,
+    tool_strategy: ToolInclusionStrategy = "used_only",
+    clean_tool_schemas: bool = True,
+    min_tools: int = 1,
+    num_proc: int | None = None,
+) -> Dataset:
+    """Prepare a DeepFabric dataset for training with optimizations.
+
+    This function applies various optimizations to reduce dataset size and
+    memory usage during training:
+    - Filters tools to only include those actually used in each conversation
+    - Removes null values from tool schemas
+    - Can be extended with additional preprocessing steps
+
+    Args:
+        dataset: HuggingFace Dataset with DeepFabric conversation format
+        tool_strategy: How to filter tools (see filter_tools_for_sample)
+        clean_tool_schemas: Whether to remove null values from tool schemas
+        min_tools: Minimum tools to keep per sample
+        num_proc: Number of processes for parallel processing
+
+    Returns:
+        Processed dataset ready for training
+
+    Example:
+        >>> from datasets import load_dataset
+        >>> from deepfabric.training import prepare_dataset_for_training
+        >>>
+        >>> dataset = load_dataset("your/dataset", split="train")
+        >>> prepared = prepare_dataset_for_training(
+        ...     dataset,
+        ...     tool_strategy="used_only",
+        ...     clean_tool_schemas=True,
+        ... )
+        >>> # Now use prepared dataset for training
+    """
+    logger.info(
+        "Preparing dataset for training: tool_strategy=%s, clean_schemas=%s",
+        tool_strategy,
+        clean_tool_schemas,
+    )
+
+    # Get initial stats
+    if "tools" in dataset.column_names:
+        initial_tool_counts = [len(sample.get("tools", []) or []) for sample in dataset]
+        avg_initial = (
+            sum(initial_tool_counts) / len(initial_tool_counts) if initial_tool_counts else 0
+        )
+        logger.info("Initial average tools per sample: %.1f", avg_initial)
+
+    # Apply tool filtering
+    processed = dataset.map(
+        lambda x: filter_tools_for_sample(
+            x,
+            strategy=tool_strategy,
+            min_tools=min_tools,
+            clean_schemas=clean_tool_schemas,
+        ),
+        num_proc=num_proc,
+        desc="Filtering tools",
+    )
+
+    # Log final stats
+    if "tools" in processed.column_names:
+        final_tool_counts = [len(sample.get("tools", []) or []) for sample in processed]
+        avg_final = sum(final_tool_counts) / len(final_tool_counts) if final_tool_counts else 0
+        logger.info("Final average tools per sample: %.1f", avg_final)
+
+    return processed

{deepfabric-4.4.1.dist-info → deepfabric-4.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.4.1
+Version: 4.5.1
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: Luke Hinds <luke@alwaysfurther.ai>
 License-File: LICENSE
@@ -77,6 +77,9 @@ Description-Content-Type: text/markdown
     <a href="https://discord.gg/pPcjYzGvbS">
       <img src="https://img.shields.io/discord/1384081906773131274?color=7289da&label=Discord&logo=discord&logoColor=white" alt="Discord"/>
     </a>
+    <a href="https://www.reddit.com/r/deepfabric/">
+      <img src="https://img.shields.io/badge/Reddit-r%2Fdeepfabric-FF4500?logo=reddit&logoColor=white" alt="Reddit"/>
+    </a>
   </p>
 </div>
 
@@ -86,7 +89,7 @@ What sets DeepFabric apart from other dataset generation tools is its ability to
 
 <img src="/assets/df-demo.gif" width="100%" height="100%"/>
 
-Constrained decoding and response validation, along with real tool executions within isolated webassembly environments, ensure that generated samples strictly adhere to structured schema, variable constraints, and execution correctness, ensuring datasets have exact syntax and structure for use in model training pipelines. Tool definations can be either directly imported from MCP (Model Context Protocol) server schemas and automatically mocked, real life interfaces along with a standard set of common tools (`list_files()`, 'read_file()` etc)
+Constrained decoding and response validation, along with real tool executions within isolated webassembly environments, ensure that generated samples strictly adhere to structured schema, variable constraints, and execution correctness, ensuring datasets have exact syntax and structure for use in model training pipelines. Tool definations can be either directly imported from MCP (Model Context Protocol) server schemas and automatically mocked, real life interfaces along with a standard set of common tools (`list_files()`, `'read_file()` etc)
 
 Once your dataset is generated, it can be automatically uploaded to Hugging Face and directly imported into popular training frameworks like TRL, Unsloth, and Axolotl. 
 
@@ -215,7 +218,7 @@ deepfabric generate config.yaml --output-save-as dataset.jsonl
 Or upload to HuggingFace Hub:
 
 ```bash
-deepfabric upload dataset.jsonl --repo your-username/my-dataset
+deepfabric upload-hf dataset.jsonl --repo your-username/my-dataset
 ```
 
 ### 2. Load and Split for Training

{deepfabric-4.4.1.dist-info → deepfabric-4.5.1.dist-info}/RECORD

@@ -2,8 +2,8 @@ deepfabric/__init__.py,sha256=BRwhcuwXhiDQ6jMlsjhTaPNaw15EyIY3KOLhhlwKcBo,1503
 deepfabric/__main__.py,sha256=Ojx6VFnAWx4wY52VehsWhW85IaEmPb8FP_sGvOk628g,113
 deepfabric/auth.py,sha256=C-GVaImVSqQIw4zm-H8PS19xBnkUOb3Z3M7sALCMsYs,12637
 deepfabric/builders.py,sha256=hqoVKjh9n-XKX6qc12SNgs_L_9Ytb1qxtLTm_ixt3vY,11529
-deepfabric/builders_agent.py,sha256=Z1hyB4wdrBB4jCoB3F-K_z_Lz4g4nzclYk2Uf7YFdFQ,47802
-deepfabric/cli.py,sha256=wOON4U6wHeboPer7Yx6-bPNSuwX1rBFhAb5ywGKXDRk,41963
+deepfabric/builders_agent.py,sha256=MZhtzuSjordwm9XvEoz3Tk52gQgtCaxJE_scl6MAhrM,48494
+deepfabric/cli.py,sha256=iAOmISksizKsxXNbou3733Qeo8-H1rea_ycpr2vKGRE,41980
 deepfabric/config.py,sha256=B3T-Q3nuvkrMFJ1R3NvTYCSM7Baw5OoRovNdrNsbaes,33698
 deepfabric/config_manager.py,sha256=VuHa2MxGYkGwt8fQ-J3d0ctikWGt6lmNShwaqVRWxnY,9067
 deepfabric/constants.py,sha256=MwADziDmnt0zi9t9gG65EM7AJvIQP0FSsXgGj7Yqxm8,2578
@@ -11,14 +11,14 @@ deepfabric/dataset_manager.py,sha256=fJ6VFG05FLTpmbkLKlnVTTi7aim8q7eWI1cgOmKaP5s
 deepfabric/error_codes.py,sha256=HGGWsahUTI8UG996C74X-XgNuaPX8RHo4gOidlaJql4,17630
 deepfabric/exceptions.py,sha256=068iSPLG6iSA1KY8Hp19Tc_Y2dlew5lxihV31dGenUE,1322
 deepfabric/factory.py,sha256=OCqo3w-eiYNWvK_I_egDZuWj192kf18yD3SPj8rrPxU,753
-deepfabric/generator.py,sha256=EoFhe08niU5JZwhjeJrFqh2FjoqLsVsGgsKh5ejrs9w,44011
+deepfabric/generator.py,sha256=p0oI_r4pEPnZ7j4-CQKxmAFVc-plQc17sRbh-GbuTAo,44278
 deepfabric/graph.py,sha256=xyQ0zEESkZhJRYmm8a-IdyNKkXZDzAmlow8bci77u80,20502
-deepfabric/hf_hub.py,sha256=OsiVF-9gVs5E_B8mAiEbA2eIM8kk8WP9r0x3DFCSxsE,7639
+deepfabric/hf_hub.py,sha256=hw2CWqZ3CzyAzMo552VPZKVWtuv-j0TQ2_gV5K0AUto,7670
 deepfabric/kaggle_hub.py,sha256=CXVO1Lv3IRhdO0bp9_IQr6nUs-v5jOWi5k4EwPkbJmw,7927
 deepfabric/metrics.py,sha256=iwtNHBX4ZTYUg2FZgtFcG3U0e9RlV2c1cm1Kp34FeWU,6129
 deepfabric/progress.py,sha256=3XQQrf2pUZlyd-8eRcNATH1v0Oi8JMedVHGbhPcca-8,9354
 deepfabric/prompts.py,sha256=JVFMeeBa2qqOMvmP_xx8bWzZ6ot9eyqOP3u8XzzPx3g,10290
-deepfabric/schemas.py,sha256=6kidH2UhXMXEX5M6v56SGo2BC3GBS9fX0rTGAWqtx1I,36527
+deepfabric/schemas.py,sha256=ef3Ai6i1hNmW8aRyjxc7YJ4ldYp-kn3SauoCdcQK-Lw,36531
 deepfabric/stream_simulator.py,sha256=GzvAxWxHVsuTwgXlqwXNfrTUDn6sND2kJOoQuYg88FA,3028
 deepfabric/topic_manager.py,sha256=xbc3yv8u7EgPcV8rWzpAt3U1fDXM7YIBlvcuvZeBhVo,10992
 deepfabric/topic_model.py,sha256=i_wYpw2kUl8NLodOSaqNu-C4_d6caYT1kPe_vkKjoyw,707
@@ -28,22 +28,23 @@ deepfabric/update_checker.py,sha256=AUa9iUdkGNzu7tWkQRxIlF19YRmKLetwxu-Ys2ONS8Y,
 deepfabric/utils.py,sha256=DWcH7NeVGc_NVJ3v1rdLoQSSWeb2rwfUkDpKO1NGBB8,4507
 deepfabric/validation.py,sha256=UY_oocCQTn77xZ6cIO6d1cRWVHCwBVqKdhUuQ07c7mQ,5123
 deepfabric/evaluation/__init__.py,sha256=7xMLmYXaNC1U7qf88S9fMxWTABoDRiOcimSYfCt_PSo,1224
-deepfabric/evaluation/evaluator.py,sha256=rxmx95wn03H9WE4QehHmItesJK-12VSCfVpwaSzWfYQ,32574
-deepfabric/evaluation/inference.py,sha256=cHAk3IEl3LcQZDMlxNIo8lwSR8g2rQCEjHBhaSzfTXc,4450
-deepfabric/evaluation/metrics.py,sha256=t8-6FP-XfDL4tfzn2sAMHUzgYNxu0Acoq5fGCTuzQqk,11882
+deepfabric/evaluation/evaluator.py,sha256=ExUrL5Zil4DzibzjzngA7dfxnmGIVp9H7319FhLHYmk,33918
+deepfabric/evaluation/inference.py,sha256=y7JA0IsBDwe0sJzVQeItYHAV5wUJn6Bjp1Wsp3r7qYQ,7644
+deepfabric/evaluation/metrics.py,sha256=ITNevYj7CBXzYs-rYhsihO6-rE9n30CYRaVUfdTbcFQ,12026
 deepfabric/evaluation/parser.py,sha256=GZc_Sce0r8zs8s6UCtga8Glp37Ffj2qLJeSfK-Vm8gI,10707
-deepfabric/evaluation/backends/__init__.py,sha256=b9pTrURapGGL_SoLar3rJtgjpSh54vG1SgHxl3V4L5s,760
-deepfabric/evaluation/backends/ollama_backend.py,sha256=obt7sdvRlIyjF0YrdkQcczhz5tVww62_ncM9Uun6Rvc,4387
-deepfabric/evaluation/backends/tool_call_parsers.py,sha256=Q3qZPcyNAyjfrRGExM7tMNipFmbul26znSEfz5giZKc,14167
-deepfabric/evaluation/backends/transformers_backend.py,sha256=3pUx-FFTqavvxNDkGn_1WJbypjVftFFgyHq4YRWFoQ8,12039
+deepfabric/evaluation/backends/__init__.py,sha256=GqC0FfpWmtgJmjHd0kVKNg7g-NjhRoh5h2MtAoOhUOY,827
+deepfabric/evaluation/backends/llm_eval_backend.py,sha256=4jp5tnTp7v_0pHCGhcPbI55ig79-eVxdzooesi2PymA,18827
+deepfabric/evaluation/backends/ollama_backend.py,sha256=mtPp1JtIDRjb76X_rTa1jS1ETzMjte8t3WJjuYV1oDQ,4372
+deepfabric/evaluation/backends/tool_call_parsers.py,sha256=Ufg4Xt3mrDS-WbGor6tOOr4xZNCHk3Co2C-z_o-pAkM,14126
+deepfabric/evaluation/backends/transformers_backend.py,sha256=WcqB9gkayQpjx2Em00lhzJg8RcWdQEYbctDNXLzFChA,14484
 deepfabric/evaluation/evaluators/__init__.py,sha256=NdH65YvanskRGe6r7JepkTNGGt8xA-GLugagU3VQ_WM,353
 deepfabric/evaluation/evaluators/base.py,sha256=1TiLr-_oF9dRmdSgJs94dDbf0gTwRS8TGGz2C1Z3nag,2946
 deepfabric/evaluation/evaluators/registry.py,sha256=VGeb1AHFGkn9TLpcqfuGIZi1jgh7Qw0NNILT6z3Se6M,2171
 deepfabric/evaluation/evaluators/builtin/__init__.py,sha256=s-9SOOZVMHL5E3QR8obZ8Z_mAWBYY_UbxT2hxpdHQZ8,111
-deepfabric/evaluation/evaluators/builtin/tool_calling.py,sha256=wVDAqES_5VrU_mQMB8qJHIyK7N4Lfyb1p16XNYQumDY,3391
+deepfabric/evaluation/evaluators/builtin/tool_calling.py,sha256=S-odftiLM0OBGQryJgvL9Ysqxajiv5QlEyJI6_ABw2g,3697
 deepfabric/evaluation/reporters/__init__.py,sha256=o1eteq-gpfoPkiLpacl0VB-RAQ7Yj-81cTpXgP48M3c,299
 deepfabric/evaluation/reporters/base.py,sha256=YUJYRv5L0rpdd21KhLkz6NmvCzNex0n5c3zT-7rpBnI,1654
-deepfabric/evaluation/reporters/cloud_reporter.py,sha256=y6p4fN4vMDsqOxgyunYdGX7FZ-bihxH7RN_3LhO0ynM,7579
+deepfabric/evaluation/reporters/cloud_reporter.py,sha256=bMiAKbibQ8cf8DmvuRfhXNzekDRPBTbTa7wcmT8piWM,7574
 deepfabric/evaluation/reporters/file_reporter.py,sha256=cRSrtFjYdrUoSwCHPLd6e5irqczP-OyH85npYQ8XkDo,1781
 deepfabric/evaluation/reporters/multi_reporter.py,sha256=Faxi_cgLIVsRzti9n6zd_fxWUToSam65Rqylp6RX97c,1621
 deepfabric/llm/__init__.py,sha256=lfWDLLQ6VWJ4birKVRFh3ypk9zEOfG1ZtU08iD8bd3U,1038
@@ -60,12 +61,13 @@ deepfabric/tools/__init__.py,sha256=hV65lJmVH2qrWCvzHb-IS3VxYP9lal1j8-J3DzBGieM,
 deepfabric/tools/defaults.py,sha256=NcvrYo88OC1ID4U0CuKg_WYKz2pwFowsjBjSMZip-bI,2372
 deepfabric/tools/loader.py,sha256=Bv56D-76JChlK_QXfHLw_rneGLZYRhkn5ETbJMIdJsA,2910
 deepfabric/tools/mcp_client.py,sha256=uQRrlDSVwF0ZatOl9bidBNU7IgXgJKQU-xG50dK0Uy4,23377
-deepfabric/training/__init__.py,sha256=MrfbMhPUa_h4zqpTWDiIu_dXKhmlUH1tJsO5mIsm6tM,920
+deepfabric/training/__init__.py,sha256=MqIyBnloX-4_zqAgoEaGzKXTVXroi40wEs4V7lbQNqk,1563
 deepfabric/training/api_key_prompt.py,sha256=bzcdzeK6ql_8Vz0cbR2vmxxtMRNRFpzYAJx7i8GNI3U,9315
 deepfabric/training/callback.py,sha256=d4heAO5gcBCGXmJIgA9MI9lFBBHnYtdm76rhPh6RDSA,11427
+deepfabric/training/dataset_utils.py,sha256=klx8DoawEwuMigBDP-RpMAfe7FvYxRbhj599MErxBr4,7313
 deepfabric/training/metrics_sender.py,sha256=Fh_qvqrK9mNpTHfG8jgU7t1oSMKbov3mmj3r6o_Q6X4,9347
-deepfabric-4.4.1.dist-info/METADATA,sha256=EGTNsfWuaMrduG33KLktG7mLyVF73HgYw1y3VJEKN2g,22606
-deepfabric-4.4.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-deepfabric-4.4.1.dist-info/entry_points.txt,sha256=zatevils13hfs8x29_vmUyivQ6rTtq7hE2RBusZw1Fo,50
-deepfabric-4.4.1.dist-info/licenses/LICENSE,sha256=-qRt8wmrhQ9aMf7KhmZXc2vrTETYZF-6_T1KCeUhvHY,11340
-deepfabric-4.4.1.dist-info/RECORD,,
+deepfabric-4.5.1.dist-info/METADATA,sha256=fbtl_pTG40JhAPcdC_VIoA988RJ2emjN8iAKvuS_rCo,22789
+deepfabric-4.5.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+deepfabric-4.5.1.dist-info/entry_points.txt,sha256=zatevils13hfs8x29_vmUyivQ6rTtq7hE2RBusZw1Fo,50
+deepfabric-4.5.1.dist-info/licenses/LICENSE,sha256=-qRt8wmrhQ9aMf7KhmZXc2vrTETYZF-6_T1KCeUhvHY,11340
+deepfabric-4.5.1.dist-info/RECORD,,