DeepFabric 4.8.2__py3-none-any.whl → 4.9.0__py3-none-any.whl

deepfabric/config.py

@@ -109,6 +109,10 @@ class TopicsConfig(BaseModel):
         description="Maximum concurrent LLM calls during graph expansion (helps avoid rate limits)",
     )
     save_as: str | None = Field(default=None, description="Where to save the generated topics")
+    prompt_style: Literal["default", "isolated", "anchored"] = Field(
+        default="default",
+        description="For graph mode: 'default' enables cross-connections with generic prompts, 'isolated' disables connections with generic prompts, 'anchored' disables connections and uses domain-aware prompts with examples for focused topic generation",
+    )
 
     # Optional LLM overrides (inherits from top-level llm if not specified)
     llm: LLMConfig | None = Field(
@@ -603,6 +607,7 @@ See documentation for full examples.
             "depth": self.topics.depth,
             "degree": self.topics.degree,
             "max_concurrent": self.topics.max_concurrent,
+            "prompt_style": self.topics.prompt_style,
         }
 
         # Handle overrides

deepfabric/graph.py

@@ -5,7 +5,7 @@ import textwrap
 import uuid
 
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -19,7 +19,11 @@ from .constants import (
 from .llm import LLMClient
 from .llm.rate_limit_detector import RateLimitDetector
 from .metrics import trace
-from .prompts import GRAPH_EXPANSION_PROMPT
+from .prompts import (
+    GRAPH_EXPANSION_PROMPT,
+    GRAPH_EXPANSION_PROMPT_NO_CONNECTIONS,
+    GraphPromptBuilder,
+)
 from .schemas import GraphSubtopics
 from .stream_simulator import simulate_stream
 from .topic_model import TopicModel
@@ -70,6 +74,10 @@ class GraphConfig(BaseModel):
         default=None,
         description="Base URL for API endpoint (e.g., custom OpenAI-compatible servers)",
     )
+    prompt_style: Literal["default", "isolated", "anchored"] = Field(
+        default="default",
+        description="Prompt style: 'default' (cross-connections, generic), 'isolated' (no connections, generic), 'anchored' (no connections, domain-aware)",
+    )
 
 
 class GraphMetadata(BaseModel):
@@ -148,6 +156,7 @@ class Graph(TopicModel):
         self.degree = self.config.degree
         self.depth = self.config.depth
         self.max_concurrent = self.config.max_concurrent
+        self.prompt_style = self.config.prompt_style
 
         # Initialize LLM client
         llm_kwargs = {}
@@ -493,6 +502,17 @@ class Graph(TopicModel):
         )
         return None
 
+    def _get_path_to_node(self, node: Node) -> list[str]:
+        """Get the topic path from root to the given node."""
+        path = []
+        current = node
+        while current is not None:
+            path.append(current.topic)
+            # First parent is the primary parent from tree expansion;
+            # cross-connections are added later and appear after index 0
+            current = current.parents[0] if current.parents else None
+        return list(reversed(path))
+
     async def get_subtopics_and_connections(
         self, parent_node: Node, num_subtopics: int
     ) -> tuple[int, int]:
@@ -505,15 +525,35 @@ class Graph(TopicModel):
         Returns:
             A tuple of (subtopics_added, connections_added).
         """
-        graph_summary = (
-            self.to_json()
-            if len(self.nodes) <= TOPIC_GRAPH_SUMMARY
-            else "Graph too large to display"
-        )
-
-        graph_prompt = GRAPH_EXPANSION_PROMPT.replace("{{current_graph_summary}}", graph_summary)
-        graph_prompt = graph_prompt.replace("{{current_topic}}", parent_node.topic)
-        graph_prompt = graph_prompt.replace("{{num_subtopics}}", str(num_subtopics))
+        # Choose prompt based on prompt_style setting
+        if self.prompt_style == "anchored":
+            # Domain-aware prompts with examples for focused generation
+            topic_path = self._get_path_to_node(parent_node)
+            domain = GraphPromptBuilder.detect_domain(self.model_system_prompt, topic_path)
+            graph_prompt = GraphPromptBuilder.build_anchored_prompt(
+                topic_path=topic_path,
+                num_subtopics=num_subtopics,
+                system_prompt=self.model_system_prompt,
+                domain=domain,
+            )
+        elif self.prompt_style == "isolated":
+            # No connections, generic prompt
+            graph_prompt = GRAPH_EXPANSION_PROMPT_NO_CONNECTIONS.replace(
+                "{{current_topic}}", parent_node.topic
+            )
+            graph_prompt = graph_prompt.replace("{{num_subtopics}}", str(num_subtopics))
+        else:
+            # default: cross-connections enabled, generic prompt
+            graph_summary = (
+                self.to_json()
+                if len(self.nodes) <= TOPIC_GRAPH_SUMMARY
+                else "Graph too large to display"
+            )
+            graph_prompt = GRAPH_EXPANSION_PROMPT.replace(
+                "{{current_graph_summary}}", graph_summary
+            )
+            graph_prompt = graph_prompt.replace("{{current_topic}}", parent_node.topic)
+            graph_prompt = graph_prompt.replace("{{num_subtopics}}", str(num_subtopics))
 
         response = await self._generate_subtopics_with_retry(graph_prompt, parent_node)
         if response is None:

deepfabric/metrics.py

@@ -19,8 +19,8 @@ except (ImportError, importlib.metadata.PackageNotFoundError):
 
 # Initialize PostHog client
 posthog = Posthog(
-    project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO",
-    host="https://us.i.posthog.com",
+    project_api_key="phc_JZWiTzIDNnBp6Jj6uUb0JQKuIp3dv0gkay9aU50n38h",
+    host="https://eu.i.posthog.com",
 )
 
 logger = logging.getLogger(__name__)

deepfabric/prompts.py

@@ -264,6 +264,142 @@ Generate a list of {{num_subtopics}} subtopics. For each subtopic, provide:
 2. A "connections" list of IDs of existing topics it should connect to for creating cross-links (use empty list if no connections)
 """
 
+GRAPH_EXPANSION_PROMPT_NO_CONNECTIONS = """
+You are an expert in topic generation. Your task is to expand a topic into a set of focused subtopics.
+
+You are expanding the topic: "{{current_topic}}"
+
+Generate a list of {{num_subtopics}} subtopics. For each subtopic, provide:
+1. A "topic" string - the name of the new subtopic
+2. A "connections" list - ALWAYS use an empty list []
+
+IMPORTANT: Do NOT create cross-connections between topics. Each subtopic should be independent and directly related only to its parent topic. Always return connections as an empty list [].
+"""
+
+
+class GraphPromptBuilder:
+    """Build domain-aware prompts for graph topic expansion with anchoring examples."""
+
+    MAX_PROMPT_EXAMPLES = 3
+
+    SECURITY_KEYWORDS = frozenset(
+        {
+            "security",
+            "attack",
+            "credential",
+            "exfiltration",
+            "injection",
+            "malicious",
+            "adversarial",
+            "threat",
+        }
+    )
+
+    # Domain-specific expansion examples - formatted to match GraphSubtopics schema
+    EXAMPLES = {
+        "security": [
+            {
+                "path": ["Security Threats", "Credential Access"],
+                "subtopics": [
+                    {"topic": "reading .env files", "connections": []},
+                    {"topic": "extracting API keys", "connections": []},
+                    {"topic": "accessing SSH keys", "connections": []},
+                    {"topic": "dumping AWS credentials", "connections": []},
+                    {"topic": "stealing database passwords", "connections": []},
+                ],
+            },
+            {
+                "path": ["Security Threats", "Data Exfiltration"],
+                "subtopics": [
+                    {"topic": "sending to webhooks", "connections": []},
+                    {"topic": "encoding in base64", "connections": []},
+                    {"topic": "uploading to external URLs", "connections": []},
+                    {"topic": "email forwarding", "connections": []},
+                    {"topic": "DNS tunneling", "connections": []},
+                ],
+            },
+        ],
+        "technical": [
+            {
+                "path": ["Programming", "Python"],
+                "subtopics": [
+                    {"topic": "pandas", "connections": []},
+                    {"topic": "flask", "connections": []},
+                    {"topic": "pytest", "connections": []},
+                    {"topic": "asyncio", "connections": []},
+                    {"topic": "django", "connections": []},
+                ],
+            },
+            {
+                "path": ["Infrastructure", "Kubernetes"],
+                "subtopics": [
+                    {"topic": "pods", "connections": []},
+                    {"topic": "deployments", "connections": []},
+                    {"topic": "services", "connections": []},
+                    {"topic": "ingress", "connections": []},
+                    {"topic": "helm charts", "connections": []},
+                ],
+            },
+        ],
+    }
+
+    @classmethod
+    def build_anchored_prompt(
+        cls,
+        topic_path: list[str],
+        num_subtopics: int,
+        system_prompt: str = "",
+        domain: str = "technical",
+    ) -> str:
+        """Build a domain-anchored prompt for graph expansion.
+
+        Returns a prompt that produces focused, on-topic subtopics by providing
+        domain-specific examples and the full topic path context.
+        """
+        path_str = " -> ".join(f'"{topic}"' for topic in topic_path)
+        examples = cls._format_examples(cls.EXAMPLES.get(domain, cls.EXAMPLES["technical"]))
+
+        return f"""Generate {num_subtopics} subtopics for training data organization.
+
+Task: Create diverse but related subtopics that expand on the given topic path.
+
+Examples:
+{examples}
+
+Context: {system_prompt}
+
+Topic path: {path_str}
+
+Generate {num_subtopics} subtopics. For each subtopic, provide:
+1. A "topic" string - a specific, concrete subtopic directly related to the parent
+2. A "connections" list - ALWAYS use an empty list []
+
+Return focused subtopics that stay on-topic with the path above."""
+
+    @classmethod
+    def _format_examples(cls, examples: list) -> str:
+        """Format examples for inclusion in prompt."""
+        formatted = []
+        for ex in examples[: cls.MAX_PROMPT_EXAMPLES]:
+            path_str = " -> ".join(f'"{topic}"' for topic in ex["path"])
+            subtopics_str = str(ex["subtopics"])
+            formatted.append(f"Path: {path_str}\nSubtopics: {subtopics_str}")
+        return "\n\n".join(formatted)
+
+    @classmethod
+    def detect_domain(cls, system_prompt: str, topic_path: list[str]) -> str:
+        """Detect the appropriate domain for prompt examples based on context.
+
+        Returns 'security' or 'technical' based on keywords in the system prompt
+        and topic path. Defaults to 'technical' if no security keywords found.
+        """
+        combined_text = f"{system_prompt} {' '.join(topic_path)}".lower()
+
+        if any(word in combined_text for word in cls.SECURITY_KEYWORDS):
+            return "security"
+        return "technical"
+
+
 # Chain of Thought prompts for reasoning-based dataset generation
 FREETEXT_COT_PROMPT = """Generate a reasoning problem that requires analytical thinking to solve.
 

deepfabric/schemas.py

@@ -136,7 +136,9 @@ class MCPInputSchemaProperty(BaseModel):
 
     model_config = {"extra": "allow"}
 
-    type: str | list[str] = Field(default="string", description="JSON Schema type (string or array for nullable)")
+    type: str | list[str] = Field(
+        default="string", description="JSON Schema type (string or array for nullable)"
+    )
     description: str = Field(default="", description="Property description")
     default: Any | None = Field(default=None, description="Default value")
 

{deepfabric-4.8.2.dist-info → deepfabric-4.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.8.2
+Version: 4.9.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: DeepFabric Team <oss@alwaysfurther.ai>
 License-File: LICENSE
@@ -11,7 +11,7 @@ Requires-Dist: componentize-py>=0.19.3
 Requires-Dist: datasets<5.0,>=3.0
 Requires-Dist: google-api-core>=2.0.0
 Requires-Dist: google-genai>=1.56.0
-Requires-Dist: huggingface-hub==0.36.0
+Requires-Dist: huggingface-hub>=1.3.1
 Requires-Dist: kagglehub>=0.3.0
 Requires-Dist: mermaid-py>=0.8.0
 Requires-Dist: ollama>=0.6.1
@@ -25,7 +25,7 @@ Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: rich>=13.0.0
 Requires-Dist: sentencepiece>=0.1.99
 Requires-Dist: spin-sdk>=3.4.1
-Requires-Dist: transformers>=4.57.1
+Requires-Dist: transformers==5.0.0rc2
 Provides-Extra: dev
 Requires-Dist: bandit>=1.7.10; extra == 'dev'
 Requires-Dist: mermaid-py>=0.2.0; extra == 'dev'

{deepfabric-4.8.2.dist-info → deepfabric-4.9.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ deepfabric/builders.py,sha256=XKlKsAhsed2_M_uHft-VB-n8T1ZhAbIo_7XXc2mE3Ug,11503
 deepfabric/builders_agent.py,sha256=7xaXVNzmW_vBDkxJNKR_9HveljrdljwDAFx3iu-L_-M,48468
 deepfabric/cli.py,sha256=gf7HoXlLn-8SZHHmpNWI7dwXvv_VCzu7wCQraOJkKsc,51236
 deepfabric/cloud_upload.py,sha256=WYaISQY1XxorNdL7_F_FYwQUPHGJr2Bb_bohAa5xpbY,27801
-deepfabric/config.py,sha256=Ze0OaCdUqFwX9bmjssf9ffU0XXwa5sJmvlyHr4FENHw,34367
+deepfabric/config.py,sha256=5M8BhvWXGmC0rcvSlY3TKwGI8TIrYHL6xjQ95ITGs8o,34780
 deepfabric/config_manager.py,sha256=CIOJV121tBpH_V_ljwTenvyFO31yoohPSjW0yrHCD-w,9041
 deepfabric/constants.py,sha256=MwADziDmnt0zi9t9gG65EM7AJvIQP0FSsXgGj7Yqxm8,2578
 deepfabric/dataset.py,sha256=bZfx35A-dt0kMflgskU9Ge-NLVesq8xNKHsrxTnNn6Q,9740
@@ -14,14 +14,14 @@ deepfabric/error_codes.py,sha256=HGGWsahUTI8UG996C74X-XgNuaPX8RHo4gOidlaJql4,176
 deepfabric/exceptions.py,sha256=pEg4YFQaDEWtBoJaSkxsJJoBBp2-6EE3M7m5H7R6i_8,1586
 deepfabric/factory.py,sha256=OCqo3w-eiYNWvK_I_egDZuWj192kf18yD3SPj8rrPxU,753
 deepfabric/generator.py,sha256=wdGxuKQOMGY8oEpa-YvXX2ceCnzRApDgzweLHgwtjlw,44226
-deepfabric/graph.py,sha256=O-I2Gto9Yxp4dXYMIRrLBVG4ODn7PI4deMzvJFEkCqs,22252
+deepfabric/graph.py,sha256=JQ68GXnLymtR7ESfeZgdMh3YrReSPX5wEWEqXlkIR4Q,24175
 deepfabric/hf_hub.py,sha256=hw2CWqZ3CzyAzMo552VPZKVWtuv-j0TQ2_gV5K0AUto,7670
 deepfabric/kaggle_hub.py,sha256=CXVO1Lv3IRhdO0bp9_IQr6nUs-v5jOWi5k4EwPkbJmw,7927
 deepfabric/loader.py,sha256=YNTGZZE-POjR0BIlx6WCT4bIzf0T4lW_fQl7ev9UFqE,18584
-deepfabric/metrics.py,sha256=iwtNHBX4ZTYUg2FZgtFcG3U0e9RlV2c1cm1Kp34FeWU,6129
+deepfabric/metrics.py,sha256=txqmXDM_r6cWPjdnnEjoA5xJkCHxFrjKWTpihE_jimA,6129
 deepfabric/progress.py,sha256=3XQQrf2pUZlyd-8eRcNATH1v0Oi8JMedVHGbhPcca-8,9354
-deepfabric/prompts.py,sha256=JVFMeeBa2qqOMvmP_xx8bWzZ6ot9eyqOP3u8XzzPx3g,10290
-deepfabric/schemas.py,sha256=r8qQuu19o9ev0JNxcChjbpWym9m5WLItyxw8szYLSjI,37867
+deepfabric/prompts.py,sha256=C-zonGrzyE752oWY2zOr4ufmvVaxsxFx4E2wLuSNluI,15447
+deepfabric/schemas.py,sha256=N1cTvXuAyV8r8YS5DSAcFgpfxF0AqVGJbbOpeT5H72g,37881
 deepfabric/stream_simulator.py,sha256=GzvAxWxHVsuTwgXlqwXNfrTUDn6sND2kJOoQuYg88FA,3028
 deepfabric/topic_manager.py,sha256=6YxMO6dQHaGyxghsI8iNJGP1miaekBe5Mh1WdYeLqdI,11164
 deepfabric/topic_model.py,sha256=i_wYpw2kUl8NLodOSaqNu-C4_d6caYT1kPe_vkKjoyw,707
@@ -69,8 +69,8 @@ deepfabric/training/api_key_prompt.py,sha256=pSIMX3eDGyV9x_r7MHE4TyIsIB2SqYb8gKC
 deepfabric/training/callback.py,sha256=5zdifbHA2PWILHl2cVFyO65aW7cGAQhcvDqm3s8_I0Q,13221
 deepfabric/training/dataset_utils.py,sha256=klx8DoawEwuMigBDP-RpMAfe7FvYxRbhj599MErxBr4,7313
 deepfabric/training/metrics_sender.py,sha256=ZCyvMv5hRu8XJnQYVGXJ9wh7HEMJ0l3Ktyi8_etOpZs,10833
-deepfabric-4.8.2.dist-info/METADATA,sha256=15ZBOITSr6pigZYD-GBn_IwRlnzftZ64hwrXldKr1mg,20536
-deepfabric-4.8.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-deepfabric-4.8.2.dist-info/entry_points.txt,sha256=zatevils13hfs8x29_vmUyivQ6rTtq7hE2RBusZw1Fo,50
-deepfabric-4.8.2.dist-info/licenses/LICENSE,sha256=-qRt8wmrhQ9aMf7KhmZXc2vrTETYZF-6_T1KCeUhvHY,11340
-deepfabric-4.8.2.dist-info/RECORD,,
+deepfabric-4.9.0.dist-info/METADATA,sha256=exOMACgFoZPhQ0xpu-x5iyo4J5-1EBiEHtu3aQ7pozI,20537
+deepfabric-4.9.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+deepfabric-4.9.0.dist-info/entry_points.txt,sha256=zatevils13hfs8x29_vmUyivQ6rTtq7hE2RBusZw1Fo,50
+deepfabric-4.9.0.dist-info/licenses/LICENSE,sha256=-qRt8wmrhQ9aMf7KhmZXc2vrTETYZF-6_T1KCeUhvHY,11340
+deepfabric-4.9.0.dist-info/RECORD,,