sygra 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sygra/__init__.py +426 -0
- sygra/config/configuration.yaml +32 -0
- sygra/config/models.yaml +80 -0
- sygra/config/payload_cfg.json +41 -0
- sygra/configuration/__init__.py +10 -0
- sygra/configuration/loader.py +65 -0
- sygra/core/base_task_executor.py +667 -0
- sygra/core/dataset/data_handler_base.py +65 -0
- sygra/core/dataset/dataset_config.py +220 -0
- sygra/core/dataset/dataset_processor.py +464 -0
- sygra/core/dataset/file_handler.py +169 -0
- sygra/core/dataset/huggingface_handler.py +397 -0
- sygra/core/graph/backend_factory.py +115 -0
- sygra/core/graph/edges/edge_factory.py +343 -0
- sygra/core/graph/functions/edge_condition.py +23 -0
- sygra/core/graph/functions/lambda_function.py +23 -0
- sygra/core/graph/functions/node_processor.py +63 -0
- sygra/core/graph/graph_config.py +460 -0
- sygra/core/graph/langgraph/graph_builder.py +187 -0
- sygra/core/graph/langgraph/langgraph_factory.py +160 -0
- sygra/core/graph/langgraph/langgraph_state.py +20 -0
- sygra/core/graph/nodes/agent_node.py +165 -0
- sygra/core/graph/nodes/base_node.py +182 -0
- sygra/core/graph/nodes/connector_node.py +37 -0
- sygra/core/graph/nodes/lambda_node.py +43 -0
- sygra/core/graph/nodes/llm_node.py +266 -0
- sygra/core/graph/nodes/multi_llm_node.py +66 -0
- sygra/core/graph/nodes/node_utils.py +55 -0
- sygra/core/graph/nodes/special_node.py +41 -0
- sygra/core/graph/nodes/weighted_sampler_node.py +100 -0
- sygra/core/graph/sygra_message.py +11 -0
- sygra/core/graph/sygra_state.py +12 -0
- sygra/core/judge_task_executor.py +15 -0
- sygra/core/models/client/base_client.py +126 -0
- sygra/core/models/client/client_factory.py +350 -0
- sygra/core/models/client/http_client.py +188 -0
- sygra/core/models/client/ollama_client.py +144 -0
- sygra/core/models/client/openai_azure_client.py +162 -0
- sygra/core/models/client/openai_client.py +184 -0
- sygra/core/models/custom_models.py +1338 -0
- sygra/core/models/langgraph/openai_chat_model.py +155 -0
- sygra/core/models/langgraph/sygra_base_chat_model.py +741 -0
- sygra/core/models/langgraph/vllm_chat_model.py +156 -0
- sygra/core/models/model_factory.py +121 -0
- sygra/core/models/model_postprocessor.py +60 -0
- sygra/core/models/structured_output/schemas_factory.py +8 -0
- sygra/core/models/structured_output/structured_output_config.py +212 -0
- sygra/core/resumable_execution.py +570 -0
- sygra/data/__init__.py +283 -0
- sygra/data_mapper/helper.py +117 -0
- sygra/data_mapper/mapper.py +218 -0
- sygra/data_mapper/pipelines.py +58 -0
- sygra/data_mapper/transformations.py +258 -0
- sygra/data_mapper/types.py +21 -0
- sygra/exceptions/__init__.py +61 -0
- sygra/internal/data_quality/llm_based/graph_config.yaml +507 -0
- sygra/internal/data_quality/llm_based/prompt_config.yaml +15 -0
- sygra/internal/data_quality/llm_based/sample.json +46 -0
- sygra/internal/data_quality/llm_based/task_executor.py +178 -0
- sygra/internal/data_quality/metadata_tagging/filter_tags.py +619 -0
- sygra/internal/data_quality/metadata_tagging/graph_config.yaml +136 -0
- sygra/internal/data_quality/metadata_tagging/task_executor.py +142 -0
- sygra/internal/data_quality/metadata_tagging/taxonomy/taxonomy.json +23 -0
- sygra/logger/logger_config.py +222 -0
- sygra/models/__init__.py +74 -0
- sygra/nodes/__init__.py +349 -0
- sygra/processors/data_transform.py +476 -0
- sygra/processors/output_record_generator.py +77 -0
- sygra/recipes/evol_instruct/breadth.py +13 -0
- sygra/recipes/evol_instruct/depth.py +42 -0
- sygra/recipes/evol_instruct/graph_config.yaml +25 -0
- sygra/recipes/evol_instruct/instruct_mgr.py +17 -0
- sygra/recipes/evol_instruct/task_executor.py +19 -0
- sygra/tools/__main__.py +6 -0
- sygra/tools/base_tool.py +52 -0
- sygra/tools/cli.py +36 -0
- sygra/tools/config/config.yaml +16 -0
- sygra/tools/config/config_data_quality.yaml +25 -0
- sygra/tools/config/config_oasst_mapper.yaml +8 -0
- sygra/tools/config/test_config.yaml +10 -0
- sygra/tools/config.py +76 -0
- sygra/tools/executor.py +192 -0
- sygra/tools/registry.py +62 -0
- sygra/tools/toolkits/data_quality/processor.py +254 -0
- sygra/tools/toolkits/data_quality/tasks/conversation_pretokenization_task.py +120 -0
- sygra/tools/toolkits/data_quality/tasks/data_characteristics.py +101 -0
- sygra/tools/toolkits/data_quality/tasks/language_tagging.py +93 -0
- sygra/tools/toolkits/data_quality/tasks/llm_based_quality.py +111 -0
- sygra/tools/toolkits/data_quality/tasks/metadata_tagging.py +147 -0
- sygra/tools/toolkits/data_quality/tasks/ppl_ifd_scorer.py +259 -0
- sygra/tools/toolkits/data_quality/tasks/reward_score.py +178 -0
- sygra/tools/toolkits/data_quality/tasks/ttr_tagging.py +116 -0
- sygra/tools/toolkits/dummy_processor/processor.py +52 -0
- sygra/tools/toolkits/oasst_mapper/processor.py +50 -0
- sygra/utils/audio_utils.py +220 -0
- sygra/utils/constants.py +94 -0
- sygra/utils/decorators.py +18 -0
- sygra/utils/dotenv.py +80 -0
- sygra/utils/graph_utils.py +44 -0
- sygra/utils/image_utils.py +184 -0
- sygra/utils/tool_utils.py +145 -0
- sygra/utils/utils.py +574 -0
- sygra/validators/custom_schemas.py +135 -0
- sygra/validators/custom_validations.py +51 -0
- sygra/validators/schema_validator_base.py +170 -0
- sygra/validators/schema_validator_custom.py +17 -0
- sygra/validators/type_parser.py +109 -0
- sygra/validators/yaml_loader.py +87 -0
- sygra/workflow/__init__.py +1012 -0
- sygra-1.0.0.dist-info/METADATA +218 -0
- sygra-1.0.0.dist-info/RECORD +114 -0
- sygra-1.0.0.dist-info/WHEEL +5 -0
- sygra-1.0.0.dist-info/licenses/LICENSE +201 -0
- sygra-1.0.0.dist-info/top_level.txt +1 -0
sygra/__init__.py
ADDED
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
"""
|
|
2
|
+
SyGra: Graph-oriented Synthetic data generation Pipeline library
|
|
3
|
+
|
|
4
|
+
A powerful Python library for building and executing complex data synthesis workflows
|
|
5
|
+
using graph-based architectures with LLMs, agents, and custom processing nodes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from typing import Any, Callable, Union
|
|
10
|
+
|
|
11
|
+
from .configuration import ConfigLoader, load_config
|
|
12
|
+
from .exceptions import (
|
|
13
|
+
ConfigurationError,
|
|
14
|
+
DataError,
|
|
15
|
+
ExecutionError,
|
|
16
|
+
GraSPError,
|
|
17
|
+
ModelError,
|
|
18
|
+
NodeError,
|
|
19
|
+
TimeoutError,
|
|
20
|
+
ValidationError,
|
|
21
|
+
)
|
|
22
|
+
from .models import ModelConfigBuilder
|
|
23
|
+
from .workflow import Workflow, create_graph
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
from .core.base_task_executor import BaseTaskExecutor, DefaultTaskExecutor # noqa: F401
|
|
27
|
+
from .core.dataset.dataset_processor import DatasetProcessor # noqa: F401
|
|
28
|
+
from .core.graph.graph_config import GraphConfig # noqa: F401
|
|
29
|
+
from .core.graph.sygra_message import SygraMessage # noqa: F401
|
|
30
|
+
from .core.graph.sygra_state import SygraState # noqa: F401
|
|
31
|
+
from .core.judge_task_executor import JudgeQualityTaskExecutor # noqa: F401
|
|
32
|
+
from .core.resumable_execution import ResumableExecutionManager # noqa: F401
|
|
33
|
+
|
|
34
|
+
CORE_AVAILABLE = True
|
|
35
|
+
except ImportError as e:
|
|
36
|
+
logging.warning(f"Core modules not available: {e}")
|
|
37
|
+
CORE_AVAILABLE = False
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
from .core.dataset.dataset_config import DataSourceConfig # noqa: F401
|
|
41
|
+
from .core.dataset.dataset_config import DataSourceType # noqa: F401
|
|
42
|
+
from .core.dataset.dataset_config import OutputConfig # noqa: F401
|
|
43
|
+
from .core.dataset.dataset_config import OutputType # noqa: F401
|
|
44
|
+
from .core.dataset.dataset_config import ShardConfig # noqa: F401
|
|
45
|
+
from .core.dataset.dataset_config import TransformConfig # noqa: F401
|
|
46
|
+
from .core.dataset.file_handler import FileHandler # noqa: F401
|
|
47
|
+
from .core.dataset.huggingface_handler import HuggingFaceHandler # noqa: F401
|
|
48
|
+
|
|
49
|
+
DATA_HANDLERS_AVAILABLE = True
|
|
50
|
+
except ImportError:
|
|
51
|
+
DATA_HANDLERS_AVAILABLE = False
|
|
52
|
+
|
|
53
|
+
# Node modules
|
|
54
|
+
try:
|
|
55
|
+
from .core.graph.nodes.agent_node import AgentNode as CoreAgentNode # noqa: F401
|
|
56
|
+
from .core.graph.nodes.base_node import BaseNode, NodeState, NodeType # noqa: F401
|
|
57
|
+
from .core.graph.nodes.llm_node import LLMNode as CoreLLMNode # noqa: F401
|
|
58
|
+
from .core.graph.nodes.multi_llm_node import MultiLLMNode as CoreMultiLLMNode # noqa: F401
|
|
59
|
+
from .core.graph.nodes.weighted_sampler_node import ( # noqa: F401
|
|
60
|
+
WeightedSamplerNode as CoreWeightedSamplerNode,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
NODES_AVAILABLE = True
|
|
64
|
+
except ImportError:
|
|
65
|
+
NODES_AVAILABLE = False
|
|
66
|
+
|
|
67
|
+
# Model factory modules
|
|
68
|
+
try:
|
|
69
|
+
from .core.models.model_factory import ModelFactory # noqa: F401
|
|
70
|
+
from .core.models.structured_output.schemas_factory import SimpleResponse # noqa: F401
|
|
71
|
+
from .core.models.structured_output.structured_output_config import ( # noqa: F401
|
|
72
|
+
StructuredOutputConfig,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
MODELS_AVAILABLE = True
|
|
76
|
+
except ImportError:
|
|
77
|
+
MODELS_AVAILABLE = False
|
|
78
|
+
|
|
79
|
+
# Utility modules
|
|
80
|
+
try:
|
|
81
|
+
from . import utils
|
|
82
|
+
from .logger.logger_config import logger # noqa: F401
|
|
83
|
+
from .logger.logger_config import reset_to_internal_logger # noqa: F401
|
|
84
|
+
from .logger.logger_config import set_external_logger # noqa: F401
|
|
85
|
+
from .utils import constants # noqa: F401
|
|
86
|
+
|
|
87
|
+
UTILS_AVAILABLE = True
|
|
88
|
+
except ImportError:
|
|
89
|
+
UTILS_AVAILABLE = False
|
|
90
|
+
|
|
91
|
+
# Import node builders
|
|
92
|
+
try:
|
|
93
|
+
from .nodes import AgentNodeBuilder # noqa: F401
|
|
94
|
+
from .nodes import LambdaNodeBuilder # noqa: F401
|
|
95
|
+
from .nodes import LLMNodeBuilder # noqa: F401
|
|
96
|
+
from .nodes import MultiLLMNodeBuilder # noqa: F401
|
|
97
|
+
from .nodes import SubgraphNodeBuilder # noqa: F401
|
|
98
|
+
from .nodes import WeightedSamplerNodeBuilder # noqa: F401
|
|
99
|
+
|
|
100
|
+
NODE_BUILDERS_AVAILABLE = True
|
|
101
|
+
except ImportError:
|
|
102
|
+
NODE_BUILDERS_AVAILABLE = False
|
|
103
|
+
|
|
104
|
+
# Import data utilities
|
|
105
|
+
try:
|
|
106
|
+
from .data import DataSink # noqa: F401
|
|
107
|
+
from .data import DataSinkFactory # noqa: F401
|
|
108
|
+
from .data import DataSource # noqa: F401
|
|
109
|
+
from .data import DataSourceFactory # noqa: F401
|
|
110
|
+
from .data import from_file # noqa: F401
|
|
111
|
+
from .data import from_huggingface # noqa: F401
|
|
112
|
+
from .data import to_file # noqa: F401
|
|
113
|
+
from .data import to_huggingface # noqa: F401
|
|
114
|
+
|
|
115
|
+
DATA_UTILS_AVAILABLE = True
|
|
116
|
+
except ImportError:
|
|
117
|
+
DATA_UTILS_AVAILABLE = False
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
__version__ = "1.0.0"
|
|
121
|
+
__author__ = "SyGra Team"
|
|
122
|
+
__description__ = "Graph-oriented Synthetic data generation Pipeline library"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# Quick utility functions
|
|
126
|
+
def quick_llm(model: str, prompt: str, data_source: str, output: str = "output.json"):
|
|
127
|
+
"""Quick LLM workflow creation."""
|
|
128
|
+
return (
|
|
129
|
+
Workflow(f"quick_llm_{model.replace('/', '_')}")
|
|
130
|
+
.source(data_source)
|
|
131
|
+
.llm(model, prompt)
|
|
132
|
+
.sink(output)
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def quick_agent(
|
|
137
|
+
model: str,
|
|
138
|
+
prompt: str,
|
|
139
|
+
tools: list[str],
|
|
140
|
+
data_source: str,
|
|
141
|
+
output: str = "output.json",
|
|
142
|
+
):
|
|
143
|
+
"""Quick agent workflow creation."""
|
|
144
|
+
return (
|
|
145
|
+
Workflow(f"quick_agent_{model.replace('/', '_')}")
|
|
146
|
+
.source(data_source)
|
|
147
|
+
.agent(model=model, tools=tools, prompt=prompt)
|
|
148
|
+
.sink(output)
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def quick_multi_llm(
|
|
153
|
+
models: dict[str, Any], prompt: str, data_source: str, output: str = "output.json"
|
|
154
|
+
):
|
|
155
|
+
"""Quick multi-LLM workflow creation."""
|
|
156
|
+
return (
|
|
157
|
+
Workflow("quick_multi_llm")
|
|
158
|
+
.source(data_source)
|
|
159
|
+
.multi_llm(models=models, prompt=prompt)
|
|
160
|
+
.sink(output)
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def execute_task(task_name: str, **kwargs):
|
|
165
|
+
"""Execute an existing task configuration."""
|
|
166
|
+
workflow = Workflow(task_name)
|
|
167
|
+
return workflow.run(**kwargs)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def create_multimodal_workflow(name: str) -> Workflow:
|
|
171
|
+
"""Create workflow with multimodal capabilities enabled."""
|
|
172
|
+
workflow = Workflow(name)
|
|
173
|
+
workflow._supports_multimodal = True
|
|
174
|
+
return workflow
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def create_resumable_workflow(name: str) -> Workflow:
|
|
178
|
+
"""Create workflow with resumable execution enabled."""
|
|
179
|
+
workflow = Workflow(name)
|
|
180
|
+
workflow.resumable(True)
|
|
181
|
+
return workflow
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def create_quality_workflow(name: str) -> Workflow:
|
|
185
|
+
"""Create workflow with quality tagging enabled."""
|
|
186
|
+
workflow = Workflow(name)
|
|
187
|
+
workflow.quality_tagging(True)
|
|
188
|
+
return workflow
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def create_chat_workflow(name: str, conversation_type: str = "multiturn") -> Workflow:
|
|
192
|
+
"""Create workflow optimized for chat/conversation generation."""
|
|
193
|
+
workflow = Workflow(name)
|
|
194
|
+
workflow.chat_conversation(conversation_type)
|
|
195
|
+
return workflow
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def create_structured_schema(fields: dict[str, str], name: str = "CustomSchema") -> dict[str, Any]:
|
|
199
|
+
"""Create structured output schema configuration."""
|
|
200
|
+
return {
|
|
201
|
+
"enabled": True,
|
|
202
|
+
"schema": {
|
|
203
|
+
"name": name,
|
|
204
|
+
"fields": {
|
|
205
|
+
field_name: {"type": field_type} for field_name, field_type in fields.items()
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def pydantic_schema(model_class: str) -> dict[str, Any]:
|
|
212
|
+
"""Create structured output schema from Pydantic model class path."""
|
|
213
|
+
return {"enabled": True, "schema": model_class}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def create_processor_config(processor: Union[str, Callable], **params) -> dict[str, Any]:
|
|
217
|
+
"""Create processor configuration."""
|
|
218
|
+
if callable(processor):
|
|
219
|
+
processor_path = f"{processor.__module__}.{processor.__name__}"
|
|
220
|
+
else:
|
|
221
|
+
processor_path = processor
|
|
222
|
+
|
|
223
|
+
config: dict[str, Any] = {"processor": processor_path}
|
|
224
|
+
if params:
|
|
225
|
+
config["params"] = params
|
|
226
|
+
|
|
227
|
+
return config
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def create_transformation_config(transform: Union[str, Callable], **params) -> dict[str, Any]:
|
|
231
|
+
"""Create data transformation configuration."""
|
|
232
|
+
if callable(transform):
|
|
233
|
+
transform_path = f"{transform.__module__}.{transform.__name__}"
|
|
234
|
+
else:
|
|
235
|
+
transform_path = transform
|
|
236
|
+
|
|
237
|
+
config: dict[str, Any] = {"transform": transform_path}
|
|
238
|
+
if params:
|
|
239
|
+
config["params"] = params
|
|
240
|
+
|
|
241
|
+
return config
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def get_version() -> str:
|
|
245
|
+
"""Get library version."""
|
|
246
|
+
return __version__
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def setup_logging(level: str = "INFO") -> None:
|
|
250
|
+
"""Setup logging."""
|
|
251
|
+
logging.getLogger("sygra").setLevel(getattr(logging, level.upper()))
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def validate_environment() -> dict[str, bool]:
|
|
255
|
+
"""Validate environment setup."""
|
|
256
|
+
return {
|
|
257
|
+
"core_available": CORE_AVAILABLE,
|
|
258
|
+
"data_handlers_available": DATA_HANDLERS_AVAILABLE,
|
|
259
|
+
"nodes_available": NODES_AVAILABLE,
|
|
260
|
+
"models_available": MODELS_AVAILABLE,
|
|
261
|
+
"utils_available": UTILS_AVAILABLE,
|
|
262
|
+
"node_builders_available": NODE_BUILDERS_AVAILABLE,
|
|
263
|
+
"data_utils_available": DATA_UTILS_AVAILABLE,
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def get_info() -> dict[str, Any]:
|
|
268
|
+
"""Get library information and feature availability."""
|
|
269
|
+
return {
|
|
270
|
+
"version": __version__,
|
|
271
|
+
"author": __author__,
|
|
272
|
+
"description": __description__,
|
|
273
|
+
"features": validate_environment(),
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def list_available_models() -> list[str]:
|
|
278
|
+
"""list available models from framework configuration."""
|
|
279
|
+
if not UTILS_AVAILABLE:
|
|
280
|
+
return ["Framework not available - cannot list models"]
|
|
281
|
+
|
|
282
|
+
try:
|
|
283
|
+
model_configs = utils.utils.load_model_config()
|
|
284
|
+
return list(model_configs.keys())
|
|
285
|
+
except Exception as e:
|
|
286
|
+
return [f"Error loading models: {e}"]
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def get_model_info(model_name: str) -> dict[str, Any]:
|
|
290
|
+
"""Get information about a specific model."""
|
|
291
|
+
if not UTILS_AVAILABLE:
|
|
292
|
+
return {"error": "Framework not available"}
|
|
293
|
+
|
|
294
|
+
try:
|
|
295
|
+
# Ensure the mapping type is explicit so mypy can infer correct return type
|
|
296
|
+
model_configs: dict[str, dict[str, Any]] = utils.utils.load_model_config()
|
|
297
|
+
return model_configs.get(model_name, {"error": f"Model {model_name} not found"})
|
|
298
|
+
except Exception as e:
|
|
299
|
+
return {"error": f"Error loading model info: {e}"}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# Main exports
|
|
303
|
+
__all__ = [
|
|
304
|
+
# Main classes
|
|
305
|
+
"Workflow",
|
|
306
|
+
"create_graph",
|
|
307
|
+
# Configuration
|
|
308
|
+
"load_config",
|
|
309
|
+
"ConfigLoader",
|
|
310
|
+
# Model utilities
|
|
311
|
+
"ModelConfigBuilder",
|
|
312
|
+
# Quick functions
|
|
313
|
+
"quick_llm",
|
|
314
|
+
"quick_agent",
|
|
315
|
+
"quick_multi_llm",
|
|
316
|
+
"execute_task",
|
|
317
|
+
# Advanced workflow builders
|
|
318
|
+
"create_multimodal_workflow",
|
|
319
|
+
"create_resumable_workflow",
|
|
320
|
+
"create_quality_workflow",
|
|
321
|
+
"create_chat_workflow",
|
|
322
|
+
# Structured output helpers
|
|
323
|
+
"create_structured_schema",
|
|
324
|
+
"pydantic_schema",
|
|
325
|
+
# Processing helpers
|
|
326
|
+
"create_processor_config",
|
|
327
|
+
"create_transformation_config",
|
|
328
|
+
# Utilities
|
|
329
|
+
"get_version",
|
|
330
|
+
"setup_logging",
|
|
331
|
+
"validate_environment",
|
|
332
|
+
"get_info",
|
|
333
|
+
"list_available_models",
|
|
334
|
+
"get_model_info",
|
|
335
|
+
# Exceptions
|
|
336
|
+
"GraSPError",
|
|
337
|
+
"ValidationError",
|
|
338
|
+
"ExecutionError",
|
|
339
|
+
"ConfigurationError",
|
|
340
|
+
"NodeError",
|
|
341
|
+
"DataError",
|
|
342
|
+
"ModelError",
|
|
343
|
+
"TimeoutError",
|
|
344
|
+
]
|
|
345
|
+
|
|
346
|
+
# Add conditionally available imports to __all__
|
|
347
|
+
if CORE_AVAILABLE:
|
|
348
|
+
__all__.extend(
|
|
349
|
+
[
|
|
350
|
+
"BaseTaskExecutor",
|
|
351
|
+
"DefaultTaskExecutor",
|
|
352
|
+
"JudgeQualityTaskExecutor",
|
|
353
|
+
"GraphConfig",
|
|
354
|
+
"SygraState",
|
|
355
|
+
"SygraMessage",
|
|
356
|
+
"ResumableExecutionManager",
|
|
357
|
+
"DatasetProcessor",
|
|
358
|
+
]
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
if DATA_HANDLERS_AVAILABLE:
|
|
362
|
+
__all__.extend(
|
|
363
|
+
[
|
|
364
|
+
"DataSourceConfig",
|
|
365
|
+
"OutputConfig",
|
|
366
|
+
"DataSourceType",
|
|
367
|
+
"OutputType",
|
|
368
|
+
"TransformConfig",
|
|
369
|
+
"ShardConfig",
|
|
370
|
+
"FileHandler",
|
|
371
|
+
"HuggingFaceHandler",
|
|
372
|
+
]
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
if NODES_AVAILABLE:
|
|
376
|
+
__all__.extend(
|
|
377
|
+
[
|
|
378
|
+
"BaseNode",
|
|
379
|
+
"NodeType",
|
|
380
|
+
"NodeState",
|
|
381
|
+
"CoreLLMNode",
|
|
382
|
+
"CoreAgentNode",
|
|
383
|
+
"CoreMultiLLMNode",
|
|
384
|
+
"CoreWeightedSamplerNode",
|
|
385
|
+
]
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
if MODELS_AVAILABLE:
|
|
389
|
+
__all__.extend(["ModelFactory", "StructuredOutputConfig", "SimpleResponse"])
|
|
390
|
+
|
|
391
|
+
if UTILS_AVAILABLE:
|
|
392
|
+
__all__.extend(
|
|
393
|
+
[
|
|
394
|
+
"utils",
|
|
395
|
+
"constants",
|
|
396
|
+
"logger",
|
|
397
|
+
"set_external_logger",
|
|
398
|
+
"reset_to_internal_logger",
|
|
399
|
+
]
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
if NODE_BUILDERS_AVAILABLE:
|
|
403
|
+
__all__.extend(
|
|
404
|
+
[
|
|
405
|
+
"LLMNodeBuilder",
|
|
406
|
+
"AgentNodeBuilder",
|
|
407
|
+
"MultiLLMNodeBuilder",
|
|
408
|
+
"LambdaNodeBuilder",
|
|
409
|
+
"WeightedSamplerNodeBuilder",
|
|
410
|
+
"SubgraphNodeBuilder",
|
|
411
|
+
]
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
if DATA_UTILS_AVAILABLE:
|
|
415
|
+
__all__.extend(
|
|
416
|
+
[
|
|
417
|
+
"DataSource",
|
|
418
|
+
"DataSink",
|
|
419
|
+
"DataSourceFactory",
|
|
420
|
+
"DataSinkFactory",
|
|
421
|
+
"from_file",
|
|
422
|
+
"from_huggingface",
|
|
423
|
+
"to_file",
|
|
424
|
+
"to_huggingface",
|
|
425
|
+
]
|
|
426
|
+
)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
default_transformations:
|
|
2
|
+
- transform: sygra.processors.data_transform.CreateImageUrlTransform
|
|
3
|
+
- transform: sygra.processors.data_transform.CreateAudioUrlTransform
|
|
4
|
+
|
|
5
|
+
model_config:
|
|
6
|
+
ssl_verify: true
|
|
7
|
+
ssl_cert: None
|
|
8
|
+
|
|
9
|
+
post_generation_tasks:
|
|
10
|
+
oasst_mapper:
|
|
11
|
+
required: "yes"
|
|
12
|
+
type: "sft"
|
|
13
|
+
intermediate_writing: "yes"
|
|
14
|
+
|
|
15
|
+
data_quality:
|
|
16
|
+
tasks:
|
|
17
|
+
- name: "conversation_pretokenization"
|
|
18
|
+
params:
|
|
19
|
+
hf_chat_template_model_id: "Qwen/Qwen2.5-32B-Instruct"
|
|
20
|
+
- name: "metadata_tagging"
|
|
21
|
+
- name: "llm_based_quality"
|
|
22
|
+
- name: "data_characteristics"
|
|
23
|
+
params:
|
|
24
|
+
model: "Qwen/Qwen2.5-32B-Instruct"
|
|
25
|
+
- name: "language_tagging"
|
|
26
|
+
- name: "lexical_diversity"
|
|
27
|
+
- name: "ppl_score"
|
|
28
|
+
params:
|
|
29
|
+
model_config:
|
|
30
|
+
model_type: "vllm" # or "tgi"
|
|
31
|
+
url: # Add the URL of the model server
|
|
32
|
+
model_serving_name: # Add the serving name of the model
|
sygra/config/models.yaml
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
mixtral8x7b:
|
|
2
|
+
model_type: tgi
|
|
3
|
+
# URL and auth_token should be defined at .env file as SYGRA_MIXTRAL8X7B_URL and SYGRA_MIXTRAL8X7B_TOKEN
|
|
4
|
+
hf_chat_template_model_id: meta-math/MetaMath-Mistral-7B
|
|
5
|
+
parameters:
|
|
6
|
+
max_new_tokens: 500
|
|
7
|
+
temperature: 1.0
|
|
8
|
+
|
|
9
|
+
mixtral_instruct_8x22b:
|
|
10
|
+
model_type: vllm
|
|
11
|
+
# URL and auth_token should be defined at .env file as SYGRA_MIXTRAL_INSTRUCT_8X22B_URL and SYGRA_MIXTRAL_INSTRUCT_8X22B_TOKEN
|
|
12
|
+
hf_chat_template_model_id: meta-math/MetaMath-Mistral-7B
|
|
13
|
+
parameters:
|
|
14
|
+
max_tokens: 1024
|
|
15
|
+
temperature: 1.0
|
|
16
|
+
stop: ["</s>", "[INST]"]
|
|
17
|
+
|
|
18
|
+
mistralai:
|
|
19
|
+
model_type: mistralai
|
|
20
|
+
model: azureai
|
|
21
|
+
# URL and auth_token should be defined at .env file as SYGRA_MISTRALAI_URL and SYGRA_MISTRALAI_TOKEN
|
|
22
|
+
parameters:
|
|
23
|
+
max_tokens: 500
|
|
24
|
+
temperature: 1.0
|
|
25
|
+
|
|
26
|
+
gpt4:
|
|
27
|
+
model_type: azure_openai
|
|
28
|
+
model: gpt-4-32k
|
|
29
|
+
# URL and api_key should be defined at .env file as SYGRA_GPT4_URL and SYGRA_GPT4_TOKEN
|
|
30
|
+
api_version: 2024-05-01-preview
|
|
31
|
+
parameters:
|
|
32
|
+
max_tokens: 500
|
|
33
|
+
temperature: 1.0
|
|
34
|
+
|
|
35
|
+
gpt-4o:
|
|
36
|
+
model_type: azure_openai
|
|
37
|
+
model: gpt-4o
|
|
38
|
+
# URL and api_key should be defined at .env file as SYGRA_GPT-4O_URL and SYGRA_GPT-4O_TOKEN
|
|
39
|
+
api_version: 2024-02-15-preview
|
|
40
|
+
parameters:
|
|
41
|
+
max_tokens: 500
|
|
42
|
+
temperature: 1.0
|
|
43
|
+
|
|
44
|
+
gpt-4o-mini:
|
|
45
|
+
model_type: azure_openai
|
|
46
|
+
model: gpt-4o-mini
|
|
47
|
+
# URL and api_key should be defined at .env file as SYGRA_GPT-4O-MINI_URL and SYGRA_GPT-4O-MINI_TOKEN
|
|
48
|
+
api_version: 2024-08-01-preview
|
|
49
|
+
parameters:
|
|
50
|
+
max_tokens: 5000
|
|
51
|
+
temperature: 0.0001
|
|
52
|
+
|
|
53
|
+
#QWEN VL 72b deployed in vllm
|
|
54
|
+
qwen_vl_72b:
|
|
55
|
+
# URL and api_key should be defined at .env file as SYGRA_QWEN_VL_72B_URL and SYGRA_QWEN_VL_72B_TOKEN
|
|
56
|
+
hf_chat_template_model_id: Qwen/Qwen2.5-VL-72B-Instruct
|
|
57
|
+
model_type: vllm
|
|
58
|
+
parameters:
|
|
59
|
+
max_tokens: 2048
|
|
60
|
+
temperature: 1.0
|
|
61
|
+
|
|
62
|
+
#QWEN 32B deployed in vllm
|
|
63
|
+
qwen3_32b:
|
|
64
|
+
# URL and api_key should be defined at .env file as SYGRA_QWEN3_32B_URL and SYGRA_QWEN3_32B_TOKEN
|
|
65
|
+
model_serving_name: qwen3_32b
|
|
66
|
+
hf_chat_template_model_id: Qwen/Qwen3-32B
|
|
67
|
+
model_type: vllm
|
|
68
|
+
parameters:
|
|
69
|
+
max_tokens: 2048
|
|
70
|
+
temperature: 0
|
|
71
|
+
|
|
72
|
+
#qwen3_1.7b
|
|
73
|
+
qwen3_1.7b:
|
|
74
|
+
model_type: ollama
|
|
75
|
+
# URL should be defined at .env file as SYGRA_QWEN3_1.7B_URL. auth_token is not needed
|
|
76
|
+
# Default URL for ollama is http://localhost:11434
|
|
77
|
+
hf_chat_template_model_id: Qwen/Qwen3-1.7B
|
|
78
|
+
post_process: sygra.core.models.model_postprocessor.RemoveThinkData
|
|
79
|
+
parameters:
|
|
80
|
+
temperature: 0.8
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"triton": {
|
|
3
|
+
"default": {
|
|
4
|
+
"payload_json": {
|
|
5
|
+
"id": "42",
|
|
6
|
+
"inputs": [
|
|
7
|
+
{
|
|
8
|
+
"data": "<replace this : Array of json>",
|
|
9
|
+
"datatype": "BYTES",
|
|
10
|
+
"name": "request",
|
|
11
|
+
"shape": [
|
|
12
|
+
1,
|
|
13
|
+
1
|
|
14
|
+
]
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"data": "<replace this: Array of options json>",
|
|
18
|
+
"datatype": "BYTES",
|
|
19
|
+
"name": "options",
|
|
20
|
+
"shape": [
|
|
21
|
+
1,
|
|
22
|
+
1
|
|
23
|
+
]
|
|
24
|
+
}
|
|
25
|
+
],
|
|
26
|
+
"outputs": [
|
|
27
|
+
{
|
|
28
|
+
"name": "response"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"name": "response_metadata"
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"name": "error"
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
},
|
|
38
|
+
"response_key": "model_output"
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from sygra.configuration.loader import ConfigLoader
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def load_config(config_path):
|
|
5
|
+
"""Load workflow from existing YAML configuration."""
|
|
6
|
+
loader = ConfigLoader()
|
|
7
|
+
return loader.load_and_create(config_path)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
__all__ = ["ConfigLoader", "load_config"]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Any, Union, cast
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from sygra.core.dataset.dataset_config import DataSourceConfig, OutputConfig # noqa: F401
|
|
9
|
+
from sygra.core.graph.graph_config import GraphConfig # noqa: F401
|
|
10
|
+
from sygra.utils import utils
|
|
11
|
+
|
|
12
|
+
UTILS_AVAILABLE = True
|
|
13
|
+
except ImportError:
|
|
14
|
+
UTILS_AVAILABLE = False
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ConfigLoader:
|
|
18
|
+
"""Loads existing SyGra YAML configurations and converts them to library objects."""
|
|
19
|
+
|
|
20
|
+
def load(self, config_path: Union[str, Path, dict[str, Any]]) -> dict[str, Any]:
|
|
21
|
+
"""Load configuration from file or dictionary."""
|
|
22
|
+
if isinstance(config_path, dict):
|
|
23
|
+
return config_path
|
|
24
|
+
|
|
25
|
+
config_path = Path(config_path)
|
|
26
|
+
if not config_path.exists():
|
|
27
|
+
if UTILS_AVAILABLE:
|
|
28
|
+
task_config_path = utils.get_file_in_task_dir(config_path.stem, "graph_config.yaml")
|
|
29
|
+
if os.path.exists(task_config_path):
|
|
30
|
+
config_path = Path(task_config_path)
|
|
31
|
+
else:
|
|
32
|
+
raise FileNotFoundError(f"Configuration file not found: {config_path}")
|
|
33
|
+
else:
|
|
34
|
+
raise FileNotFoundError(f"Configuration file not found: {config_path}")
|
|
35
|
+
|
|
36
|
+
with open(config_path, "r") as f:
|
|
37
|
+
config = cast(dict[str, Any], yaml.safe_load(f))
|
|
38
|
+
|
|
39
|
+
return config
|
|
40
|
+
|
|
41
|
+
def load_and_create(self, config_path: Union[str, Path, dict[str, Any]]):
|
|
42
|
+
"""Load config and create appropriate Workflow or Graph object."""
|
|
43
|
+
config = self.load(config_path)
|
|
44
|
+
|
|
45
|
+
# Import here to avoid circular imports
|
|
46
|
+
from ..workflow import Workflow
|
|
47
|
+
|
|
48
|
+
workflow = Workflow()
|
|
49
|
+
workflow._config = config
|
|
50
|
+
|
|
51
|
+
workflow._supports_subgraphs = True
|
|
52
|
+
workflow._supports_multimodal = True
|
|
53
|
+
workflow._supports_resumable = True
|
|
54
|
+
workflow._supports_quality = True
|
|
55
|
+
workflow._supports_oasst = True
|
|
56
|
+
|
|
57
|
+
if isinstance(config_path, (str, Path)):
|
|
58
|
+
workflow.name = Path(config_path).parent.name
|
|
59
|
+
else:
|
|
60
|
+
workflow.name = config.get("task_name", "loaded_workflow")
|
|
61
|
+
|
|
62
|
+
return workflow
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
__all__ = ["ConfigLoader"]
|