sygra 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. sygra/__init__.py +426 -0
  2. sygra/config/configuration.yaml +32 -0
  3. sygra/config/models.yaml +80 -0
  4. sygra/config/payload_cfg.json +41 -0
  5. sygra/configuration/__init__.py +10 -0
  6. sygra/configuration/loader.py +65 -0
  7. sygra/core/base_task_executor.py +667 -0
  8. sygra/core/dataset/data_handler_base.py +65 -0
  9. sygra/core/dataset/dataset_config.py +220 -0
  10. sygra/core/dataset/dataset_processor.py +464 -0
  11. sygra/core/dataset/file_handler.py +169 -0
  12. sygra/core/dataset/huggingface_handler.py +397 -0
  13. sygra/core/graph/backend_factory.py +115 -0
  14. sygra/core/graph/edges/edge_factory.py +343 -0
  15. sygra/core/graph/functions/edge_condition.py +23 -0
  16. sygra/core/graph/functions/lambda_function.py +23 -0
  17. sygra/core/graph/functions/node_processor.py +63 -0
  18. sygra/core/graph/graph_config.py +460 -0
  19. sygra/core/graph/langgraph/graph_builder.py +187 -0
  20. sygra/core/graph/langgraph/langgraph_factory.py +160 -0
  21. sygra/core/graph/langgraph/langgraph_state.py +20 -0
  22. sygra/core/graph/nodes/agent_node.py +165 -0
  23. sygra/core/graph/nodes/base_node.py +182 -0
  24. sygra/core/graph/nodes/connector_node.py +37 -0
  25. sygra/core/graph/nodes/lambda_node.py +43 -0
  26. sygra/core/graph/nodes/llm_node.py +266 -0
  27. sygra/core/graph/nodes/multi_llm_node.py +66 -0
  28. sygra/core/graph/nodes/node_utils.py +55 -0
  29. sygra/core/graph/nodes/special_node.py +41 -0
  30. sygra/core/graph/nodes/weighted_sampler_node.py +100 -0
  31. sygra/core/graph/sygra_message.py +11 -0
  32. sygra/core/graph/sygra_state.py +12 -0
  33. sygra/core/judge_task_executor.py +15 -0
  34. sygra/core/models/client/base_client.py +126 -0
  35. sygra/core/models/client/client_factory.py +350 -0
  36. sygra/core/models/client/http_client.py +188 -0
  37. sygra/core/models/client/ollama_client.py +144 -0
  38. sygra/core/models/client/openai_azure_client.py +162 -0
  39. sygra/core/models/client/openai_client.py +184 -0
  40. sygra/core/models/custom_models.py +1338 -0
  41. sygra/core/models/langgraph/openai_chat_model.py +155 -0
  42. sygra/core/models/langgraph/sygra_base_chat_model.py +741 -0
  43. sygra/core/models/langgraph/vllm_chat_model.py +156 -0
  44. sygra/core/models/model_factory.py +121 -0
  45. sygra/core/models/model_postprocessor.py +60 -0
  46. sygra/core/models/structured_output/schemas_factory.py +8 -0
  47. sygra/core/models/structured_output/structured_output_config.py +212 -0
  48. sygra/core/resumable_execution.py +570 -0
  49. sygra/data/__init__.py +283 -0
  50. sygra/data_mapper/helper.py +117 -0
  51. sygra/data_mapper/mapper.py +218 -0
  52. sygra/data_mapper/pipelines.py +58 -0
  53. sygra/data_mapper/transformations.py +258 -0
  54. sygra/data_mapper/types.py +21 -0
  55. sygra/exceptions/__init__.py +61 -0
  56. sygra/internal/data_quality/llm_based/graph_config.yaml +507 -0
  57. sygra/internal/data_quality/llm_based/prompt_config.yaml +15 -0
  58. sygra/internal/data_quality/llm_based/sample.json +46 -0
  59. sygra/internal/data_quality/llm_based/task_executor.py +178 -0
  60. sygra/internal/data_quality/metadata_tagging/filter_tags.py +619 -0
  61. sygra/internal/data_quality/metadata_tagging/graph_config.yaml +136 -0
  62. sygra/internal/data_quality/metadata_tagging/task_executor.py +142 -0
  63. sygra/internal/data_quality/metadata_tagging/taxonomy/taxonomy.json +23 -0
  64. sygra/logger/logger_config.py +222 -0
  65. sygra/models/__init__.py +74 -0
  66. sygra/nodes/__init__.py +349 -0
  67. sygra/processors/data_transform.py +476 -0
  68. sygra/processors/output_record_generator.py +77 -0
  69. sygra/recipes/evol_instruct/breadth.py +13 -0
  70. sygra/recipes/evol_instruct/depth.py +42 -0
  71. sygra/recipes/evol_instruct/graph_config.yaml +25 -0
  72. sygra/recipes/evol_instruct/instruct_mgr.py +17 -0
  73. sygra/recipes/evol_instruct/task_executor.py +19 -0
  74. sygra/tools/__main__.py +6 -0
  75. sygra/tools/base_tool.py +52 -0
  76. sygra/tools/cli.py +36 -0
  77. sygra/tools/config/config.yaml +16 -0
  78. sygra/tools/config/config_data_quality.yaml +25 -0
  79. sygra/tools/config/config_oasst_mapper.yaml +8 -0
  80. sygra/tools/config/test_config.yaml +10 -0
  81. sygra/tools/config.py +76 -0
  82. sygra/tools/executor.py +192 -0
  83. sygra/tools/registry.py +62 -0
  84. sygra/tools/toolkits/data_quality/processor.py +254 -0
  85. sygra/tools/toolkits/data_quality/tasks/conversation_pretokenization_task.py +120 -0
  86. sygra/tools/toolkits/data_quality/tasks/data_characteristics.py +101 -0
  87. sygra/tools/toolkits/data_quality/tasks/language_tagging.py +93 -0
  88. sygra/tools/toolkits/data_quality/tasks/llm_based_quality.py +111 -0
  89. sygra/tools/toolkits/data_quality/tasks/metadata_tagging.py +147 -0
  90. sygra/tools/toolkits/data_quality/tasks/ppl_ifd_scorer.py +259 -0
  91. sygra/tools/toolkits/data_quality/tasks/reward_score.py +178 -0
  92. sygra/tools/toolkits/data_quality/tasks/ttr_tagging.py +116 -0
  93. sygra/tools/toolkits/dummy_processor/processor.py +52 -0
  94. sygra/tools/toolkits/oasst_mapper/processor.py +50 -0
  95. sygra/utils/audio_utils.py +220 -0
  96. sygra/utils/constants.py +94 -0
  97. sygra/utils/decorators.py +18 -0
  98. sygra/utils/dotenv.py +80 -0
  99. sygra/utils/graph_utils.py +44 -0
  100. sygra/utils/image_utils.py +184 -0
  101. sygra/utils/tool_utils.py +145 -0
  102. sygra/utils/utils.py +574 -0
  103. sygra/validators/custom_schemas.py +135 -0
  104. sygra/validators/custom_validations.py +51 -0
  105. sygra/validators/schema_validator_base.py +170 -0
  106. sygra/validators/schema_validator_custom.py +17 -0
  107. sygra/validators/type_parser.py +109 -0
  108. sygra/validators/yaml_loader.py +87 -0
  109. sygra/workflow/__init__.py +1012 -0
  110. sygra-1.0.0.dist-info/METADATA +218 -0
  111. sygra-1.0.0.dist-info/RECORD +114 -0
  112. sygra-1.0.0.dist-info/WHEEL +5 -0
  113. sygra-1.0.0.dist-info/licenses/LICENSE +201 -0
  114. sygra-1.0.0.dist-info/top_level.txt +1 -0
sygra/__init__.py ADDED
@@ -0,0 +1,426 @@
1
+ """
2
+ SyGra: Graph-oriented Synthetic data generation Pipeline library
3
+
4
+ A powerful Python library for building and executing complex data synthesis workflows
5
+ using graph-based architectures with LLMs, agents, and custom processing nodes.
6
+ """
7
+
8
+ import logging
9
+ from typing import Any, Callable, Union
10
+
11
+ from .configuration import ConfigLoader, load_config
12
+ from .exceptions import (
13
+ ConfigurationError,
14
+ DataError,
15
+ ExecutionError,
16
+ GraSPError,
17
+ ModelError,
18
+ NodeError,
19
+ TimeoutError,
20
+ ValidationError,
21
+ )
22
+ from .models import ModelConfigBuilder
23
+ from .workflow import Workflow, create_graph
24
+
25
+ try:
26
+ from .core.base_task_executor import BaseTaskExecutor, DefaultTaskExecutor # noqa: F401
27
+ from .core.dataset.dataset_processor import DatasetProcessor # noqa: F401
28
+ from .core.graph.graph_config import GraphConfig # noqa: F401
29
+ from .core.graph.sygra_message import SygraMessage # noqa: F401
30
+ from .core.graph.sygra_state import SygraState # noqa: F401
31
+ from .core.judge_task_executor import JudgeQualityTaskExecutor # noqa: F401
32
+ from .core.resumable_execution import ResumableExecutionManager # noqa: F401
33
+
34
+ CORE_AVAILABLE = True
35
+ except ImportError as e:
36
+ logging.warning(f"Core modules not available: {e}")
37
+ CORE_AVAILABLE = False
38
+
39
+ try:
40
+ from .core.dataset.dataset_config import DataSourceConfig # noqa: F401
41
+ from .core.dataset.dataset_config import DataSourceType # noqa: F401
42
+ from .core.dataset.dataset_config import OutputConfig # noqa: F401
43
+ from .core.dataset.dataset_config import OutputType # noqa: F401
44
+ from .core.dataset.dataset_config import ShardConfig # noqa: F401
45
+ from .core.dataset.dataset_config import TransformConfig # noqa: F401
46
+ from .core.dataset.file_handler import FileHandler # noqa: F401
47
+ from .core.dataset.huggingface_handler import HuggingFaceHandler # noqa: F401
48
+
49
+ DATA_HANDLERS_AVAILABLE = True
50
+ except ImportError:
51
+ DATA_HANDLERS_AVAILABLE = False
52
+
53
+ # Node modules
54
+ try:
55
+ from .core.graph.nodes.agent_node import AgentNode as CoreAgentNode # noqa: F401
56
+ from .core.graph.nodes.base_node import BaseNode, NodeState, NodeType # noqa: F401
57
+ from .core.graph.nodes.llm_node import LLMNode as CoreLLMNode # noqa: F401
58
+ from .core.graph.nodes.multi_llm_node import MultiLLMNode as CoreMultiLLMNode # noqa: F401
59
+ from .core.graph.nodes.weighted_sampler_node import ( # noqa: F401
60
+ WeightedSamplerNode as CoreWeightedSamplerNode,
61
+ )
62
+
63
+ NODES_AVAILABLE = True
64
+ except ImportError:
65
+ NODES_AVAILABLE = False
66
+
67
+ # Model factory modules
68
+ try:
69
+ from .core.models.model_factory import ModelFactory # noqa: F401
70
+ from .core.models.structured_output.schemas_factory import SimpleResponse # noqa: F401
71
+ from .core.models.structured_output.structured_output_config import ( # noqa: F401
72
+ StructuredOutputConfig,
73
+ )
74
+
75
+ MODELS_AVAILABLE = True
76
+ except ImportError:
77
+ MODELS_AVAILABLE = False
78
+
79
+ # Utility modules
80
+ try:
81
+ from . import utils
82
+ from .logger.logger_config import logger # noqa: F401
83
+ from .logger.logger_config import reset_to_internal_logger # noqa: F401
84
+ from .logger.logger_config import set_external_logger # noqa: F401
85
+ from .utils import constants # noqa: F401
86
+
87
+ UTILS_AVAILABLE = True
88
+ except ImportError:
89
+ UTILS_AVAILABLE = False
90
+
91
+ # Import node builders
92
+ try:
93
+ from .nodes import AgentNodeBuilder # noqa: F401
94
+ from .nodes import LambdaNodeBuilder # noqa: F401
95
+ from .nodes import LLMNodeBuilder # noqa: F401
96
+ from .nodes import MultiLLMNodeBuilder # noqa: F401
97
+ from .nodes import SubgraphNodeBuilder # noqa: F401
98
+ from .nodes import WeightedSamplerNodeBuilder # noqa: F401
99
+
100
+ NODE_BUILDERS_AVAILABLE = True
101
+ except ImportError:
102
+ NODE_BUILDERS_AVAILABLE = False
103
+
104
+ # Import data utilities
105
+ try:
106
+ from .data import DataSink # noqa: F401
107
+ from .data import DataSinkFactory # noqa: F401
108
+ from .data import DataSource # noqa: F401
109
+ from .data import DataSourceFactory # noqa: F401
110
+ from .data import from_file # noqa: F401
111
+ from .data import from_huggingface # noqa: F401
112
+ from .data import to_file # noqa: F401
113
+ from .data import to_huggingface # noqa: F401
114
+
115
+ DATA_UTILS_AVAILABLE = True
116
+ except ImportError:
117
+ DATA_UTILS_AVAILABLE = False
118
+
119
+
120
+ __version__ = "1.0.0"
121
+ __author__ = "SyGra Team"
122
+ __description__ = "Graph-oriented Synthetic data generation Pipeline library"
123
+
124
+
125
+ # Quick utility functions
126
+ def quick_llm(model: str, prompt: str, data_source: str, output: str = "output.json"):
127
+ """Quick LLM workflow creation."""
128
+ return (
129
+ Workflow(f"quick_llm_{model.replace('/', '_')}")
130
+ .source(data_source)
131
+ .llm(model, prompt)
132
+ .sink(output)
133
+ )
134
+
135
+
136
+ def quick_agent(
137
+ model: str,
138
+ prompt: str,
139
+ tools: list[str],
140
+ data_source: str,
141
+ output: str = "output.json",
142
+ ):
143
+ """Quick agent workflow creation."""
144
+ return (
145
+ Workflow(f"quick_agent_{model.replace('/', '_')}")
146
+ .source(data_source)
147
+ .agent(model=model, tools=tools, prompt=prompt)
148
+ .sink(output)
149
+ )
150
+
151
+
152
+ def quick_multi_llm(
153
+ models: dict[str, Any], prompt: str, data_source: str, output: str = "output.json"
154
+ ):
155
+ """Quick multi-LLM workflow creation."""
156
+ return (
157
+ Workflow("quick_multi_llm")
158
+ .source(data_source)
159
+ .multi_llm(models=models, prompt=prompt)
160
+ .sink(output)
161
+ )
162
+
163
+
164
+ def execute_task(task_name: str, **kwargs):
165
+ """Execute an existing task configuration."""
166
+ workflow = Workflow(task_name)
167
+ return workflow.run(**kwargs)
168
+
169
+
170
+ def create_multimodal_workflow(name: str) -> Workflow:
171
+ """Create workflow with multimodal capabilities enabled."""
172
+ workflow = Workflow(name)
173
+ workflow._supports_multimodal = True
174
+ return workflow
175
+
176
+
177
+ def create_resumable_workflow(name: str) -> Workflow:
178
+ """Create workflow with resumable execution enabled."""
179
+ workflow = Workflow(name)
180
+ workflow.resumable(True)
181
+ return workflow
182
+
183
+
184
+ def create_quality_workflow(name: str) -> Workflow:
185
+ """Create workflow with quality tagging enabled."""
186
+ workflow = Workflow(name)
187
+ workflow.quality_tagging(True)
188
+ return workflow
189
+
190
+
191
+ def create_chat_workflow(name: str, conversation_type: str = "multiturn") -> Workflow:
192
+ """Create workflow optimized for chat/conversation generation."""
193
+ workflow = Workflow(name)
194
+ workflow.chat_conversation(conversation_type)
195
+ return workflow
196
+
197
+
198
+ def create_structured_schema(fields: dict[str, str], name: str = "CustomSchema") -> dict[str, Any]:
199
+ """Create structured output schema configuration."""
200
+ return {
201
+ "enabled": True,
202
+ "schema": {
203
+ "name": name,
204
+ "fields": {
205
+ field_name: {"type": field_type} for field_name, field_type in fields.items()
206
+ },
207
+ },
208
+ }
209
+
210
+
211
+ def pydantic_schema(model_class: str) -> dict[str, Any]:
212
+ """Create structured output schema from Pydantic model class path."""
213
+ return {"enabled": True, "schema": model_class}
214
+
215
+
216
+ def create_processor_config(processor: Union[str, Callable], **params) -> dict[str, Any]:
217
+ """Create processor configuration."""
218
+ if callable(processor):
219
+ processor_path = f"{processor.__module__}.{processor.__name__}"
220
+ else:
221
+ processor_path = processor
222
+
223
+ config: dict[str, Any] = {"processor": processor_path}
224
+ if params:
225
+ config["params"] = params
226
+
227
+ return config
228
+
229
+
230
+ def create_transformation_config(transform: Union[str, Callable], **params) -> dict[str, Any]:
231
+ """Create data transformation configuration."""
232
+ if callable(transform):
233
+ transform_path = f"{transform.__module__}.{transform.__name__}"
234
+ else:
235
+ transform_path = transform
236
+
237
+ config: dict[str, Any] = {"transform": transform_path}
238
+ if params:
239
+ config["params"] = params
240
+
241
+ return config
242
+
243
+
244
+ def get_version() -> str:
245
+ """Get library version."""
246
+ return __version__
247
+
248
+
249
+ def setup_logging(level: str = "INFO") -> None:
250
+ """Setup logging."""
251
+ logging.getLogger("sygra").setLevel(getattr(logging, level.upper()))
252
+
253
+
254
+ def validate_environment() -> dict[str, bool]:
255
+ """Validate environment setup."""
256
+ return {
257
+ "core_available": CORE_AVAILABLE,
258
+ "data_handlers_available": DATA_HANDLERS_AVAILABLE,
259
+ "nodes_available": NODES_AVAILABLE,
260
+ "models_available": MODELS_AVAILABLE,
261
+ "utils_available": UTILS_AVAILABLE,
262
+ "node_builders_available": NODE_BUILDERS_AVAILABLE,
263
+ "data_utils_available": DATA_UTILS_AVAILABLE,
264
+ }
265
+
266
+
267
+ def get_info() -> dict[str, Any]:
268
+ """Get library information and feature availability."""
269
+ return {
270
+ "version": __version__,
271
+ "author": __author__,
272
+ "description": __description__,
273
+ "features": validate_environment(),
274
+ }
275
+
276
+
277
+ def list_available_models() -> list[str]:
278
+ """list available models from framework configuration."""
279
+ if not UTILS_AVAILABLE:
280
+ return ["Framework not available - cannot list models"]
281
+
282
+ try:
283
+ model_configs = utils.utils.load_model_config()
284
+ return list(model_configs.keys())
285
+ except Exception as e:
286
+ return [f"Error loading models: {e}"]
287
+
288
+
289
+ def get_model_info(model_name: str) -> dict[str, Any]:
290
+ """Get information about a specific model."""
291
+ if not UTILS_AVAILABLE:
292
+ return {"error": "Framework not available"}
293
+
294
+ try:
295
+ # Ensure the mapping type is explicit so mypy can infer correct return type
296
+ model_configs: dict[str, dict[str, Any]] = utils.utils.load_model_config()
297
+ return model_configs.get(model_name, {"error": f"Model {model_name} not found"})
298
+ except Exception as e:
299
+ return {"error": f"Error loading model info: {e}"}
300
+
301
+
302
+ # Main exports
303
+ __all__ = [
304
+ # Main classes
305
+ "Workflow",
306
+ "create_graph",
307
+ # Configuration
308
+ "load_config",
309
+ "ConfigLoader",
310
+ # Model utilities
311
+ "ModelConfigBuilder",
312
+ # Quick functions
313
+ "quick_llm",
314
+ "quick_agent",
315
+ "quick_multi_llm",
316
+ "execute_task",
317
+ # Advanced workflow builders
318
+ "create_multimodal_workflow",
319
+ "create_resumable_workflow",
320
+ "create_quality_workflow",
321
+ "create_chat_workflow",
322
+ # Structured output helpers
323
+ "create_structured_schema",
324
+ "pydantic_schema",
325
+ # Processing helpers
326
+ "create_processor_config",
327
+ "create_transformation_config",
328
+ # Utilities
329
+ "get_version",
330
+ "setup_logging",
331
+ "validate_environment",
332
+ "get_info",
333
+ "list_available_models",
334
+ "get_model_info",
335
+ # Exceptions
336
+ "GraSPError",
337
+ "ValidationError",
338
+ "ExecutionError",
339
+ "ConfigurationError",
340
+ "NodeError",
341
+ "DataError",
342
+ "ModelError",
343
+ "TimeoutError",
344
+ ]
345
+
346
+ # Add conditionally available imports to __all__
347
+ if CORE_AVAILABLE:
348
+ __all__.extend(
349
+ [
350
+ "BaseTaskExecutor",
351
+ "DefaultTaskExecutor",
352
+ "JudgeQualityTaskExecutor",
353
+ "GraphConfig",
354
+ "SygraState",
355
+ "SygraMessage",
356
+ "ResumableExecutionManager",
357
+ "DatasetProcessor",
358
+ ]
359
+ )
360
+
361
+ if DATA_HANDLERS_AVAILABLE:
362
+ __all__.extend(
363
+ [
364
+ "DataSourceConfig",
365
+ "OutputConfig",
366
+ "DataSourceType",
367
+ "OutputType",
368
+ "TransformConfig",
369
+ "ShardConfig",
370
+ "FileHandler",
371
+ "HuggingFaceHandler",
372
+ ]
373
+ )
374
+
375
+ if NODES_AVAILABLE:
376
+ __all__.extend(
377
+ [
378
+ "BaseNode",
379
+ "NodeType",
380
+ "NodeState",
381
+ "CoreLLMNode",
382
+ "CoreAgentNode",
383
+ "CoreMultiLLMNode",
384
+ "CoreWeightedSamplerNode",
385
+ ]
386
+ )
387
+
388
+ if MODELS_AVAILABLE:
389
+ __all__.extend(["ModelFactory", "StructuredOutputConfig", "SimpleResponse"])
390
+
391
+ if UTILS_AVAILABLE:
392
+ __all__.extend(
393
+ [
394
+ "utils",
395
+ "constants",
396
+ "logger",
397
+ "set_external_logger",
398
+ "reset_to_internal_logger",
399
+ ]
400
+ )
401
+
402
+ if NODE_BUILDERS_AVAILABLE:
403
+ __all__.extend(
404
+ [
405
+ "LLMNodeBuilder",
406
+ "AgentNodeBuilder",
407
+ "MultiLLMNodeBuilder",
408
+ "LambdaNodeBuilder",
409
+ "WeightedSamplerNodeBuilder",
410
+ "SubgraphNodeBuilder",
411
+ ]
412
+ )
413
+
414
+ if DATA_UTILS_AVAILABLE:
415
+ __all__.extend(
416
+ [
417
+ "DataSource",
418
+ "DataSink",
419
+ "DataSourceFactory",
420
+ "DataSinkFactory",
421
+ "from_file",
422
+ "from_huggingface",
423
+ "to_file",
424
+ "to_huggingface",
425
+ ]
426
+ )
@@ -0,0 +1,32 @@
1
+ default_transformations:
2
+ - transform: sygra.processors.data_transform.CreateImageUrlTransform
3
+ - transform: sygra.processors.data_transform.CreateAudioUrlTransform
4
+
5
+ model_config:
6
+ ssl_verify: true
7
+ ssl_cert: None
8
+
9
+ post_generation_tasks:
10
+ oasst_mapper:
11
+ required: "yes"
12
+ type: "sft"
13
+ intermediate_writing: "yes"
14
+
15
+ data_quality:
16
+ tasks:
17
+ - name: "conversation_pretokenization"
18
+ params:
19
+ hf_chat_template_model_id: "Qwen/Qwen2.5-32B-Instruct"
20
+ - name: "metadata_tagging"
21
+ - name: "llm_based_quality"
22
+ - name: "data_characteristics"
23
+ params:
24
+ model: "Qwen/Qwen2.5-32B-Instruct"
25
+ - name: "language_tagging"
26
+ - name: "lexical_diversity"
27
+ - name: "ppl_score"
28
+ params:
29
+ model_config:
30
+ model_type: "vllm" # or "tgi"
31
+ url: # Add the URL of the model server
32
+ model_serving_name: # Add the serving name of the model
@@ -0,0 +1,80 @@
1
+ mixtral8x7b:
2
+ model_type: tgi
3
+ # URL and auth_token should be defined at .env file as SYGRA_MIXTRAL8X7B_URL and SYGRA_MIXTRAL8X7B_TOKEN
4
+ hf_chat_template_model_id: meta-math/MetaMath-Mistral-7B
5
+ parameters:
6
+ max_new_tokens: 500
7
+ temperature: 1.0
8
+
9
+ mixtral_instruct_8x22b:
10
+ model_type: vllm
11
+ # URL and auth_token should be defined at .env file as SYGRA_MIXTRAL_INSTRUCT_8X22B_URL and SYGRA_MIXTRAL_INSTRUCT_8X22B_TOKEN
12
+ hf_chat_template_model_id: meta-math/MetaMath-Mistral-7B
13
+ parameters:
14
+ max_tokens: 1024
15
+ temperature: 1.0
16
+ stop: ["</s>", "[INST]"]
17
+
18
+ mistralai:
19
+ model_type: mistralai
20
+ model: azureai
21
+ # URL and auth_token should be defined at .env file as SYGRA_MISTRALAI_URL and SYGRA_MISTRALAI_TOKEN
22
+ parameters:
23
+ max_tokens: 500
24
+ temperature: 1.0
25
+
26
+ gpt4:
27
+ model_type: azure_openai
28
+ model: gpt-4-32k
29
+ # URL and api_key should be defined at .env file as SYGRA_GPT4_URL and SYGRA_GPT4_TOKEN
30
+ api_version: 2024-05-01-preview
31
+ parameters:
32
+ max_tokens: 500
33
+ temperature: 1.0
34
+
35
+ gpt-4o:
36
+ model_type: azure_openai
37
+ model: gpt-4o
38
+ # URL and api_key should be defined at .env file as SYGRA_GPT-4O_URL and SYGRA_GPT-4O_TOKEN
39
+ api_version: 2024-02-15-preview
40
+ parameters:
41
+ max_tokens: 500
42
+ temperature: 1.0
43
+
44
+ gpt-4o-mini:
45
+ model_type: azure_openai
46
+ model: gpt-4o-mini
47
+ # URL and api_key should be defined at .env file as SYGRA_GPT-4O-MINI_URL and SYGRA_GPT-4O-MINI_TOKEN
48
+ api_version: 2024-08-01-preview
49
+ parameters:
50
+ max_tokens: 5000
51
+ temperature: 0.0001
52
+
53
+ #QWEN VL 72b deployed in vllm
54
+ qwen_vl_72b:
55
+ # URL and api_key should be defined at .env file as SYGRA_QWEN_VL_72B_URL and SYGRA_QWEN_VL_72B_TOKEN
56
+ hf_chat_template_model_id: Qwen/Qwen2.5-VL-72B-Instruct
57
+ model_type: vllm
58
+ parameters:
59
+ max_tokens: 2048
60
+ temperature: 1.0
61
+
62
+ #QWEN 32B deployed in vllm
63
+ qwen3_32b:
64
+ # URL and api_key should be defined at .env file as SYGRA_QWEN3_32B_URL and SYGRA_QWEN3_32B_TOKEN
65
+ model_serving_name: qwen3_32b
66
+ hf_chat_template_model_id: Qwen/Qwen3-32B
67
+ model_type: vllm
68
+ parameters:
69
+ max_tokens: 2048
70
+ temperature: 0
71
+
72
+ #qwen3_1.7b
73
+ qwen3_1.7b:
74
+ model_type: ollama
75
+ # URL should be defined at .env file as SYGRA_QWEN3_1.7B_URL. auth_token is not needed
76
+ # Default URL for ollama is http://localhost:11434
77
+ hf_chat_template_model_id: Qwen/Qwen3-1.7B
78
+ post_process: sygra.core.models.model_postprocessor.RemoveThinkData
79
+ parameters:
80
+ temperature: 0.8
@@ -0,0 +1,41 @@
1
+ {
2
+ "triton": {
3
+ "default": {
4
+ "payload_json": {
5
+ "id": "42",
6
+ "inputs": [
7
+ {
8
+ "data": "<replace this : Array of json>",
9
+ "datatype": "BYTES",
10
+ "name": "request",
11
+ "shape": [
12
+ 1,
13
+ 1
14
+ ]
15
+ },
16
+ {
17
+ "data": "<replace this: Array of options json>",
18
+ "datatype": "BYTES",
19
+ "name": "options",
20
+ "shape": [
21
+ 1,
22
+ 1
23
+ ]
24
+ }
25
+ ],
26
+ "outputs": [
27
+ {
28
+ "name": "response"
29
+ },
30
+ {
31
+ "name": "response_metadata"
32
+ },
33
+ {
34
+ "name": "error"
35
+ }
36
+ ]
37
+ },
38
+ "response_key": "model_output"
39
+ }
40
+ }
41
+ }
@@ -0,0 +1,10 @@
1
+ from sygra.configuration.loader import ConfigLoader
2
+
3
+
4
+ def load_config(config_path):
5
+ """Load workflow from existing YAML configuration."""
6
+ loader = ConfigLoader()
7
+ return loader.load_and_create(config_path)
8
+
9
+
10
+ __all__ = ["ConfigLoader", "load_config"]
@@ -0,0 +1,65 @@
1
+ import os
2
+ from pathlib import Path
3
+ from typing import Any, Union, cast
4
+
5
+ import yaml
6
+
7
+ try:
8
+ from sygra.core.dataset.dataset_config import DataSourceConfig, OutputConfig # noqa: F401
9
+ from sygra.core.graph.graph_config import GraphConfig # noqa: F401
10
+ from sygra.utils import utils
11
+
12
+ UTILS_AVAILABLE = True
13
+ except ImportError:
14
+ UTILS_AVAILABLE = False
15
+
16
+
17
+ class ConfigLoader:
18
+ """Loads existing SyGra YAML configurations and converts them to library objects."""
19
+
20
+ def load(self, config_path: Union[str, Path, dict[str, Any]]) -> dict[str, Any]:
21
+ """Load configuration from file or dictionary."""
22
+ if isinstance(config_path, dict):
23
+ return config_path
24
+
25
+ config_path = Path(config_path)
26
+ if not config_path.exists():
27
+ if UTILS_AVAILABLE:
28
+ task_config_path = utils.get_file_in_task_dir(config_path.stem, "graph_config.yaml")
29
+ if os.path.exists(task_config_path):
30
+ config_path = Path(task_config_path)
31
+ else:
32
+ raise FileNotFoundError(f"Configuration file not found: {config_path}")
33
+ else:
34
+ raise FileNotFoundError(f"Configuration file not found: {config_path}")
35
+
36
+ with open(config_path, "r") as f:
37
+ config = cast(dict[str, Any], yaml.safe_load(f))
38
+
39
+ return config
40
+
41
+ def load_and_create(self, config_path: Union[str, Path, dict[str, Any]]):
42
+ """Load config and create appropriate Workflow or Graph object."""
43
+ config = self.load(config_path)
44
+
45
+ # Import here to avoid circular imports
46
+ from ..workflow import Workflow
47
+
48
+ workflow = Workflow()
49
+ workflow._config = config
50
+
51
+ workflow._supports_subgraphs = True
52
+ workflow._supports_multimodal = True
53
+ workflow._supports_resumable = True
54
+ workflow._supports_quality = True
55
+ workflow._supports_oasst = True
56
+
57
+ if isinstance(config_path, (str, Path)):
58
+ workflow.name = Path(config_path).parent.name
59
+ else:
60
+ workflow.name = config.get("task_name", "loaded_workflow")
61
+
62
+ return workflow
63
+
64
+
65
+ __all__ = ["ConfigLoader"]