DeepFabric 4.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. deepfabric/__init__.py +70 -0
  2. deepfabric/__main__.py +6 -0
  3. deepfabric/auth.py +382 -0
  4. deepfabric/builders.py +303 -0
  5. deepfabric/builders_agent.py +1304 -0
  6. deepfabric/cli.py +1288 -0
  7. deepfabric/config.py +899 -0
  8. deepfabric/config_manager.py +251 -0
  9. deepfabric/constants.py +94 -0
  10. deepfabric/dataset_manager.py +534 -0
  11. deepfabric/error_codes.py +581 -0
  12. deepfabric/evaluation/__init__.py +47 -0
  13. deepfabric/evaluation/backends/__init__.py +32 -0
  14. deepfabric/evaluation/backends/ollama_backend.py +137 -0
  15. deepfabric/evaluation/backends/tool_call_parsers.py +409 -0
  16. deepfabric/evaluation/backends/transformers_backend.py +326 -0
  17. deepfabric/evaluation/evaluator.py +845 -0
  18. deepfabric/evaluation/evaluators/__init__.py +13 -0
  19. deepfabric/evaluation/evaluators/base.py +104 -0
  20. deepfabric/evaluation/evaluators/builtin/__init__.py +5 -0
  21. deepfabric/evaluation/evaluators/builtin/tool_calling.py +93 -0
  22. deepfabric/evaluation/evaluators/registry.py +66 -0
  23. deepfabric/evaluation/inference.py +155 -0
  24. deepfabric/evaluation/metrics.py +397 -0
  25. deepfabric/evaluation/parser.py +304 -0
  26. deepfabric/evaluation/reporters/__init__.py +13 -0
  27. deepfabric/evaluation/reporters/base.py +56 -0
  28. deepfabric/evaluation/reporters/cloud_reporter.py +195 -0
  29. deepfabric/evaluation/reporters/file_reporter.py +61 -0
  30. deepfabric/evaluation/reporters/multi_reporter.py +56 -0
  31. deepfabric/exceptions.py +67 -0
  32. deepfabric/factory.py +26 -0
  33. deepfabric/generator.py +1084 -0
  34. deepfabric/graph.py +545 -0
  35. deepfabric/hf_hub.py +214 -0
  36. deepfabric/kaggle_hub.py +219 -0
  37. deepfabric/llm/__init__.py +41 -0
  38. deepfabric/llm/api_key_verifier.py +534 -0
  39. deepfabric/llm/client.py +1206 -0
  40. deepfabric/llm/errors.py +105 -0
  41. deepfabric/llm/rate_limit_config.py +262 -0
  42. deepfabric/llm/rate_limit_detector.py +278 -0
  43. deepfabric/llm/retry_handler.py +270 -0
  44. deepfabric/metrics.py +212 -0
  45. deepfabric/progress.py +262 -0
  46. deepfabric/prompts.py +290 -0
  47. deepfabric/schemas.py +1000 -0
  48. deepfabric/spin/__init__.py +6 -0
  49. deepfabric/spin/client.py +263 -0
  50. deepfabric/spin/models.py +26 -0
  51. deepfabric/stream_simulator.py +90 -0
  52. deepfabric/tools/__init__.py +5 -0
  53. deepfabric/tools/defaults.py +85 -0
  54. deepfabric/tools/loader.py +87 -0
  55. deepfabric/tools/mcp_client.py +677 -0
  56. deepfabric/topic_manager.py +303 -0
  57. deepfabric/topic_model.py +20 -0
  58. deepfabric/training/__init__.py +35 -0
  59. deepfabric/training/api_key_prompt.py +302 -0
  60. deepfabric/training/callback.py +363 -0
  61. deepfabric/training/metrics_sender.py +301 -0
  62. deepfabric/tree.py +438 -0
  63. deepfabric/tui.py +1267 -0
  64. deepfabric/update_checker.py +166 -0
  65. deepfabric/utils.py +150 -0
  66. deepfabric/validation.py +143 -0
  67. deepfabric-4.4.0.dist-info/METADATA +702 -0
  68. deepfabric-4.4.0.dist-info/RECORD +71 -0
  69. deepfabric-4.4.0.dist-info/WHEEL +4 -0
  70. deepfabric-4.4.0.dist-info/entry_points.txt +2 -0
  71. deepfabric-4.4.0.dist-info/licenses/LICENSE +201 -0
deepfabric/__init__.py ADDED
@@ -0,0 +1,70 @@
1
+ from .auth import (
2
+ clear_tokens,
3
+ device_flow_login,
4
+ get_auth_token,
5
+ get_config,
6
+ get_stored_token,
7
+ is_authenticated,
8
+ prompt_cloud_signup,
9
+ save_config,
10
+ store_tokens,
11
+ )
12
+ from .cli import cli
13
+ from .config import DeepFabricConfig
14
+ from .exceptions import (
15
+ APIError,
16
+ ConfigurationError,
17
+ DatasetError,
18
+ DataSetGeneratorError,
19
+ DeepFabricError,
20
+ HubUploadError,
21
+ JSONParsingError,
22
+ ModelError,
23
+ RetryExhaustedError,
24
+ TreeError,
25
+ ValidationError,
26
+ )
27
+ from .generator import DataSetGenerator, DataSetGeneratorConfig
28
+ from .graph import Graph, GraphConfig
29
+ from .hf_hub import HFUploader
30
+ from .training import DeepFabricCallback, MetricsSender
31
+ from .tree import Tree, TreeConfig
32
+
33
+ __version__ = "0.1.0"
34
+
35
+ __all__ = [
36
+ "Tree",
37
+ "TreeConfig",
38
+ "Graph",
39
+ "GraphConfig",
40
+ "DataSetGenerator",
41
+ "DataSetGeneratorConfig",
42
+ "DeepFabricConfig",
43
+ "HFUploader",
44
+ "cli",
45
+ # Training metrics logging
46
+ "DeepFabricCallback",
47
+ "MetricsSender",
48
+ # Authentication
49
+ "get_config",
50
+ "save_config",
51
+ "get_stored_token",
52
+ "store_tokens",
53
+ "clear_tokens",
54
+ "is_authenticated",
55
+ "get_auth_token",
56
+ "prompt_cloud_signup",
57
+ "device_flow_login",
58
+ # Exceptions
59
+ "DeepFabricError",
60
+ "ConfigurationError",
61
+ "ValidationError",
62
+ "ModelError",
63
+ "TreeError",
64
+ "DataSetGeneratorError",
65
+ "DatasetError",
66
+ "HubUploadError",
67
+ "JSONParsingError",
68
+ "APIError",
69
+ "RetryExhaustedError",
70
+ ]
deepfabric/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Main entry point for deepfabric CLI."""
2
+
3
+ from deepfabric.cli import cli
4
+
5
+ if __name__ == "__main__":
6
+ cli()
deepfabric/auth.py ADDED
@@ -0,0 +1,382 @@
1
+ import contextlib
2
+ import json
3
+ import os
4
+ import time
5
+ import webbrowser
6
+
7
+ from pathlib import Path
8
+
9
+ import click
10
+ import httpx
11
+
12
+ from .tui import get_tui
13
+
14
+ DEFAULT_API_URL = os.getenv("DEEPFABRIC_API_URL", "https://api.deepfabric.dev")
15
+ CONFIG_DIR = Path.home() / ".deepfabric"
16
+ CONFIG_FILE = CONFIG_DIR / "config.json"
17
+
18
+
19
+ def get_config() -> dict:
20
+ """Load authentication config from disk."""
21
+ if not CONFIG_FILE.exists():
22
+ return {}
23
+ try:
24
+ with open(CONFIG_FILE) as f:
25
+ return json.load(f)
26
+ except (json.JSONDecodeError, OSError):
27
+ return {}
28
+
29
+
30
+ def save_config(config: dict) -> None:
31
+ """Save authentication config to disk."""
32
+ CONFIG_DIR.mkdir(parents=True, exist_ok=True)
33
+ with open(CONFIG_FILE, "w") as f:
34
+ json.dump(config, f, indent=2)
35
+ # Make config file readable only by owner
36
+ CONFIG_FILE.chmod(0o600)
37
+
38
+
39
+ def get_stored_token() -> str | None:
40
+ """Get stored access token."""
41
+ config = get_config()
42
+ return config.get("access_token")
43
+
44
+
45
+ def store_tokens(access_token: str, refresh_token: str) -> None:
46
+ """Store access and refresh tokens."""
47
+ config = get_config()
48
+ config["access_token"] = access_token
49
+ config["refresh_token"] = refresh_token
50
+ save_config(config)
51
+
52
+
53
+ def clear_tokens() -> None:
54
+ """Clear stored tokens."""
55
+ if CONFIG_FILE.exists():
56
+ CONFIG_FILE.unlink()
57
+
58
+
59
+ def is_authenticated() -> bool:
60
+ """Check if user is authenticated."""
61
+ config = get_config()
62
+ return bool(config.get("access_token") or config.get("api_key"))
63
+
64
+
65
+ def get_auth_token() -> str | None:
66
+ """Get authentication token (API key or access token)."""
67
+ config = get_config()
68
+ return config.get("api_key") or config.get("access_token")
69
+
70
+
71
+ def prompt_cloud_signup(api_url: str = DEFAULT_API_URL) -> bool:
72
+ """
73
+ Prompt user to login/signup for cloud features.
74
+
75
+ Returns:
76
+ True if user successfully authenticated, False otherwise
77
+ """
78
+ tui = get_tui()
79
+
80
+ tui.console.print("")
81
+ tui.info("DeepFabric Cloud can save and track your evaluations")
82
+ tui.info(" - Compare models across runs")
83
+ tui.info(" - Track performance over time")
84
+ tui.info(" - Share results with your team")
85
+ tui.console.print("")
86
+
87
+ response = click.prompt(
88
+ "Would you like to record this evaluation in DeepFabric Cloud?",
89
+ type=click.Choice(["y", "n"], case_sensitive=False),
90
+ default="y",
91
+ show_choices=True,
92
+ )
93
+
94
+ if response.lower() != "y":
95
+ tui.info("Skipping cloud reporting. Results will be saved locally only.")
96
+ return False
97
+
98
+ tui.console.print("")
99
+ tui.info("Great! Let's get you authenticated.")
100
+ tui.console.print("")
101
+
102
+ auth_choice = click.prompt(
103
+ "Choose authentication method",
104
+ type=click.Choice(["login", "register", "skip"], case_sensitive=False),
105
+ default="login",
106
+ show_choices=True,
107
+ )
108
+
109
+ if auth_choice == "skip":
110
+ tui.info("Skipping authentication. Results will be saved locally only.")
111
+ return False
112
+
113
+ if auth_choice == "register":
114
+ tui.info("Opening DeepFabric Cloud registration page...")
115
+ register_url = api_url.replace("/api", "").rstrip("/") + "/signup"
116
+ with contextlib.suppress(Exception):
117
+ webbrowser.open(register_url)
118
+ tui.info("After registering, come back here to log in.")
119
+ tui.console.print("")
120
+
121
+ if not click.confirm("Ready to log in?", default=True):
122
+ tui.info("Skipping authentication. Results will be saved locally only.")
123
+ return False
124
+
125
+ # Proceed with login
126
+ success = device_flow_login(api_url)
127
+ if success:
128
+ tui.success("Authentication successful! Your evaluation will be saved to the cloud.")
129
+ return True
130
+ tui.warning("Authentication failed. Results will be saved locally only.")
131
+ return False
132
+
133
+
134
+ def device_flow_login(api_url: str = DEFAULT_API_URL, debug: bool = False) -> bool: # noqa: PLR0911
135
+ """
136
+ Perform device flow OAuth login.
137
+
138
+ Args:
139
+ api_url: The DeepFabric API URL
140
+ debug: If True, print debug information during polling
141
+
142
+ Returns:
143
+ True if login successful, False otherwise
144
+ """
145
+ tui = get_tui()
146
+
147
+ try:
148
+ # Request device code
149
+ tui.info("Initiating device authorization...")
150
+ with httpx.Client() as client:
151
+ response = client.post(
152
+ f"{api_url}/api/v1/oauth/device/code",
153
+ json={"client_id": "deepfabric-cli"},
154
+ timeout=10.0,
155
+ )
156
+ response.raise_for_status()
157
+ device_data = response.json()
158
+
159
+ device_code = device_data["device_code"]
160
+ user_code = device_data["user_code"]
161
+ verification_uri = device_data["verification_uri_complete"]
162
+ expires_in = device_data["expires_in"]
163
+ interval = device_data.get("interval", 5)
164
+
165
+ tui.console.print("")
166
+ tui.success(f"Your user code is: {user_code}")
167
+ tui.info(f"Opening browser to: {verification_uri}")
168
+ tui.info("If the browser doesn't open, please visit the URL above manually.")
169
+ tui.console.print("")
170
+
171
+ if debug:
172
+ tui.console.print(f"[dim]Debug: device_code={device_code[:20]}...[/dim]")
173
+ tui.console.print(f"[dim]Debug: expires_in={expires_in}s, interval={interval}s[/dim]")
174
+
175
+ # Open browser
176
+ with contextlib.suppress(Exception):
177
+ webbrowser.open(verification_uri)
178
+
179
+ # Poll for authorization
180
+ start_time = time.time()
181
+ tui.info("Waiting for authorization...")
182
+ poll_count = 0
183
+
184
+ with httpx.Client() as client:
185
+ while time.time() - start_time < expires_in:
186
+ time.sleep(interval)
187
+ poll_count += 1
188
+
189
+ try:
190
+ response = client.post(
191
+ f"{api_url}/api/v1/oauth/device/token",
192
+ json={
193
+ "device_code": device_code,
194
+ "client_id": "deepfabric-cli",
195
+ },
196
+ timeout=10.0,
197
+ )
198
+ token_data = response.json()
199
+
200
+ if debug:
201
+ # Mask sensitive data in debug output
202
+ debug_data = {
203
+ k: (
204
+ v[:20] + "..."
205
+ if k in ("access_token", "refresh_token") and v
206
+ else v
207
+ )
208
+ for k, v in token_data.items()
209
+ }
210
+ tui.console.print(f"[dim]Debug poll #{poll_count}: {debug_data}[/dim]")
211
+
212
+ # Check for error responses (OAuth 2.0 device flow standard)
213
+ error = token_data.get("error")
214
+ if error == "authorization_pending":
215
+ continue
216
+ if error == "slow_down":
217
+ interval += 5
218
+ if debug:
219
+ tui.console.print(
220
+ f"[dim]Debug: slow_down, new interval={interval}s[/dim]"
221
+ )
222
+ continue
223
+ if error == "expired_token":
224
+ tui.error("Device code expired. Please try again.")
225
+ return False
226
+ if error == "access_denied":
227
+ tui.error("Access denied by user.")
228
+ return False
229
+ if error:
230
+ tui.error(f"Authorization failed: {error}")
231
+ return False
232
+
233
+ # Success - check for tokens
234
+ access_token = token_data.get("access_token")
235
+ refresh_token = token_data.get("refresh_token")
236
+
237
+ if access_token and refresh_token:
238
+ store_tokens(access_token, refresh_token)
239
+ tui.success("Login successful!")
240
+ return True
241
+ if access_token:
242
+ # Some flows may only return access token
243
+ store_tokens(access_token, "")
244
+ tui.success("Login successful!")
245
+ return True
246
+
247
+ # No error and no tokens - unexpected response
248
+ if debug:
249
+ tui.console.print("[dim]Debug: No error and no tokens in response[/dim]")
250
+
251
+ except httpx.HTTPStatusError as e:
252
+ # Non-2xx response - check if it contains OAuth error
253
+ if debug:
254
+ tui.console.print(f"[dim]Debug: HTTP {e.response.status_code}[/dim]")
255
+ try:
256
+ error_data = e.response.json()
257
+ error = error_data.get("error")
258
+ if error == "authorization_pending":
259
+ continue
260
+ if error == "slow_down":
261
+ interval += 5
262
+ continue
263
+ except Exception:
264
+ tui.error("Unexpected response during polling.")
265
+ pass
266
+ tui.error(f"Request failed: {e}")
267
+ return False
268
+ except httpx.HTTPError as e:
269
+ tui.error(f"Network error: {e}")
270
+ return False
271
+
272
+ tui.error("Authorization timed out. Please try again.")
273
+ return False
274
+
275
+ except Exception as e:
276
+ tui.error(f"Login failed: {e}")
277
+ return False
278
+
279
+
280
+ @click.group()
281
+ def auth():
282
+ """Authentication commands for DeepFabric Cloud."""
283
+ pass
284
+
285
+
286
+ @auth.command()
287
+ @click.option(
288
+ "--api-key",
289
+ help="Authenticate using an API key instead of device flow",
290
+ )
291
+ @click.option(
292
+ "--api-url",
293
+ default=DEFAULT_API_URL,
294
+ help=f"DeepFabric API URL (default: {DEFAULT_API_URL})",
295
+ )
296
+ @click.pass_context
297
+ def login(ctx: click.Context, api_key: str | None, api_url: str):
298
+ """
299
+ Log in to DeepFabric Cloud.
300
+
301
+ By default, uses OAuth device flow (opens browser).
302
+ Use --api-key to authenticate with an API key instead.
303
+ """
304
+ tui = get_tui()
305
+ debug = ctx.obj.get("debug", False) if ctx.obj else False
306
+
307
+ if api_key:
308
+ # API key authentication
309
+ config = get_config()
310
+ config["api_key"] = api_key
311
+ config["api_url"] = api_url
312
+ save_config(config)
313
+ tui.success("API key saved successfully!")
314
+ tui.info("You can now use DeepFabric Cloud features.")
315
+ else:
316
+ # Device flow authentication
317
+ success = device_flow_login(api_url, debug=debug)
318
+ if success:
319
+ tui.info("You can now use DeepFabric Cloud features.")
320
+ else:
321
+ raise click.ClickException("Login failed")
322
+
323
+
324
+ @auth.command()
325
+ def logout():
326
+ """Log out from DeepFabric Cloud."""
327
+ tui = get_tui()
328
+
329
+ clear_tokens()
330
+ tui.success("Logged out successfully!")
331
+
332
+
333
+ @auth.command()
334
+ @click.option(
335
+ "--api-url",
336
+ default=DEFAULT_API_URL,
337
+ help=f"DeepFabric API URL (default: {DEFAULT_API_URL})",
338
+ )
339
+ def status(api_url: str):
340
+ """Check authentication status."""
341
+ tui = get_tui()
342
+
343
+ config = get_config()
344
+ api_key = config.get("api_key")
345
+ access_token = get_stored_token()
346
+
347
+ if not api_key and not access_token:
348
+ tui.warning("Not logged in")
349
+ tui.info("Run 'deepfabric auth login' to authenticate")
350
+ return
351
+
352
+ # Try to verify authentication with API
353
+ try:
354
+ headers = {}
355
+ if api_key:
356
+ headers["Authorization"] = f"Bearer {api_key}"
357
+ elif access_token:
358
+ headers["Authorization"] = f"Bearer {access_token}"
359
+
360
+ with httpx.Client() as client:
361
+ response = client.get(
362
+ f"{api_url}/api/v1/auth/me",
363
+ headers=headers,
364
+ timeout=10.0,
365
+ )
366
+ response.raise_for_status()
367
+ user_data = response.json()
368
+
369
+ tui.success("Authenticated")
370
+ tui.info(f"Email: {user_data['email']}")
371
+ tui.info(f"Name: {user_data.get('name', 'N/A')}")
372
+ tui.info(f"User ID: {user_data['id']}")
373
+
374
+ if api_key:
375
+ tui.info("Auth method: API Key")
376
+ else:
377
+ tui.info("Auth method: OAuth")
378
+
379
+ except Exception as e:
380
+ tui.error(f"Authentication verification failed: {e}")
381
+ tui.warning("Your stored credentials may be invalid or expired")
382
+ tui.info("Run 'deepfabric auth login' to re-authenticate")