mcli-framework 7.12.0__py3-none-any.whl → 7.12.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mcli-framework might be problematic. Click here for more details.
- mcli/app/__init__.py +0 -2
- mcli/app/commands_cmd.py +19 -23
- mcli/app/completion_helpers.py +5 -5
- mcli/app/init_cmd.py +10 -10
- mcli/app/lock_cmd.py +82 -27
- mcli/app/main.py +4 -50
- mcli/app/model/model.py +5 -10
- mcli/app/store_cmd.py +8 -8
- mcli/app/video/__init__.py +0 -2
- mcli/app/video/video.py +1 -14
- mcli/chat/chat.py +90 -108
- mcli/chat/command_rag.py +0 -4
- mcli/chat/enhanced_chat.py +32 -41
- mcli/chat/system_controller.py +37 -37
- mcli/chat/system_integration.py +4 -5
- mcli/cli.py +2 -3
- mcli/lib/api/api.py +4 -9
- mcli/lib/api/daemon_client.py +19 -20
- mcli/lib/api/daemon_client_local.py +1 -3
- mcli/lib/api/daemon_decorator.py +6 -6
- mcli/lib/api/mcli_decorators.py +4 -8
- mcli/lib/auth/__init__.py +0 -1
- mcli/lib/auth/auth.py +4 -5
- mcli/lib/auth/mcli_manager.py +7 -12
- mcli/lib/auth/token_util.py +5 -5
- mcli/lib/config/__init__.py +29 -1
- mcli/lib/config/config.py +0 -1
- mcli/lib/custom_commands.py +1 -1
- mcli/lib/discovery/command_discovery.py +15 -15
- mcli/lib/erd/erd.py +7 -7
- mcli/lib/files/files.py +1 -1
- mcli/lib/fs/__init__.py +31 -1
- mcli/lib/fs/fs.py +12 -13
- mcli/lib/lib.py +0 -1
- mcli/lib/logger/logger.py +7 -10
- mcli/lib/performance/optimizer.py +25 -27
- mcli/lib/performance/rust_bridge.py +22 -27
- mcli/lib/performance/uvloop_config.py +0 -1
- mcli/lib/pickles/__init__.py +0 -1
- mcli/lib/pickles/pickles.py +0 -2
- mcli/lib/secrets/commands.py +0 -2
- mcli/lib/secrets/manager.py +0 -1
- mcli/lib/secrets/repl.py +2 -3
- mcli/lib/secrets/store.py +1 -2
- mcli/lib/services/data_pipeline.py +34 -34
- mcli/lib/services/lsh_client.py +38 -40
- mcli/lib/shell/shell.py +2 -2
- mcli/lib/toml/__init__.py +0 -1
- mcli/lib/ui/styling.py +0 -1
- mcli/lib/ui/visual_effects.py +33 -41
- mcli/lib/watcher/watcher.py +0 -1
- mcli/ml/__init__.py +1 -1
- mcli/ml/api/__init__.py +1 -1
- mcli/ml/api/app.py +8 -9
- mcli/ml/api/middleware.py +10 -10
- mcli/ml/api/routers/__init__.py +1 -1
- mcli/ml/api/routers/admin_router.py +3 -3
- mcli/ml/api/routers/auth_router.py +17 -18
- mcli/ml/api/routers/backtest_router.py +2 -2
- mcli/ml/api/routers/data_router.py +2 -2
- mcli/ml/api/routers/model_router.py +14 -15
- mcli/ml/api/routers/monitoring_router.py +2 -2
- mcli/ml/api/routers/portfolio_router.py +2 -2
- mcli/ml/api/routers/prediction_router.py +10 -9
- mcli/ml/api/routers/trade_router.py +2 -2
- mcli/ml/api/routers/websocket_router.py +6 -7
- mcli/ml/api/schemas.py +2 -2
- mcli/ml/auth/__init__.py +1 -1
- mcli/ml/auth/auth_manager.py +22 -23
- mcli/ml/auth/models.py +17 -17
- mcli/ml/auth/permissions.py +17 -17
- mcli/ml/backtesting/__init__.py +1 -1
- mcli/ml/backtesting/backtest_engine.py +31 -35
- mcli/ml/backtesting/performance_metrics.py +12 -14
- mcli/ml/backtesting/run.py +1 -2
- mcli/ml/cache.py +35 -36
- mcli/ml/cli/__init__.py +1 -1
- mcli/ml/cli/main.py +21 -24
- mcli/ml/config/__init__.py +1 -1
- mcli/ml/config/settings.py +28 -29
- mcli/ml/configs/__init__.py +1 -1
- mcli/ml/configs/dvc_config.py +14 -15
- mcli/ml/configs/mlflow_config.py +12 -13
- mcli/ml/configs/mlops_manager.py +19 -21
- mcli/ml/dashboard/__init__.py +4 -4
- mcli/ml/dashboard/app.py +20 -30
- mcli/ml/dashboard/app_supabase.py +16 -19
- mcli/ml/dashboard/app_training.py +11 -14
- mcli/ml/dashboard/cli.py +2 -2
- mcli/ml/dashboard/common.py +2 -3
- mcli/ml/dashboard/components/__init__.py +1 -1
- mcli/ml/dashboard/components/charts.py +13 -11
- mcli/ml/dashboard/components/metrics.py +7 -7
- mcli/ml/dashboard/components/tables.py +12 -9
- mcli/ml/dashboard/overview.py +2 -2
- mcli/ml/dashboard/pages/__init__.py +1 -1
- mcli/ml/dashboard/pages/cicd.py +15 -18
- mcli/ml/dashboard/pages/debug_dependencies.py +7 -7
- mcli/ml/dashboard/pages/monte_carlo_predictions.py +11 -18
- mcli/ml/dashboard/pages/predictions_enhanced.py +24 -32
- mcli/ml/dashboard/pages/scrapers_and_logs.py +22 -24
- mcli/ml/dashboard/pages/test_portfolio.py +3 -6
- mcli/ml/dashboard/pages/trading.py +16 -18
- mcli/ml/dashboard/pages/workflows.py +20 -30
- mcli/ml/dashboard/utils.py +9 -9
- mcli/ml/dashboard/warning_suppression.py +3 -3
- mcli/ml/data_ingestion/__init__.py +1 -1
- mcli/ml/data_ingestion/api_connectors.py +41 -46
- mcli/ml/data_ingestion/data_pipeline.py +36 -46
- mcli/ml/data_ingestion/stream_processor.py +43 -46
- mcli/ml/database/__init__.py +1 -1
- mcli/ml/database/migrations/env.py +2 -2
- mcli/ml/database/models.py +22 -24
- mcli/ml/database/session.py +14 -14
- mcli/ml/experimentation/__init__.py +1 -1
- mcli/ml/experimentation/ab_testing.py +45 -46
- mcli/ml/features/__init__.py +1 -1
- mcli/ml/features/ensemble_features.py +22 -27
- mcli/ml/features/recommendation_engine.py +30 -30
- mcli/ml/features/stock_features.py +29 -32
- mcli/ml/features/test_feature_engineering.py +10 -11
- mcli/ml/logging.py +4 -4
- mcli/ml/mlops/__init__.py +1 -1
- mcli/ml/mlops/data_versioning.py +29 -30
- mcli/ml/mlops/experiment_tracker.py +24 -24
- mcli/ml/mlops/model_serving.py +31 -34
- mcli/ml/mlops/pipeline_orchestrator.py +27 -35
- mcli/ml/models/__init__.py +5 -6
- mcli/ml/models/base_models.py +23 -23
- mcli/ml/models/ensemble_models.py +31 -31
- mcli/ml/models/recommendation_models.py +18 -19
- mcli/ml/models/test_models.py +14 -16
- mcli/ml/monitoring/__init__.py +1 -1
- mcli/ml/monitoring/drift_detection.py +32 -36
- mcli/ml/monitoring/metrics.py +2 -2
- mcli/ml/optimization/__init__.py +1 -1
- mcli/ml/optimization/optimize.py +1 -2
- mcli/ml/optimization/portfolio_optimizer.py +30 -32
- mcli/ml/predictions/__init__.py +1 -1
- mcli/ml/preprocessing/__init__.py +1 -1
- mcli/ml/preprocessing/data_cleaners.py +22 -23
- mcli/ml/preprocessing/feature_extractors.py +23 -26
- mcli/ml/preprocessing/ml_pipeline.py +23 -23
- mcli/ml/preprocessing/test_preprocessing.py +7 -8
- mcli/ml/scripts/populate_sample_data.py +0 -4
- mcli/ml/serving/serve.py +1 -2
- mcli/ml/tasks.py +17 -17
- mcli/ml/tests/test_integration.py +29 -30
- mcli/ml/tests/test_training_dashboard.py +21 -21
- mcli/ml/trading/__init__.py +1 -1
- mcli/ml/trading/migrations.py +5 -5
- mcli/ml/trading/models.py +21 -23
- mcli/ml/trading/paper_trading.py +16 -13
- mcli/ml/trading/risk_management.py +17 -18
- mcli/ml/trading/trading_service.py +25 -28
- mcli/ml/training/__init__.py +1 -1
- mcli/ml/training/train.py +0 -1
- mcli/public/oi/oi.py +1 -2
- mcli/self/completion_cmd.py +6 -10
- mcli/self/logs_cmd.py +19 -24
- mcli/self/migrate_cmd.py +22 -20
- mcli/self/redis_cmd.py +10 -11
- mcli/self/self_cmd.py +62 -18
- mcli/self/store_cmd.py +10 -12
- mcli/self/visual_cmd.py +9 -14
- mcli/self/zsh_cmd.py +2 -4
- mcli/workflow/daemon/async_command_database.py +23 -24
- mcli/workflow/daemon/async_process_manager.py +27 -29
- mcli/workflow/daemon/client.py +27 -33
- mcli/workflow/daemon/daemon.py +32 -36
- mcli/workflow/daemon/enhanced_daemon.py +24 -33
- mcli/workflow/daemon/process_cli.py +11 -12
- mcli/workflow/daemon/process_manager.py +23 -26
- mcli/workflow/daemon/test_daemon.py +4 -5
- mcli/workflow/dashboard/dashboard_cmd.py +0 -1
- mcli/workflow/doc_convert.py +15 -17
- mcli/workflow/gcloud/__init__.py +0 -1
- mcli/workflow/gcloud/gcloud.py +11 -8
- mcli/workflow/git_commit/ai_service.py +14 -15
- mcli/workflow/lsh_integration.py +9 -11
- mcli/workflow/model_service/client.py +26 -31
- mcli/workflow/model_service/download_and_run_efficient_models.py +10 -14
- mcli/workflow/model_service/lightweight_embedder.py +25 -35
- mcli/workflow/model_service/lightweight_model_server.py +26 -32
- mcli/workflow/model_service/lightweight_test.py +7 -10
- mcli/workflow/model_service/model_service.py +80 -91
- mcli/workflow/model_service/ollama_efficient_runner.py +14 -18
- mcli/workflow/model_service/openai_adapter.py +23 -23
- mcli/workflow/model_service/pdf_processor.py +21 -26
- mcli/workflow/model_service/test_efficient_runner.py +12 -16
- mcli/workflow/model_service/test_example.py +11 -13
- mcli/workflow/model_service/test_integration.py +3 -5
- mcli/workflow/model_service/test_new_features.py +7 -8
- mcli/workflow/notebook/converter.py +1 -1
- mcli/workflow/notebook/notebook_cmd.py +5 -6
- mcli/workflow/notebook/schema.py +0 -1
- mcli/workflow/notebook/validator.py +7 -3
- mcli/workflow/openai/openai.py +1 -2
- mcli/workflow/registry/registry.py +4 -1
- mcli/workflow/repo/repo.py +6 -7
- mcli/workflow/scheduler/cron_parser.py +16 -19
- mcli/workflow/scheduler/job.py +10 -10
- mcli/workflow/scheduler/monitor.py +15 -15
- mcli/workflow/scheduler/persistence.py +17 -18
- mcli/workflow/scheduler/scheduler.py +37 -38
- mcli/workflow/secrets/__init__.py +1 -1
- mcli/workflow/sync/test_cmd.py +0 -1
- mcli/workflow/wakatime/__init__.py +5 -9
- mcli/workflow/wakatime/wakatime.py +1 -2
- {mcli_framework-7.12.0.dist-info → mcli_framework-7.12.3.dist-info}/METADATA +1 -1
- mcli_framework-7.12.3.dist-info/RECORD +279 -0
- mcli_framework-7.12.0.dist-info/RECORD +0 -279
- {mcli_framework-7.12.0.dist-info → mcli_framework-7.12.3.dist-info}/WHEEL +0 -0
- {mcli_framework-7.12.0.dist-info → mcli_framework-7.12.3.dist-info}/entry_points.txt +0 -0
- {mcli_framework-7.12.0.dist-info → mcli_framework-7.12.3.dist-info}/licenses/LICENSE +0 -0
- {mcli_framework-7.12.0.dist-info → mcli_framework-7.12.3.dist-info}/top_level.txt +0 -0
mcli/ml/cache.py
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
"""Redis caching layer for ML system"""
|
|
1
|
+
"""Redis caching layer for ML system."""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import hashlib
|
|
5
5
|
import json
|
|
6
6
|
import pickle
|
|
7
|
-
from datetime import timedelta
|
|
8
7
|
from functools import wraps
|
|
9
|
-
from typing import Any, Callable, Optional
|
|
8
|
+
from typing import Any, Callable, Optional
|
|
10
9
|
|
|
11
10
|
import redis
|
|
12
11
|
from redis import asyncio as aioredis
|
|
@@ -18,7 +17,7 @@ logger = get_logger(__name__)
|
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
class CacheManager:
|
|
21
|
-
"""Manage Redis cache connections and operations"""
|
|
20
|
+
"""Manage Redis cache connections and operations."""
|
|
22
21
|
|
|
23
22
|
def __init__(self):
|
|
24
23
|
self.redis_client: Optional[redis.Redis] = None
|
|
@@ -26,7 +25,7 @@ class CacheManager:
|
|
|
26
25
|
self._initialized = False
|
|
27
26
|
|
|
28
27
|
def initialize(self):
|
|
29
|
-
"""Initialize Redis connections"""
|
|
28
|
+
"""Initialize Redis connections."""
|
|
30
29
|
if self._initialized:
|
|
31
30
|
return
|
|
32
31
|
|
|
@@ -59,7 +58,7 @@ class CacheManager:
|
|
|
59
58
|
self.redis_client = None
|
|
60
59
|
|
|
61
60
|
async def initialize_async(self):
|
|
62
|
-
"""Initialize async Redis connection"""
|
|
61
|
+
"""Initialize async Redis connection."""
|
|
63
62
|
if self.async_redis_client:
|
|
64
63
|
return
|
|
65
64
|
|
|
@@ -80,41 +79,41 @@ class CacheManager:
|
|
|
80
79
|
self.async_redis_client = None
|
|
81
80
|
|
|
82
81
|
def _make_key(self, key: str, prefix: str = "mcli:ml:") -> str:
|
|
83
|
-
"""Create cache key with prefix"""
|
|
82
|
+
"""Create cache key with prefix."""
|
|
84
83
|
return f"{prefix}{key}"
|
|
85
84
|
|
|
86
85
|
def _serialize(self, value: Any) -> bytes:
|
|
87
|
-
"""Serialize value for storage"""
|
|
88
|
-
try:
|
|
86
|
+
"""Serialize value for storage."""
|
|
87
|
+
try: # noqa: SIM105
|
|
89
88
|
# Try JSON first (for simple types)
|
|
90
89
|
if isinstance(value, (dict, list, str, int, float, bool, type(None))):
|
|
91
90
|
return json.dumps(value).encode("utf-8")
|
|
92
|
-
except:
|
|
91
|
+
except Exception:
|
|
93
92
|
pass
|
|
94
93
|
|
|
95
94
|
# Fall back to pickle for complex objects
|
|
96
95
|
return pickle.dumps(value)
|
|
97
96
|
|
|
98
97
|
def _deserialize(self, value: bytes) -> Any:
|
|
99
|
-
"""Deserialize value from storage"""
|
|
98
|
+
"""Deserialize value from storage."""
|
|
100
99
|
if value is None:
|
|
101
100
|
return None
|
|
102
101
|
|
|
103
102
|
# Try JSON first
|
|
104
|
-
try:
|
|
103
|
+
try: # noqa: SIM105
|
|
105
104
|
return json.loads(value.decode("utf-8"))
|
|
106
|
-
except:
|
|
105
|
+
except Exception:
|
|
107
106
|
pass
|
|
108
107
|
|
|
109
108
|
# Fall back to pickle
|
|
110
109
|
try:
|
|
111
110
|
return pickle.loads(value)
|
|
112
|
-
except:
|
|
111
|
+
except Exception:
|
|
113
112
|
logger.error("Failed to deserialize cache value")
|
|
114
113
|
return None
|
|
115
114
|
|
|
116
115
|
def set(self, key: str, value: Any, expire: int = 3600) -> bool:
|
|
117
|
-
"""Set cache value"""
|
|
116
|
+
"""Set cache value."""
|
|
118
117
|
if not self.redis_client:
|
|
119
118
|
self.initialize()
|
|
120
119
|
|
|
@@ -130,7 +129,7 @@ class CacheManager:
|
|
|
130
129
|
return False
|
|
131
130
|
|
|
132
131
|
def get(self, key: str) -> Any:
|
|
133
|
-
"""Get cache value"""
|
|
132
|
+
"""Get cache value."""
|
|
134
133
|
if not self.redis_client:
|
|
135
134
|
self.initialize()
|
|
136
135
|
|
|
@@ -146,7 +145,7 @@ class CacheManager:
|
|
|
146
145
|
return None
|
|
147
146
|
|
|
148
147
|
async def set_async(self, key: str, value: Any, expire: int = 3600) -> bool:
|
|
149
|
-
"""Set cache value asynchronously"""
|
|
148
|
+
"""Set cache value asynchronously."""
|
|
150
149
|
if not self.async_redis_client:
|
|
151
150
|
await self.initialize_async()
|
|
152
151
|
|
|
@@ -163,7 +162,7 @@ class CacheManager:
|
|
|
163
162
|
return False
|
|
164
163
|
|
|
165
164
|
async def get_async(self, key: str) -> Any:
|
|
166
|
-
"""Get cache value asynchronously"""
|
|
165
|
+
"""Get cache value asynchronously."""
|
|
167
166
|
if not self.async_redis_client:
|
|
168
167
|
await self.initialize_async()
|
|
169
168
|
|
|
@@ -179,7 +178,7 @@ class CacheManager:
|
|
|
179
178
|
return None
|
|
180
179
|
|
|
181
180
|
def delete(self, key: str) -> bool:
|
|
182
|
-
"""Delete cache entry"""
|
|
181
|
+
"""Delete cache entry."""
|
|
183
182
|
if not self.redis_client:
|
|
184
183
|
return False
|
|
185
184
|
|
|
@@ -191,7 +190,7 @@ class CacheManager:
|
|
|
191
190
|
return False
|
|
192
191
|
|
|
193
192
|
async def delete_async(self, key: str) -> bool:
|
|
194
|
-
"""Delete cache entry asynchronously"""
|
|
193
|
+
"""Delete cache entry asynchronously."""
|
|
195
194
|
if not self.async_redis_client:
|
|
196
195
|
return False
|
|
197
196
|
|
|
@@ -204,7 +203,7 @@ class CacheManager:
|
|
|
204
203
|
return False
|
|
205
204
|
|
|
206
205
|
def invalidate_pattern(self, pattern: str) -> int:
|
|
207
|
-
"""Invalidate all keys matching pattern"""
|
|
206
|
+
"""Invalidate all keys matching pattern."""
|
|
208
207
|
if not self.redis_client:
|
|
209
208
|
return 0
|
|
210
209
|
|
|
@@ -219,7 +218,7 @@ class CacheManager:
|
|
|
219
218
|
return 0
|
|
220
219
|
|
|
221
220
|
def get_or_set(self, key: str, func: Callable, expire: int = 3600) -> Any:
|
|
222
|
-
"""Get from cache or compute and set"""
|
|
221
|
+
"""Get from cache or compute and set."""
|
|
223
222
|
value = self.get(key)
|
|
224
223
|
if value is not None:
|
|
225
224
|
return value
|
|
@@ -229,7 +228,7 @@ class CacheManager:
|
|
|
229
228
|
return value
|
|
230
229
|
|
|
231
230
|
async def get_or_set_async(self, key: str, func: Callable, expire: int = 3600) -> Any:
|
|
232
|
-
"""Get from cache or compute and set asynchronously"""
|
|
231
|
+
"""Get from cache or compute and set asynchronously."""
|
|
233
232
|
value = await self.get_async(key)
|
|
234
233
|
if value is not None:
|
|
235
234
|
return value
|
|
@@ -243,13 +242,13 @@ class CacheManager:
|
|
|
243
242
|
return value
|
|
244
243
|
|
|
245
244
|
def close(self):
|
|
246
|
-
"""Close Redis connections"""
|
|
245
|
+
"""Close Redis connections."""
|
|
247
246
|
if self.redis_client:
|
|
248
247
|
self.redis_client.close()
|
|
249
248
|
self.redis_client = None
|
|
250
249
|
|
|
251
250
|
async def close_async(self):
|
|
252
|
-
"""Close async Redis connection"""
|
|
251
|
+
"""Close async Redis connection."""
|
|
253
252
|
if self.async_redis_client:
|
|
254
253
|
await self.async_redis_client.close()
|
|
255
254
|
self.async_redis_client = None
|
|
@@ -261,7 +260,7 @@ cache_manager = CacheManager()
|
|
|
261
260
|
|
|
262
261
|
# Decorator for caching function results
|
|
263
262
|
def cached(expire: int = 3600, key_prefix: str = None):
|
|
264
|
-
"""Decorator to cache function results"""
|
|
263
|
+
"""Decorator to cache function results."""
|
|
265
264
|
|
|
266
265
|
def decorator(func: Callable):
|
|
267
266
|
@wraps(func)
|
|
@@ -334,19 +333,19 @@ def cached(expire: int = 3600, key_prefix: str = None):
|
|
|
334
333
|
|
|
335
334
|
# Cache invalidation helpers
|
|
336
335
|
def invalidate_user_cache(user_id: str):
|
|
337
|
-
"""Invalidate all cache entries for a user"""
|
|
336
|
+
"""Invalidate all cache entries for a user."""
|
|
338
337
|
pattern = f"user:{user_id}:*"
|
|
339
338
|
return cache_manager.invalidate_pattern(pattern)
|
|
340
339
|
|
|
341
340
|
|
|
342
341
|
def invalidate_model_cache(model_id: str):
|
|
343
|
-
"""Invalidate all cache entries for a model"""
|
|
342
|
+
"""Invalidate all cache entries for a model."""
|
|
344
343
|
pattern = f"model:{model_id}:*"
|
|
345
344
|
return cache_manager.invalidate_pattern(pattern)
|
|
346
345
|
|
|
347
346
|
|
|
348
347
|
def invalidate_prediction_cache(prediction_id: str = None):
|
|
349
|
-
"""Invalidate prediction cache"""
|
|
348
|
+
"""Invalidate prediction cache."""
|
|
350
349
|
if prediction_id:
|
|
351
350
|
pattern = f"prediction:{prediction_id}:*"
|
|
352
351
|
else:
|
|
@@ -356,17 +355,17 @@ def invalidate_prediction_cache(prediction_id: str = None):
|
|
|
356
355
|
|
|
357
356
|
# Convenience functions
|
|
358
357
|
async def init_cache():
|
|
359
|
-
"""Initialize cache manager"""
|
|
358
|
+
"""Initialize cache manager."""
|
|
360
359
|
await cache_manager.initialize_async()
|
|
361
360
|
|
|
362
361
|
|
|
363
362
|
async def close_cache():
|
|
364
|
-
"""Close cache connections"""
|
|
363
|
+
"""Close cache connections."""
|
|
365
364
|
await cache_manager.close_async()
|
|
366
365
|
|
|
367
366
|
|
|
368
367
|
async def check_cache_health() -> bool:
|
|
369
|
-
"""Check if cache is healthy"""
|
|
368
|
+
"""Check if cache is healthy."""
|
|
370
369
|
try:
|
|
371
370
|
if not cache_manager.async_redis_client:
|
|
372
371
|
await cache_manager.initialize_async()
|
|
@@ -375,20 +374,20 @@ async def check_cache_health() -> bool:
|
|
|
375
374
|
await cache_manager.async_redis_client.ping()
|
|
376
375
|
return True
|
|
377
376
|
return False
|
|
378
|
-
except:
|
|
377
|
+
except Exception:
|
|
379
378
|
return False
|
|
380
379
|
|
|
381
380
|
|
|
382
381
|
def cache_set(key: str, value: Any, expire: int = 3600):
|
|
383
|
-
"""Set cache value"""
|
|
382
|
+
"""Set cache value."""
|
|
384
383
|
return cache_manager.set(key, value, expire)
|
|
385
384
|
|
|
386
385
|
|
|
387
386
|
def cache_get(key: str):
|
|
388
|
-
"""Get cache value"""
|
|
387
|
+
"""Get cache value."""
|
|
389
388
|
return cache_manager.get(key)
|
|
390
389
|
|
|
391
390
|
|
|
392
391
|
def cache_delete(key: str):
|
|
393
|
-
"""Delete cache entry"""
|
|
392
|
+
"""Delete cache entry."""
|
|
394
393
|
return cache_manager.delete(key)
|
mcli/ml/cli/__init__.py
CHANGED
mcli/ml/cli/main.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Main CLI interface for ML system"""
|
|
1
|
+
"""Main CLI interface for ML system."""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
from pathlib import Path
|
|
@@ -14,10 +14,7 @@ from mcli.ml.config import create_settings, settings
|
|
|
14
14
|
from mcli.ml.experimentation.ab_testing import ABTestingFramework
|
|
15
15
|
from mcli.ml.mlops.pipeline_orchestrator import MLPipeline, PipelineConfig
|
|
16
16
|
from mcli.ml.monitoring.drift_detection import ModelMonitor
|
|
17
|
-
from mcli.ml.optimization.portfolio_optimizer import
|
|
18
|
-
AdvancedPortfolioOptimizer,
|
|
19
|
-
OptimizationObjective,
|
|
20
|
-
)
|
|
17
|
+
from mcli.ml.optimization.portfolio_optimizer import OptimizationObjective
|
|
21
18
|
|
|
22
19
|
app = typer.Typer(
|
|
23
20
|
name="mcli-ml",
|
|
@@ -41,9 +38,9 @@ def train(
|
|
|
41
38
|
False, "--dry-run", help="Validate configuration without training"
|
|
42
39
|
),
|
|
43
40
|
):
|
|
44
|
-
"""Train ML models for stock recommendations"""
|
|
41
|
+
"""Train ML models for stock recommendations."""
|
|
45
42
|
|
|
46
|
-
console.print(
|
|
43
|
+
console.print("[bold blue]Training ML Model[/bold blue]")
|
|
47
44
|
console.print(f"Experiment: {experiment_name}")
|
|
48
45
|
|
|
49
46
|
# Override settings if provided
|
|
@@ -66,7 +63,7 @@ def train(
|
|
|
66
63
|
|
|
67
64
|
if dry_run:
|
|
68
65
|
console.print("[yellow]Dry run mode - validating configuration...[/yellow]")
|
|
69
|
-
|
|
66
|
+
MLPipeline(pipeline_config)
|
|
70
67
|
console.print("[green]✓ Configuration valid[/green]")
|
|
71
68
|
return
|
|
72
69
|
|
|
@@ -114,9 +111,9 @@ def serve(
|
|
|
114
111
|
reload: bool = typer.Option(False, "--reload", help="Enable auto-reload"),
|
|
115
112
|
model_path: Optional[Path] = typer.Option(None, "--model", help="Path to model file"),
|
|
116
113
|
):
|
|
117
|
-
"""Serve trained models via REST API"""
|
|
114
|
+
"""Serve trained models via REST API."""
|
|
118
115
|
|
|
119
|
-
console.print(
|
|
116
|
+
console.print("[bold blue]Starting Model Server[/bold blue]")
|
|
120
117
|
console.print(f"Host: {host}")
|
|
121
118
|
console.print(f"Port: {port}")
|
|
122
119
|
console.print(f"Workers: {workers}")
|
|
@@ -143,9 +140,9 @@ def backtest(
|
|
|
143
140
|
commission: float = typer.Option(0.001, "--commission", help="Commission rate"),
|
|
144
141
|
output_dir: Optional[Path] = typer.Option(None, "--output", help="Output directory"),
|
|
145
142
|
):
|
|
146
|
-
"""Run backtesting on trading strategies"""
|
|
143
|
+
"""Run backtesting on trading strategies."""
|
|
147
144
|
|
|
148
|
-
console.print(
|
|
145
|
+
console.print("[bold blue]Running Backtest[/bold blue]")
|
|
149
146
|
console.print(f"Strategy: {strategy}")
|
|
150
147
|
console.print(f"Initial Capital: ${initial_capital:,.2f}")
|
|
151
148
|
|
|
@@ -157,7 +154,7 @@ def backtest(
|
|
|
157
154
|
)
|
|
158
155
|
|
|
159
156
|
async def run_backtest():
|
|
160
|
-
|
|
157
|
+
BacktestEngine(config)
|
|
161
158
|
|
|
162
159
|
with Progress(
|
|
163
160
|
SpinnerColumn(),
|
|
@@ -203,14 +200,14 @@ def optimize(
|
|
|
203
200
|
max_weight: float = typer.Option(0.4, "--max-weight", help="Maximum weight per asset"),
|
|
204
201
|
risk_free_rate: float = typer.Option(0.02, "--risk-free-rate", help="Risk-free rate"),
|
|
205
202
|
):
|
|
206
|
-
"""Optimize portfolio allocation"""
|
|
203
|
+
"""Optimize portfolio allocation."""
|
|
207
204
|
|
|
208
|
-
console.print(
|
|
205
|
+
console.print("[bold blue]Portfolio Optimization[/bold blue]")
|
|
209
206
|
console.print(f"Objective: {objective}")
|
|
210
207
|
console.print(f"Tickers: {', '.join(tickers)}")
|
|
211
208
|
|
|
212
209
|
try:
|
|
213
|
-
|
|
210
|
+
OptimizationObjective(objective)
|
|
214
211
|
except ValueError:
|
|
215
212
|
console.print(f"[red]Invalid objective: {objective}[/red]")
|
|
216
213
|
console.print(
|
|
@@ -272,12 +269,12 @@ def monitor(
|
|
|
272
269
|
check_drift: bool = typer.Option(True, "--drift", help="Check for data drift"),
|
|
273
270
|
generate_report: bool = typer.Option(False, "--report", help="Generate monitoring report"),
|
|
274
271
|
):
|
|
275
|
-
"""Monitor model performance and data drift"""
|
|
272
|
+
"""Monitor model performance and data drift."""
|
|
276
273
|
|
|
277
|
-
console.print(
|
|
274
|
+
console.print("[bold blue]Model Monitoring[/bold blue]")
|
|
278
275
|
console.print(f"Model: {model_name}")
|
|
279
276
|
|
|
280
|
-
|
|
277
|
+
ModelMonitor(model_name)
|
|
281
278
|
|
|
282
279
|
if check_drift:
|
|
283
280
|
console.print(
|
|
@@ -296,9 +293,9 @@ def experiment(
|
|
|
296
293
|
experiment_id: Optional[str] = typer.Option(None, "--id", help="Experiment ID"),
|
|
297
294
|
name: Optional[str] = typer.Option(None, "--name", help="Experiment name"),
|
|
298
295
|
):
|
|
299
|
-
"""Manage A/B testing experiments"""
|
|
296
|
+
"""Manage A/B testing experiments."""
|
|
300
297
|
|
|
301
|
-
console.print(
|
|
298
|
+
console.print("[bold blue]A/B Testing Experiment[/bold blue]")
|
|
302
299
|
console.print(f"Action: {action}")
|
|
303
300
|
|
|
304
301
|
framework = ABTestingFramework()
|
|
@@ -339,7 +336,7 @@ def experiment(
|
|
|
339
336
|
|
|
340
337
|
@app.command()
|
|
341
338
|
def status():
|
|
342
|
-
"""Show system status and health"""
|
|
339
|
+
"""Show system status and health."""
|
|
343
340
|
|
|
344
341
|
status_table = Table(title="ML System Status")
|
|
345
342
|
status_table.add_column("Component", style="cyan")
|
|
@@ -376,11 +373,11 @@ def config(
|
|
|
376
373
|
environment: Optional[str] = typer.Option(None, "--env", help="Set environment"),
|
|
377
374
|
debug: Optional[bool] = typer.Option(None, "--debug", help="Set debug mode"),
|
|
378
375
|
):
|
|
379
|
-
"""Manage system configuration"""
|
|
376
|
+
"""Manage system configuration."""
|
|
380
377
|
global settings
|
|
381
378
|
|
|
382
379
|
if show:
|
|
383
|
-
console.print(
|
|
380
|
+
console.print("[bold blue]Current Configuration[/bold blue]")
|
|
384
381
|
|
|
385
382
|
# Show all settings in a tree structure
|
|
386
383
|
console.print(f"Environment: {settings.environment}")
|
mcli/ml/config/__init__.py
CHANGED
mcli/ml/config/settings.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
"""Configuration management for ML system"""
|
|
1
|
+
"""Configuration management for ML system."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
import os
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
from typing import Any, Dict, List, Optional
|
|
7
6
|
|
|
@@ -12,7 +11,7 @@ logger = logging.getLogger(__name__)
|
|
|
12
11
|
|
|
13
12
|
|
|
14
13
|
class DatabaseSettings(BaseSettings):
|
|
15
|
-
"""Database configuration"""
|
|
14
|
+
"""Database configuration."""
|
|
16
15
|
|
|
17
16
|
model_config = SettingsConfigDict(env_prefix="DB_")
|
|
18
17
|
|
|
@@ -29,7 +28,7 @@ class DatabaseSettings(BaseSettings):
|
|
|
29
28
|
|
|
30
29
|
@property
|
|
31
30
|
def url(self) -> str:
|
|
32
|
-
"""Get database URL"""
|
|
31
|
+
"""Get database URL."""
|
|
33
32
|
# Use SQLite for local development if no user is specified
|
|
34
33
|
if not self.user:
|
|
35
34
|
return f"sqlite:///{self.name}"
|
|
@@ -37,7 +36,7 @@ class DatabaseSettings(BaseSettings):
|
|
|
37
36
|
|
|
38
37
|
@property
|
|
39
38
|
def async_url(self) -> str:
|
|
40
|
-
"""Get async database URL"""
|
|
39
|
+
"""Get async database URL."""
|
|
41
40
|
# Use aiosqlite for local development if no user is specified
|
|
42
41
|
if not self.user:
|
|
43
42
|
return f"sqlite+aiosqlite:///{self.name}"
|
|
@@ -47,7 +46,7 @@ class DatabaseSettings(BaseSettings):
|
|
|
47
46
|
|
|
48
47
|
|
|
49
48
|
class RedisSettings(BaseSettings):
|
|
50
|
-
"""Redis configuration"""
|
|
49
|
+
"""Redis configuration."""
|
|
51
50
|
|
|
52
51
|
model_config = SettingsConfigDict(env_prefix="REDIS_")
|
|
53
52
|
|
|
@@ -62,13 +61,13 @@ class RedisSettings(BaseSettings):
|
|
|
62
61
|
|
|
63
62
|
@property
|
|
64
63
|
def url(self) -> str:
|
|
65
|
-
"""Get Redis URL"""
|
|
64
|
+
"""Get Redis URL."""
|
|
66
65
|
auth_part = f":{self.password}@" if self.password else ""
|
|
67
66
|
return f"redis://{auth_part}{self.host}:{self.port}/{self.db}"
|
|
68
67
|
|
|
69
68
|
|
|
70
69
|
class MLflowSettings(BaseSettings):
|
|
71
|
-
"""MLflow configuration"""
|
|
70
|
+
"""MLflow configuration."""
|
|
72
71
|
|
|
73
72
|
model_config = SettingsConfigDict(env_prefix="MLFLOW_")
|
|
74
73
|
|
|
@@ -86,7 +85,7 @@ class MLflowSettings(BaseSettings):
|
|
|
86
85
|
|
|
87
86
|
|
|
88
87
|
class ModelSettings(BaseSettings):
|
|
89
|
-
"""Model configuration"""
|
|
88
|
+
"""Model configuration."""
|
|
90
89
|
|
|
91
90
|
model_config = SettingsConfigDict(env_prefix="MODEL_")
|
|
92
91
|
|
|
@@ -110,12 +109,12 @@ class ModelSettings(BaseSettings):
|
|
|
110
109
|
@field_validator("model_dir", "cache_dir", mode="before")
|
|
111
110
|
@classmethod
|
|
112
111
|
def validate_paths(cls, v):
|
|
113
|
-
"""Ensure paths are Path objects"""
|
|
112
|
+
"""Ensure paths are Path objects."""
|
|
114
113
|
return Path(v) if not isinstance(v, Path) else v
|
|
115
114
|
|
|
116
115
|
|
|
117
116
|
class DataSettings(BaseSettings):
|
|
118
|
-
"""Data configuration"""
|
|
117
|
+
"""Data configuration."""
|
|
119
118
|
|
|
120
119
|
model_config = SettingsConfigDict(env_prefix="DATA_")
|
|
121
120
|
|
|
@@ -137,12 +136,12 @@ class DataSettings(BaseSettings):
|
|
|
137
136
|
@field_validator("data_dir", "raw_dir", "processed_dir", "dvc_cache_dir", mode="before")
|
|
138
137
|
@classmethod
|
|
139
138
|
def validate_paths(cls, v):
|
|
140
|
-
"""Ensure paths are Path objects"""
|
|
139
|
+
"""Ensure paths are Path objects."""
|
|
141
140
|
return Path(v) if not isinstance(v, Path) else v
|
|
142
141
|
|
|
143
142
|
|
|
144
143
|
class APISettings(BaseSettings):
|
|
145
|
-
"""API configuration"""
|
|
144
|
+
"""API configuration."""
|
|
146
145
|
|
|
147
146
|
model_config = SettingsConfigDict(env_prefix="API_")
|
|
148
147
|
|
|
@@ -166,7 +165,7 @@ class APISettings(BaseSettings):
|
|
|
166
165
|
|
|
167
166
|
|
|
168
167
|
class MonitoringSettings(BaseSettings):
|
|
169
|
-
"""Monitoring configuration"""
|
|
168
|
+
"""Monitoring configuration."""
|
|
170
169
|
|
|
171
170
|
model_config = SettingsConfigDict(env_prefix="MONITORING_")
|
|
172
171
|
|
|
@@ -188,7 +187,7 @@ class MonitoringSettings(BaseSettings):
|
|
|
188
187
|
|
|
189
188
|
|
|
190
189
|
class SecuritySettings(BaseSettings):
|
|
191
|
-
"""Security configuration"""
|
|
190
|
+
"""Security configuration."""
|
|
192
191
|
|
|
193
192
|
model_config = SettingsConfigDict(env_prefix="SECURITY_")
|
|
194
193
|
|
|
@@ -207,12 +206,12 @@ class SecuritySettings(BaseSettings):
|
|
|
207
206
|
@field_validator("ssl_cert_path", "ssl_key_path", mode="before")
|
|
208
207
|
@classmethod
|
|
209
208
|
def validate_ssl_paths(cls, v):
|
|
210
|
-
"""Ensure SSL paths are Path objects if provided"""
|
|
209
|
+
"""Ensure SSL paths are Path objects if provided."""
|
|
211
210
|
return Path(v) if v and not isinstance(v, Path) else v
|
|
212
211
|
|
|
213
212
|
|
|
214
213
|
class Settings(BaseSettings):
|
|
215
|
-
"""Main application settings"""
|
|
214
|
+
"""Main application settings."""
|
|
216
215
|
|
|
217
216
|
model_config = SettingsConfigDict(
|
|
218
217
|
env_file=".env", env_file_encoding="utf-8", case_sensitive=False, extra="ignore"
|
|
@@ -237,7 +236,7 @@ class Settings(BaseSettings):
|
|
|
237
236
|
@field_validator("environment")
|
|
238
237
|
@classmethod
|
|
239
238
|
def validate_environment(cls, v):
|
|
240
|
-
"""Validate environment value"""
|
|
239
|
+
"""Validate environment value."""
|
|
241
240
|
valid_envs = ["development", "staging", "production"]
|
|
242
241
|
if v not in valid_envs:
|
|
243
242
|
raise ValueError(f"Environment must be one of {valid_envs}")
|
|
@@ -248,7 +247,7 @@ class Settings(BaseSettings):
|
|
|
248
247
|
self._create_directories()
|
|
249
248
|
|
|
250
249
|
def _create_directories(self):
|
|
251
|
-
"""Create necessary directories"""
|
|
250
|
+
"""Create necessary directories."""
|
|
252
251
|
directories = [
|
|
253
252
|
self.model.model_dir,
|
|
254
253
|
self.model.cache_dir,
|
|
@@ -263,16 +262,16 @@ class Settings(BaseSettings):
|
|
|
263
262
|
|
|
264
263
|
@property
|
|
265
264
|
def is_production(self) -> bool:
|
|
266
|
-
"""Check if running in production"""
|
|
265
|
+
"""Check if running in production."""
|
|
267
266
|
return self.environment == "production"
|
|
268
267
|
|
|
269
268
|
@property
|
|
270
269
|
def is_development(self) -> bool:
|
|
271
|
-
"""Check if running in development"""
|
|
270
|
+
"""Check if running in development."""
|
|
272
271
|
return self.environment == "development"
|
|
273
272
|
|
|
274
273
|
def get_database_config(self) -> Dict[str, Any]:
|
|
275
|
-
"""Get database configuration for SQLAlchemy"""
|
|
274
|
+
"""Get database configuration for SQLAlchemy."""
|
|
276
275
|
return {
|
|
277
276
|
"pool_size": self.database.pool_size,
|
|
278
277
|
"max_overflow": self.database.max_overflow,
|
|
@@ -282,7 +281,7 @@ class Settings(BaseSettings):
|
|
|
282
281
|
}
|
|
283
282
|
|
|
284
283
|
def get_redis_config(self) -> Dict[str, Any]:
|
|
285
|
-
"""Get Redis configuration"""
|
|
284
|
+
"""Get Redis configuration."""
|
|
286
285
|
return {
|
|
287
286
|
"host": self.redis.host,
|
|
288
287
|
"port": self.redis.port,
|
|
@@ -299,12 +298,12 @@ settings = Settings()
|
|
|
299
298
|
|
|
300
299
|
|
|
301
300
|
def get_settings() -> Settings:
|
|
302
|
-
"""Get settings instance (for dependency injection)"""
|
|
301
|
+
"""Get settings instance (for dependency injection)."""
|
|
303
302
|
return settings
|
|
304
303
|
|
|
305
304
|
|
|
306
305
|
def update_settings(**kwargs) -> Settings:
|
|
307
|
-
"""Update settings with new values"""
|
|
306
|
+
"""Update settings with new values."""
|
|
308
307
|
global settings
|
|
309
308
|
|
|
310
309
|
# Create new settings instance with updated values
|
|
@@ -317,7 +316,7 @@ def update_settings(**kwargs) -> Settings:
|
|
|
317
316
|
|
|
318
317
|
# Environment-specific configurations
|
|
319
318
|
def get_development_config() -> Dict[str, Any]:
|
|
320
|
-
"""Get development-specific configuration overrides"""
|
|
319
|
+
"""Get development-specific configuration overrides."""
|
|
321
320
|
return {
|
|
322
321
|
"debug": True,
|
|
323
322
|
"database": {
|
|
@@ -341,7 +340,7 @@ def get_development_config() -> Dict[str, Any]:
|
|
|
341
340
|
|
|
342
341
|
|
|
343
342
|
def get_production_config() -> Dict[str, Any]:
|
|
344
|
-
"""Get production-specific configuration overrides"""
|
|
343
|
+
"""Get production-specific configuration overrides."""
|
|
345
344
|
return {
|
|
346
345
|
"debug": False,
|
|
347
346
|
"monitoring": {
|
|
@@ -356,7 +355,7 @@ def get_production_config() -> Dict[str, Any]:
|
|
|
356
355
|
|
|
357
356
|
|
|
358
357
|
def get_testing_config() -> Dict[str, Any]:
|
|
359
|
-
"""Get testing-specific configuration overrides"""
|
|
358
|
+
"""Get testing-specific configuration overrides."""
|
|
360
359
|
return {
|
|
361
360
|
"debug": True,
|
|
362
361
|
"database": {
|
|
@@ -374,7 +373,7 @@ def get_testing_config() -> Dict[str, Any]:
|
|
|
374
373
|
|
|
375
374
|
# Configuration factory
|
|
376
375
|
def create_settings(environment: str = "development") -> Settings:
|
|
377
|
-
"""Create settings for specific environment"""
|
|
376
|
+
"""Create settings for specific environment."""
|
|
378
377
|
base_config = {}
|
|
379
378
|
|
|
380
379
|
if environment == "development":
|
mcli/ml/configs/__init__.py
CHANGED