mcli-framework 7.12.1__py3-none-any.whl → 7.12.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mcli-framework might be problematic. Click here for more details.
- mcli/app/__init__.py +0 -2
- mcli/app/commands_cmd.py +19 -23
- mcli/app/completion_helpers.py +5 -5
- mcli/app/init_cmd.py +10 -10
- mcli/app/lock_cmd.py +82 -27
- mcli/app/main.py +2 -8
- mcli/app/model/model.py +5 -10
- mcli/app/store_cmd.py +8 -8
- mcli/app/video/__init__.py +0 -2
- mcli/app/video/video.py +1 -14
- mcli/chat/chat.py +90 -108
- mcli/chat/command_rag.py +0 -4
- mcli/chat/enhanced_chat.py +32 -41
- mcli/chat/system_controller.py +37 -37
- mcli/chat/system_integration.py +4 -5
- mcli/cli.py +2 -3
- mcli/lib/api/api.py +4 -9
- mcli/lib/api/daemon_client.py +19 -20
- mcli/lib/api/daemon_client_local.py +1 -3
- mcli/lib/api/daemon_decorator.py +6 -6
- mcli/lib/api/mcli_decorators.py +4 -8
- mcli/lib/auth/__init__.py +0 -1
- mcli/lib/auth/auth.py +4 -5
- mcli/lib/auth/mcli_manager.py +7 -12
- mcli/lib/auth/token_util.py +5 -5
- mcli/lib/config/__init__.py +29 -1
- mcli/lib/config/config.py +0 -1
- mcli/lib/custom_commands.py +1 -1
- mcli/lib/discovery/command_discovery.py +15 -15
- mcli/lib/erd/erd.py +7 -7
- mcli/lib/files/files.py +1 -1
- mcli/lib/fs/__init__.py +31 -1
- mcli/lib/fs/fs.py +12 -13
- mcli/lib/lib.py +0 -1
- mcli/lib/logger/logger.py +7 -10
- mcli/lib/performance/optimizer.py +25 -27
- mcli/lib/performance/rust_bridge.py +22 -27
- mcli/lib/performance/uvloop_config.py +0 -1
- mcli/lib/pickles/__init__.py +0 -1
- mcli/lib/pickles/pickles.py +0 -2
- mcli/lib/secrets/commands.py +0 -2
- mcli/lib/secrets/manager.py +0 -1
- mcli/lib/secrets/repl.py +2 -3
- mcli/lib/secrets/store.py +1 -2
- mcli/lib/services/data_pipeline.py +34 -34
- mcli/lib/services/lsh_client.py +38 -40
- mcli/lib/shell/shell.py +2 -2
- mcli/lib/toml/__init__.py +0 -1
- mcli/lib/ui/styling.py +0 -1
- mcli/lib/ui/visual_effects.py +33 -41
- mcli/lib/watcher/watcher.py +0 -1
- mcli/ml/__init__.py +1 -1
- mcli/ml/api/__init__.py +1 -1
- mcli/ml/api/app.py +8 -9
- mcli/ml/api/middleware.py +10 -10
- mcli/ml/api/routers/__init__.py +1 -1
- mcli/ml/api/routers/admin_router.py +3 -3
- mcli/ml/api/routers/auth_router.py +17 -18
- mcli/ml/api/routers/backtest_router.py +2 -2
- mcli/ml/api/routers/data_router.py +2 -2
- mcli/ml/api/routers/model_router.py +14 -15
- mcli/ml/api/routers/monitoring_router.py +2 -2
- mcli/ml/api/routers/portfolio_router.py +2 -2
- mcli/ml/api/routers/prediction_router.py +10 -9
- mcli/ml/api/routers/trade_router.py +2 -2
- mcli/ml/api/routers/websocket_router.py +6 -7
- mcli/ml/api/schemas.py +2 -2
- mcli/ml/auth/__init__.py +1 -1
- mcli/ml/auth/auth_manager.py +22 -23
- mcli/ml/auth/models.py +17 -17
- mcli/ml/auth/permissions.py +17 -17
- mcli/ml/backtesting/__init__.py +1 -1
- mcli/ml/backtesting/backtest_engine.py +31 -35
- mcli/ml/backtesting/performance_metrics.py +12 -14
- mcli/ml/backtesting/run.py +1 -2
- mcli/ml/cache.py +35 -36
- mcli/ml/cli/__init__.py +1 -1
- mcli/ml/cli/main.py +21 -24
- mcli/ml/config/__init__.py +1 -1
- mcli/ml/config/settings.py +28 -29
- mcli/ml/configs/__init__.py +1 -1
- mcli/ml/configs/dvc_config.py +14 -15
- mcli/ml/configs/mlflow_config.py +12 -13
- mcli/ml/configs/mlops_manager.py +19 -21
- mcli/ml/dashboard/__init__.py +4 -4
- mcli/ml/dashboard/app.py +20 -30
- mcli/ml/dashboard/app_supabase.py +16 -19
- mcli/ml/dashboard/app_training.py +11 -14
- mcli/ml/dashboard/cli.py +2 -2
- mcli/ml/dashboard/common.py +2 -3
- mcli/ml/dashboard/components/__init__.py +1 -1
- mcli/ml/dashboard/components/charts.py +13 -11
- mcli/ml/dashboard/components/metrics.py +7 -7
- mcli/ml/dashboard/components/tables.py +12 -9
- mcli/ml/dashboard/overview.py +2 -2
- mcli/ml/dashboard/pages/__init__.py +1 -1
- mcli/ml/dashboard/pages/cicd.py +15 -18
- mcli/ml/dashboard/pages/debug_dependencies.py +7 -7
- mcli/ml/dashboard/pages/monte_carlo_predictions.py +11 -18
- mcli/ml/dashboard/pages/predictions_enhanced.py +24 -32
- mcli/ml/dashboard/pages/scrapers_and_logs.py +22 -24
- mcli/ml/dashboard/pages/test_portfolio.py +3 -6
- mcli/ml/dashboard/pages/trading.py +16 -18
- mcli/ml/dashboard/pages/workflows.py +20 -30
- mcli/ml/dashboard/utils.py +9 -9
- mcli/ml/dashboard/warning_suppression.py +3 -3
- mcli/ml/data_ingestion/__init__.py +1 -1
- mcli/ml/data_ingestion/api_connectors.py +41 -46
- mcli/ml/data_ingestion/data_pipeline.py +36 -46
- mcli/ml/data_ingestion/stream_processor.py +43 -46
- mcli/ml/database/__init__.py +1 -1
- mcli/ml/database/migrations/env.py +2 -2
- mcli/ml/database/models.py +22 -24
- mcli/ml/database/session.py +14 -14
- mcli/ml/experimentation/__init__.py +1 -1
- mcli/ml/experimentation/ab_testing.py +45 -46
- mcli/ml/features/__init__.py +1 -1
- mcli/ml/features/ensemble_features.py +22 -27
- mcli/ml/features/recommendation_engine.py +30 -30
- mcli/ml/features/stock_features.py +29 -32
- mcli/ml/features/test_feature_engineering.py +10 -11
- mcli/ml/logging.py +4 -4
- mcli/ml/mlops/__init__.py +1 -1
- mcli/ml/mlops/data_versioning.py +29 -30
- mcli/ml/mlops/experiment_tracker.py +24 -24
- mcli/ml/mlops/model_serving.py +31 -34
- mcli/ml/mlops/pipeline_orchestrator.py +27 -35
- mcli/ml/models/__init__.py +5 -6
- mcli/ml/models/base_models.py +23 -23
- mcli/ml/models/ensemble_models.py +31 -31
- mcli/ml/models/recommendation_models.py +18 -19
- mcli/ml/models/test_models.py +14 -16
- mcli/ml/monitoring/__init__.py +1 -1
- mcli/ml/monitoring/drift_detection.py +32 -36
- mcli/ml/monitoring/metrics.py +2 -2
- mcli/ml/optimization/__init__.py +1 -1
- mcli/ml/optimization/optimize.py +1 -2
- mcli/ml/optimization/portfolio_optimizer.py +30 -32
- mcli/ml/predictions/__init__.py +1 -1
- mcli/ml/preprocessing/__init__.py +1 -1
- mcli/ml/preprocessing/data_cleaners.py +22 -23
- mcli/ml/preprocessing/feature_extractors.py +23 -26
- mcli/ml/preprocessing/ml_pipeline.py +23 -23
- mcli/ml/preprocessing/test_preprocessing.py +7 -8
- mcli/ml/scripts/populate_sample_data.py +0 -4
- mcli/ml/serving/serve.py +1 -2
- mcli/ml/tasks.py +17 -17
- mcli/ml/tests/test_integration.py +29 -30
- mcli/ml/tests/test_training_dashboard.py +21 -21
- mcli/ml/trading/__init__.py +1 -1
- mcli/ml/trading/migrations.py +5 -5
- mcli/ml/trading/models.py +21 -23
- mcli/ml/trading/paper_trading.py +16 -13
- mcli/ml/trading/risk_management.py +17 -18
- mcli/ml/trading/trading_service.py +25 -28
- mcli/ml/training/__init__.py +1 -1
- mcli/ml/training/train.py +0 -1
- mcli/public/oi/oi.py +1 -2
- mcli/self/completion_cmd.py +6 -10
- mcli/self/logs_cmd.py +19 -24
- mcli/self/migrate_cmd.py +22 -20
- mcli/self/redis_cmd.py +10 -11
- mcli/self/self_cmd.py +10 -18
- mcli/self/store_cmd.py +10 -12
- mcli/self/visual_cmd.py +9 -14
- mcli/self/zsh_cmd.py +2 -4
- mcli/workflow/daemon/async_command_database.py +23 -24
- mcli/workflow/daemon/async_process_manager.py +27 -29
- mcli/workflow/daemon/client.py +27 -33
- mcli/workflow/daemon/daemon.py +32 -36
- mcli/workflow/daemon/enhanced_daemon.py +24 -33
- mcli/workflow/daemon/process_cli.py +11 -12
- mcli/workflow/daemon/process_manager.py +23 -26
- mcli/workflow/daemon/test_daemon.py +4 -5
- mcli/workflow/dashboard/dashboard_cmd.py +0 -1
- mcli/workflow/doc_convert.py +15 -17
- mcli/workflow/gcloud/__init__.py +0 -1
- mcli/workflow/gcloud/gcloud.py +11 -8
- mcli/workflow/git_commit/ai_service.py +14 -15
- mcli/workflow/lsh_integration.py +9 -11
- mcli/workflow/model_service/client.py +26 -31
- mcli/workflow/model_service/download_and_run_efficient_models.py +10 -14
- mcli/workflow/model_service/lightweight_embedder.py +25 -35
- mcli/workflow/model_service/lightweight_model_server.py +26 -32
- mcli/workflow/model_service/lightweight_test.py +7 -10
- mcli/workflow/model_service/model_service.py +80 -91
- mcli/workflow/model_service/ollama_efficient_runner.py +14 -18
- mcli/workflow/model_service/openai_adapter.py +23 -23
- mcli/workflow/model_service/pdf_processor.py +21 -26
- mcli/workflow/model_service/test_efficient_runner.py +12 -16
- mcli/workflow/model_service/test_example.py +11 -13
- mcli/workflow/model_service/test_integration.py +3 -5
- mcli/workflow/model_service/test_new_features.py +7 -8
- mcli/workflow/notebook/converter.py +1 -1
- mcli/workflow/notebook/notebook_cmd.py +5 -6
- mcli/workflow/notebook/schema.py +0 -1
- mcli/workflow/notebook/validator.py +7 -3
- mcli/workflow/openai/openai.py +1 -2
- mcli/workflow/registry/registry.py +4 -1
- mcli/workflow/repo/repo.py +6 -7
- mcli/workflow/scheduler/cron_parser.py +16 -19
- mcli/workflow/scheduler/job.py +10 -10
- mcli/workflow/scheduler/monitor.py +15 -15
- mcli/workflow/scheduler/persistence.py +17 -18
- mcli/workflow/scheduler/scheduler.py +37 -38
- mcli/workflow/secrets/__init__.py +1 -1
- mcli/workflow/sync/test_cmd.py +0 -1
- mcli/workflow/wakatime/__init__.py +5 -9
- mcli/workflow/wakatime/wakatime.py +1 -2
- {mcli_framework-7.12.1.dist-info → mcli_framework-7.12.3.dist-info}/METADATA +1 -1
- mcli_framework-7.12.3.dist-info/RECORD +279 -0
- mcli_framework-7.12.1.dist-info/RECORD +0 -279
- {mcli_framework-7.12.1.dist-info → mcli_framework-7.12.3.dist-info}/WHEEL +0 -0
- {mcli_framework-7.12.1.dist-info → mcli_framework-7.12.3.dist-info}/entry_points.txt +0 -0
- {mcli_framework-7.12.1.dist-info → mcli_framework-7.12.3.dist-info}/licenses/LICENSE +0 -0
- {mcli_framework-7.12.1.dist-info → mcli_framework-7.12.3.dist-info}/top_level.txt +0 -0
|
@@ -4,18 +4,18 @@ Job monitoring and execution tracking for the MCLI scheduler
|
|
|
4
4
|
|
|
5
5
|
import threading
|
|
6
6
|
import time
|
|
7
|
-
from datetime import datetime
|
|
7
|
+
from datetime import datetime
|
|
8
8
|
from typing import Callable, Dict, List, Optional
|
|
9
9
|
|
|
10
10
|
from mcli.lib.logger.logger import get_logger
|
|
11
11
|
|
|
12
|
-
from .job import
|
|
12
|
+
from .job import ScheduledJob
|
|
13
13
|
|
|
14
14
|
logger = get_logger(__name__)
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class JobMonitor:
|
|
18
|
-
"""Monitors running jobs and handles timeouts, retries, and status updates"""
|
|
18
|
+
"""Monitors running jobs and handles timeouts, retries, and status updates."""
|
|
19
19
|
|
|
20
20
|
def __init__(self, status_callback: Optional[Callable] = None):
|
|
21
21
|
self.running_jobs: Dict[str, threading.Thread] = {}
|
|
@@ -26,7 +26,7 @@ class JobMonitor:
|
|
|
26
26
|
self.lock = threading.Lock()
|
|
27
27
|
|
|
28
28
|
def start_monitoring(self):
|
|
29
|
-
"""Start the monitoring thread"""
|
|
29
|
+
"""Start the monitoring thread."""
|
|
30
30
|
if self.monitoring:
|
|
31
31
|
return
|
|
32
32
|
|
|
@@ -36,14 +36,14 @@ class JobMonitor:
|
|
|
36
36
|
logger.info("Job monitor started")
|
|
37
37
|
|
|
38
38
|
def stop_monitoring(self):
|
|
39
|
-
"""Stop the monitoring thread"""
|
|
39
|
+
"""Stop the monitoring thread."""
|
|
40
40
|
self.monitoring = False
|
|
41
41
|
if self.monitor_thread:
|
|
42
42
|
self.monitor_thread.join(timeout=5)
|
|
43
43
|
logger.info("Job monitor stopped")
|
|
44
44
|
|
|
45
45
|
def _monitor_loop(self):
|
|
46
|
-
"""Main monitoring loop"""
|
|
46
|
+
"""Main monitoring loop."""
|
|
47
47
|
while self.monitoring:
|
|
48
48
|
try:
|
|
49
49
|
self._check_running_jobs()
|
|
@@ -52,7 +52,7 @@ class JobMonitor:
|
|
|
52
52
|
logger.error(f"Error in monitor loop: {e}")
|
|
53
53
|
|
|
54
54
|
def _check_running_jobs(self):
|
|
55
|
-
"""Check status of running jobs"""
|
|
55
|
+
"""Check status of running jobs."""
|
|
56
56
|
with self.lock:
|
|
57
57
|
current_time = datetime.now()
|
|
58
58
|
jobs_to_remove = []
|
|
@@ -61,7 +61,7 @@ class JobMonitor:
|
|
|
61
61
|
start_time = self.job_start_times.get(job_id)
|
|
62
62
|
|
|
63
63
|
if start_time:
|
|
64
|
-
|
|
64
|
+
(current_time - start_time).total_seconds()
|
|
65
65
|
|
|
66
66
|
# Check if thread is still alive
|
|
67
67
|
if not thread.is_alive():
|
|
@@ -76,29 +76,29 @@ class JobMonitor:
|
|
|
76
76
|
self._remove_job(job_id)
|
|
77
77
|
|
|
78
78
|
def add_job(self, job: ScheduledJob, thread: threading.Thread):
|
|
79
|
-
"""Add a job to monitoring"""
|
|
79
|
+
"""Add a job to monitoring."""
|
|
80
80
|
with self.lock:
|
|
81
81
|
self.running_jobs[job.id] = thread
|
|
82
82
|
self.job_start_times[job.id] = datetime.now()
|
|
83
83
|
logger.debug(f"Added job {job.id} to monitor")
|
|
84
84
|
|
|
85
85
|
def _remove_job(self, job_id: str):
|
|
86
|
-
"""Remove a job from monitoring"""
|
|
86
|
+
"""Remove a job from monitoring."""
|
|
87
87
|
self.running_jobs.pop(job_id, None)
|
|
88
88
|
self.job_start_times.pop(job_id, None)
|
|
89
89
|
|
|
90
90
|
def get_running_jobs(self) -> List[str]:
|
|
91
|
-
"""Get list of currently running job IDs"""
|
|
91
|
+
"""Get list of currently running job IDs."""
|
|
92
92
|
with self.lock:
|
|
93
93
|
return list(self.running_jobs.keys())
|
|
94
94
|
|
|
95
95
|
def is_job_running(self, job_id: str) -> bool:
|
|
96
|
-
"""Check if a specific job is currently running"""
|
|
96
|
+
"""Check if a specific job is currently running."""
|
|
97
97
|
with self.lock:
|
|
98
98
|
return job_id in self.running_jobs
|
|
99
99
|
|
|
100
100
|
def get_job_runtime(self, job_id: str) -> Optional[int]:
|
|
101
|
-
"""Get runtime in seconds for a running job"""
|
|
101
|
+
"""Get runtime in seconds for a running job."""
|
|
102
102
|
with self.lock:
|
|
103
103
|
start_time = self.job_start_times.get(job_id)
|
|
104
104
|
if start_time:
|
|
@@ -106,7 +106,7 @@ class JobMonitor:
|
|
|
106
106
|
return None
|
|
107
107
|
|
|
108
108
|
def kill_job(self, job_id: str) -> bool:
|
|
109
|
-
"""Attempt to kill a running job"""
|
|
109
|
+
"""Attempt to kill a running job."""
|
|
110
110
|
with self.lock:
|
|
111
111
|
thread = self.running_jobs.get(job_id)
|
|
112
112
|
if thread and thread.is_alive():
|
|
@@ -117,7 +117,7 @@ class JobMonitor:
|
|
|
117
117
|
return True
|
|
118
118
|
|
|
119
119
|
def get_monitor_stats(self) -> dict:
|
|
120
|
-
"""Get monitoring statistics"""
|
|
120
|
+
"""Get monitoring statistics."""
|
|
121
121
|
with self.lock:
|
|
122
122
|
stats = {
|
|
123
123
|
"monitoring": self.monitoring,
|
|
@@ -5,11 +5,10 @@ Handles saving/loading jobs to/from disk, ensuring persistence across power cycl
|
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
7
|
import json
|
|
8
|
-
import os
|
|
9
8
|
import threading
|
|
10
9
|
from datetime import datetime
|
|
11
10
|
from pathlib import Path
|
|
12
|
-
from typing import
|
|
11
|
+
from typing import List, Optional
|
|
13
12
|
|
|
14
13
|
from mcli.lib.logger.logger import get_logger
|
|
15
14
|
|
|
@@ -19,7 +18,7 @@ logger = get_logger(__name__)
|
|
|
19
18
|
|
|
20
19
|
|
|
21
20
|
class JobStorage:
|
|
22
|
-
"""Handles persistent storage of scheduled jobs"""
|
|
21
|
+
"""Handles persistent storage of scheduled jobs."""
|
|
23
22
|
|
|
24
23
|
def __init__(self, storage_dir: Optional[str] = None):
|
|
25
24
|
self.storage_dir = Path(storage_dir) if storage_dir else self._get_default_storage_dir()
|
|
@@ -34,13 +33,13 @@ class JobStorage:
|
|
|
34
33
|
self._initialize_storage()
|
|
35
34
|
|
|
36
35
|
def _get_default_storage_dir(self) -> Path:
|
|
37
|
-
"""Get default storage directory"""
|
|
36
|
+
"""Get default storage directory."""
|
|
38
37
|
home = Path.home()
|
|
39
38
|
storage_dir = home / ".mcli" / "scheduler"
|
|
40
39
|
return storage_dir
|
|
41
40
|
|
|
42
41
|
def _initialize_storage(self):
|
|
43
|
-
"""Initialize storage files if they don't exist"""
|
|
42
|
+
"""Initialize storage files if they don't exist."""
|
|
44
43
|
if not self.jobs_file.exists():
|
|
45
44
|
self._write_json_file(self.jobs_file, {"jobs": [], "version": "1.0"})
|
|
46
45
|
|
|
@@ -48,7 +47,7 @@ class JobStorage:
|
|
|
48
47
|
self._write_json_file(self.history_file, {"history": [], "version": "1.0"})
|
|
49
48
|
|
|
50
49
|
def _read_json_file(self, file_path: Path) -> dict:
|
|
51
|
-
"""Safely read JSON file with error handling"""
|
|
50
|
+
"""Safely read JSON file with error handling."""
|
|
52
51
|
try:
|
|
53
52
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
54
53
|
return json.load(f)
|
|
@@ -63,7 +62,7 @@ class JobStorage:
|
|
|
63
62
|
return {}
|
|
64
63
|
|
|
65
64
|
def _write_json_file(self, file_path: Path, data: dict):
|
|
66
|
-
"""Safely write JSON file with atomic operation"""
|
|
65
|
+
"""Safely write JSON file with atomic operation."""
|
|
67
66
|
temp_file = file_path.with_suffix(".tmp")
|
|
68
67
|
try:
|
|
69
68
|
with open(temp_file, "w", encoding="utf-8") as f:
|
|
@@ -78,7 +77,7 @@ class JobStorage:
|
|
|
78
77
|
temp_file.unlink()
|
|
79
78
|
|
|
80
79
|
def save_jobs(self, jobs: List[ScheduledJob]) -> bool:
|
|
81
|
-
"""Save list of jobs to persistent storage"""
|
|
80
|
+
"""Save list of jobs to persistent storage."""
|
|
82
81
|
with self.lock:
|
|
83
82
|
try:
|
|
84
83
|
jobs_data = {
|
|
@@ -97,7 +96,7 @@ class JobStorage:
|
|
|
97
96
|
return False
|
|
98
97
|
|
|
99
98
|
def load_jobs(self) -> List[ScheduledJob]:
|
|
100
|
-
"""Load jobs from persistent storage"""
|
|
99
|
+
"""Load jobs from persistent storage."""
|
|
101
100
|
with self.lock:
|
|
102
101
|
try:
|
|
103
102
|
data = self._read_json_file(self.jobs_file)
|
|
@@ -119,7 +118,7 @@ class JobStorage:
|
|
|
119
118
|
return []
|
|
120
119
|
|
|
121
120
|
def save_job(self, job: ScheduledJob) -> bool:
|
|
122
|
-
"""Save a single job (update existing or add new)"""
|
|
121
|
+
"""Save a single job (update existing or add new)."""
|
|
123
122
|
jobs = self.load_jobs()
|
|
124
123
|
|
|
125
124
|
# Find existing job or add new one
|
|
@@ -136,7 +135,7 @@ class JobStorage:
|
|
|
136
135
|
return self.save_jobs(jobs)
|
|
137
136
|
|
|
138
137
|
def delete_job(self, job_id: str) -> bool:
|
|
139
|
-
"""Delete a job from storage"""
|
|
138
|
+
"""Delete a job from storage."""
|
|
140
139
|
jobs = self.load_jobs()
|
|
141
140
|
original_count = len(jobs)
|
|
142
141
|
|
|
@@ -147,7 +146,7 @@ class JobStorage:
|
|
|
147
146
|
return False
|
|
148
147
|
|
|
149
148
|
def get_job(self, job_id: str) -> Optional[ScheduledJob]:
|
|
150
|
-
"""Get a specific job by ID"""
|
|
149
|
+
"""Get a specific job by ID."""
|
|
151
150
|
jobs = self.load_jobs()
|
|
152
151
|
for job in jobs:
|
|
153
152
|
if job.id == job_id:
|
|
@@ -155,7 +154,7 @@ class JobStorage:
|
|
|
155
154
|
return None
|
|
156
155
|
|
|
157
156
|
def record_job_execution(self, job: ScheduledJob, execution_data: dict):
|
|
158
|
-
"""Record job execution in history"""
|
|
157
|
+
"""Record job execution in history."""
|
|
159
158
|
with self.lock:
|
|
160
159
|
try:
|
|
161
160
|
history_data = self._read_json_file(self.history_file)
|
|
@@ -192,7 +191,7 @@ class JobStorage:
|
|
|
192
191
|
logger.error(f"Failed to record job execution: {e}")
|
|
193
192
|
|
|
194
193
|
def get_job_history(self, job_id: Optional[str] = None, limit: int = 100) -> List[dict]:
|
|
195
|
-
"""Get job execution history"""
|
|
194
|
+
"""Get job execution history."""
|
|
196
195
|
try:
|
|
197
196
|
history_data = self._read_json_file(self.history_file)
|
|
198
197
|
history = history_data.get("history", [])
|
|
@@ -210,7 +209,7 @@ class JobStorage:
|
|
|
210
209
|
return []
|
|
211
210
|
|
|
212
211
|
def cleanup_old_history(self, days: int = 30):
|
|
213
|
-
"""Remove job history older than specified days"""
|
|
212
|
+
"""Remove job history older than specified days."""
|
|
214
213
|
with self.lock:
|
|
215
214
|
try:
|
|
216
215
|
cutoff_date = datetime.now() - timedelta(days=days)
|
|
@@ -240,7 +239,7 @@ class JobStorage:
|
|
|
240
239
|
logger.error(f"Failed to cleanup old history: {e}")
|
|
241
240
|
|
|
242
241
|
def export_jobs(self, export_path: str) -> bool:
|
|
243
|
-
"""Export all jobs to a file"""
|
|
242
|
+
"""Export all jobs to a file."""
|
|
244
243
|
try:
|
|
245
244
|
jobs = self.load_jobs()
|
|
246
245
|
export_data = {
|
|
@@ -261,7 +260,7 @@ class JobStorage:
|
|
|
261
260
|
return False
|
|
262
261
|
|
|
263
262
|
def import_jobs(self, import_path: str, replace: bool = False) -> int:
|
|
264
|
-
"""Import jobs from a file"""
|
|
263
|
+
"""Import jobs from a file."""
|
|
265
264
|
try:
|
|
266
265
|
with open(import_path, "r", encoding="utf-8") as f:
|
|
267
266
|
import_data = json.load(f)
|
|
@@ -296,7 +295,7 @@ class JobStorage:
|
|
|
296
295
|
return 0
|
|
297
296
|
|
|
298
297
|
def get_storage_info(self) -> dict:
|
|
299
|
-
"""Get information about storage usage"""
|
|
298
|
+
"""Get information about storage usage."""
|
|
300
299
|
try:
|
|
301
300
|
jobs_size = self.jobs_file.stat().st_size if self.jobs_file.exists() else 0
|
|
302
301
|
history_size = self.history_file.stat().st_size if self.history_file.exists() else 0
|
|
@@ -5,7 +5,6 @@ Coordinates job scheduling, execution, monitoring, and persistence.
|
|
|
5
5
|
Provides the primary interface for the cron scheduling system.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
import asyncio
|
|
9
8
|
import json
|
|
10
9
|
import os
|
|
11
10
|
import signal
|
|
@@ -13,7 +12,7 @@ import subprocess
|
|
|
13
12
|
import threading
|
|
14
13
|
import time
|
|
15
14
|
from datetime import datetime, timedelta
|
|
16
|
-
from typing import Any,
|
|
15
|
+
from typing import Any, Dict, List, Optional
|
|
17
16
|
|
|
18
17
|
from mcli.lib.logger.logger import get_logger
|
|
19
18
|
|
|
@@ -26,14 +25,14 @@ logger = get_logger(__name__)
|
|
|
26
25
|
|
|
27
26
|
|
|
28
27
|
class JobExecutor:
|
|
29
|
-
"""Handles job execution in separate processes/threads"""
|
|
28
|
+
"""Handles job execution in separate processes/threads."""
|
|
30
29
|
|
|
31
30
|
def __init__(self):
|
|
32
31
|
self.running_processes: Dict[str, subprocess.Popen] = {}
|
|
33
32
|
self.lock = threading.Lock()
|
|
34
33
|
|
|
35
34
|
def execute_job(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
36
|
-
"""Execute a job and return execution results"""
|
|
35
|
+
"""Execute a job and return execution results."""
|
|
37
36
|
start_time = datetime.now()
|
|
38
37
|
result = {
|
|
39
38
|
"job_id": job.id,
|
|
@@ -85,7 +84,7 @@ class JobExecutor:
|
|
|
85
84
|
return result
|
|
86
85
|
|
|
87
86
|
def _execute_command(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
88
|
-
"""Execute shell command"""
|
|
87
|
+
"""Execute shell command."""
|
|
89
88
|
env = os.environ.copy()
|
|
90
89
|
env.update(job.environment)
|
|
91
90
|
|
|
@@ -119,7 +118,7 @@ class JobExecutor:
|
|
|
119
118
|
self.running_processes.pop(job.id, None)
|
|
120
119
|
|
|
121
120
|
def _execute_python(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
122
|
-
"""Execute Python code"""
|
|
121
|
+
"""Execute Python code."""
|
|
123
122
|
try:
|
|
124
123
|
# Create temporary Python file
|
|
125
124
|
import tempfile
|
|
@@ -156,7 +155,7 @@ class JobExecutor:
|
|
|
156
155
|
}
|
|
157
156
|
|
|
158
157
|
def _execute_cleanup(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
159
|
-
"""Execute file system cleanup tasks"""
|
|
158
|
+
"""Execute file system cleanup tasks."""
|
|
160
159
|
try:
|
|
161
160
|
# Parse cleanup command (JSON format expected)
|
|
162
161
|
cleanup_config = json.loads(job.command)
|
|
@@ -189,12 +188,12 @@ class JobExecutor:
|
|
|
189
188
|
}
|
|
190
189
|
|
|
191
190
|
def _execute_system(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
192
|
-
"""Execute system maintenance tasks"""
|
|
191
|
+
"""Execute system maintenance tasks."""
|
|
193
192
|
# Similar to cleanup but for system-level tasks
|
|
194
193
|
return self._execute_command(job)
|
|
195
194
|
|
|
196
195
|
def _execute_api_call(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
197
|
-
"""Execute HTTP API calls"""
|
|
196
|
+
"""Execute HTTP API calls."""
|
|
198
197
|
try:
|
|
199
198
|
import requests
|
|
200
199
|
|
|
@@ -237,12 +236,12 @@ class JobExecutor:
|
|
|
237
236
|
}
|
|
238
237
|
|
|
239
238
|
def _execute_custom(self, job: ScheduledJob) -> Dict[str, Any]:
|
|
240
|
-
"""Execute custom job types"""
|
|
239
|
+
"""Execute custom job types."""
|
|
241
240
|
# Default to command execution
|
|
242
241
|
return self._execute_command(job)
|
|
243
242
|
|
|
244
243
|
def _cleanup_old_files(self, path: str, days: int, pattern: str) -> Dict[str, Any]:
|
|
245
|
-
"""Clean up old files in a directory"""
|
|
244
|
+
"""Clean up old files in a directory."""
|
|
246
245
|
try:
|
|
247
246
|
import glob
|
|
248
247
|
from pathlib import Path
|
|
@@ -269,7 +268,7 @@ class JobExecutor:
|
|
|
269
268
|
return {"task": "delete_old_files", "error": str(e)}
|
|
270
269
|
|
|
271
270
|
def _empty_trash(self) -> Dict[str, Any]:
|
|
272
|
-
"""Empty system trash/recycle bin"""
|
|
271
|
+
"""Empty system trash/recycle bin."""
|
|
273
272
|
try:
|
|
274
273
|
import platform
|
|
275
274
|
|
|
@@ -295,7 +294,7 @@ class JobExecutor:
|
|
|
295
294
|
return {"task": "empty_trash", "error": str(e)}
|
|
296
295
|
|
|
297
296
|
def _organize_desktop(self) -> Dict[str, Any]:
|
|
298
|
-
"""Organize desktop files into folders"""
|
|
297
|
+
"""Organize desktop files into folders."""
|
|
299
298
|
try:
|
|
300
299
|
desktop_path = os.path.join(os.path.expanduser("~"), "Desktop")
|
|
301
300
|
if not os.path.exists(desktop_path):
|
|
@@ -303,8 +302,8 @@ class JobExecutor:
|
|
|
303
302
|
|
|
304
303
|
organized_files = []
|
|
305
304
|
file_types = {
|
|
306
|
-
"Documents": [".
|
|
307
|
-
"Images": [".jpg", ".jpeg", ".png", ".
|
|
305
|
+
"Documents": [".pd", ".doc", ".docx", ".txt", ".rt"],
|
|
306
|
+
"Images": [".jpg", ".jpeg", ".png", ".gi", ".bmp", ".svg"],
|
|
308
307
|
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz"],
|
|
309
308
|
"Videos": [".mp4", ".avi", ".mov", ".mkv", ".wmv"],
|
|
310
309
|
"Audio": [".mp3", ".wav", ".flac", ".aac", ".ogg"],
|
|
@@ -335,7 +334,7 @@ class JobExecutor:
|
|
|
335
334
|
return {"task": "organize_desktop", "error": str(e)}
|
|
336
335
|
|
|
337
336
|
def kill_job(self, job_id: str) -> bool:
|
|
338
|
-
"""Kill a running job process"""
|
|
337
|
+
"""Kill a running job process."""
|
|
339
338
|
with self.lock:
|
|
340
339
|
process = self.running_processes.get(job_id)
|
|
341
340
|
if process and process.poll() is None:
|
|
@@ -352,7 +351,7 @@ class JobExecutor:
|
|
|
352
351
|
|
|
353
352
|
|
|
354
353
|
class JobScheduler:
|
|
355
|
-
"""Main scheduler that coordinates all cron functionality"""
|
|
354
|
+
"""Main scheduler that coordinates all cron functionality."""
|
|
356
355
|
|
|
357
356
|
def __init__(self, storage_dir: Optional[str] = None):
|
|
358
357
|
self.storage = JobStorage(storage_dir)
|
|
@@ -372,18 +371,18 @@ class JobScheduler:
|
|
|
372
371
|
signal.signal(signal.SIGTERM, self._signal_handler)
|
|
373
372
|
|
|
374
373
|
def _load_jobs(self):
|
|
375
|
-
"""Load jobs from persistent storage"""
|
|
374
|
+
"""Load jobs from persistent storage."""
|
|
376
375
|
jobs = self.storage.load_jobs()
|
|
377
376
|
self.jobs = {job.id: job for job in jobs}
|
|
378
377
|
logger.info(f"Loaded {len(self.jobs)} jobs from storage")
|
|
379
378
|
|
|
380
379
|
def _save_jobs(self):
|
|
381
|
-
"""Save all jobs to persistent storage"""
|
|
380
|
+
"""Save all jobs to persistent storage."""
|
|
382
381
|
jobs_list = list(self.jobs.values())
|
|
383
382
|
self.storage.save_jobs(jobs_list)
|
|
384
383
|
|
|
385
384
|
def start(self):
|
|
386
|
-
"""Start the scheduler"""
|
|
385
|
+
"""Start the scheduler."""
|
|
387
386
|
if self.running:
|
|
388
387
|
logger.warning("Scheduler already running")
|
|
389
388
|
return
|
|
@@ -401,7 +400,7 @@ class JobScheduler:
|
|
|
401
400
|
logger.info("Job scheduler started")
|
|
402
401
|
|
|
403
402
|
def stop(self):
|
|
404
|
-
"""Stop the scheduler"""
|
|
403
|
+
"""Stop the scheduler."""
|
|
405
404
|
if not self.running:
|
|
406
405
|
return
|
|
407
406
|
|
|
@@ -417,12 +416,12 @@ class JobScheduler:
|
|
|
417
416
|
logger.info("Job scheduler stopped")
|
|
418
417
|
|
|
419
418
|
def _signal_handler(self, signum, frame):
|
|
420
|
-
"""Handle system signals"""
|
|
419
|
+
"""Handle system signals."""
|
|
421
420
|
logger.info(f"Received signal {signum}, shutting down scheduler...")
|
|
422
421
|
self.stop()
|
|
423
422
|
|
|
424
423
|
def _scheduler_loop(self):
|
|
425
|
-
"""Main scheduling loop"""
|
|
424
|
+
"""Main scheduling loop."""
|
|
426
425
|
while self.running:
|
|
427
426
|
try:
|
|
428
427
|
current_time = datetime.now()
|
|
@@ -455,7 +454,7 @@ class JobScheduler:
|
|
|
455
454
|
time.sleep(60) # Wait longer on error
|
|
456
455
|
|
|
457
456
|
def _should_run_job(self, job: ScheduledJob, current_time: datetime) -> bool:
|
|
458
|
-
"""Check if a job should run at the current time"""
|
|
457
|
+
"""Check if a job should run at the current time."""
|
|
459
458
|
if job.status == JobStatus.RUNNING:
|
|
460
459
|
return False
|
|
461
460
|
|
|
@@ -478,7 +477,7 @@ class JobScheduler:
|
|
|
478
477
|
return False
|
|
479
478
|
|
|
480
479
|
def _queue_job_execution(self, job: ScheduledJob):
|
|
481
|
-
"""Queue a job for execution"""
|
|
480
|
+
"""Queue a job for execution."""
|
|
482
481
|
|
|
483
482
|
def execute_job_thread():
|
|
484
483
|
try:
|
|
@@ -505,7 +504,7 @@ class JobScheduler:
|
|
|
505
504
|
self.monitor.add_job(job, thread)
|
|
506
505
|
|
|
507
506
|
def _update_job_next_run(self, job: ScheduledJob):
|
|
508
|
-
"""Update job's next run time"""
|
|
507
|
+
"""Update job's next run time."""
|
|
509
508
|
try:
|
|
510
509
|
cron = CronExpression(job.cron_expression)
|
|
511
510
|
if not cron.is_reboot:
|
|
@@ -514,13 +513,13 @@ class JobScheduler:
|
|
|
514
513
|
logger.error(f"Error updating next run time for {job.name}: {e}")
|
|
515
514
|
|
|
516
515
|
def _update_next_run_times(self):
|
|
517
|
-
"""Update next run times for all jobs"""
|
|
516
|
+
"""Update next run times for all jobs."""
|
|
518
517
|
for job in self.jobs.values():
|
|
519
518
|
if job.enabled and job.next_run is None:
|
|
520
519
|
self._update_job_next_run(job)
|
|
521
520
|
|
|
522
521
|
def _execute_reboot_jobs(self):
|
|
523
|
-
"""Execute jobs marked with @reboot"""
|
|
522
|
+
"""Execute jobs marked with @reboot."""
|
|
524
523
|
reboot_jobs = [
|
|
525
524
|
job
|
|
526
525
|
for job in self.jobs.values()
|
|
@@ -534,7 +533,7 @@ class JobScheduler:
|
|
|
534
533
|
# Public API methods
|
|
535
534
|
|
|
536
535
|
def add_job(self, job: ScheduledJob) -> bool:
|
|
537
|
-
"""Add a new job to the scheduler"""
|
|
536
|
+
"""Add a new job to the scheduler."""
|
|
538
537
|
try:
|
|
539
538
|
with self.lock:
|
|
540
539
|
self.jobs[job.id] = job
|
|
@@ -549,7 +548,7 @@ class JobScheduler:
|
|
|
549
548
|
return False
|
|
550
549
|
|
|
551
550
|
def remove_job(self, job_id: str) -> bool:
|
|
552
|
-
"""Remove a job from the scheduler"""
|
|
551
|
+
"""Remove a job from the scheduler."""
|
|
553
552
|
try:
|
|
554
553
|
with self.lock:
|
|
555
554
|
job = self.jobs.pop(job_id, None)
|
|
@@ -567,15 +566,15 @@ class JobScheduler:
|
|
|
567
566
|
return False
|
|
568
567
|
|
|
569
568
|
def get_job(self, job_id: str) -> Optional[ScheduledJob]:
|
|
570
|
-
"""Get a job by ID"""
|
|
569
|
+
"""Get a job by ID."""
|
|
571
570
|
return self.jobs.get(job_id)
|
|
572
571
|
|
|
573
572
|
def get_all_jobs(self) -> List[ScheduledJob]:
|
|
574
|
-
"""Get all jobs"""
|
|
573
|
+
"""Get all jobs."""
|
|
575
574
|
return list(self.jobs.values())
|
|
576
575
|
|
|
577
576
|
def get_job_status(self, job_id: str) -> Optional[Dict[str, Any]]:
|
|
578
|
-
"""Get detailed status of a job"""
|
|
577
|
+
"""Get detailed status of a job."""
|
|
579
578
|
job = self.jobs.get(job_id)
|
|
580
579
|
if not job:
|
|
581
580
|
return None
|
|
@@ -588,7 +587,7 @@ class JobScheduler:
|
|
|
588
587
|
}
|
|
589
588
|
|
|
590
589
|
def get_scheduler_stats(self) -> Dict[str, Any]:
|
|
591
|
-
"""Get scheduler statistics"""
|
|
590
|
+
"""Get scheduler statistics."""
|
|
592
591
|
total_jobs = len(self.jobs)
|
|
593
592
|
enabled_jobs = len([j for j in self.jobs.values() if j.enabled])
|
|
594
593
|
running_jobs = len(self.monitor.get_running_jobs())
|
|
@@ -603,7 +602,7 @@ class JobScheduler:
|
|
|
603
602
|
}
|
|
604
603
|
|
|
605
604
|
def create_json_response(self) -> Dict[str, Any]:
|
|
606
|
-
"""Create JSON response for frontend integration"""
|
|
605
|
+
"""Create JSON response for frontend integration."""
|
|
607
606
|
jobs_data = []
|
|
608
607
|
for job in self.jobs.values():
|
|
609
608
|
job_data = job.to_dict()
|
|
@@ -626,7 +625,7 @@ def create_desktop_cleanup_job(
|
|
|
626
625
|
cron_expression: str = "0 9 * * 1", # Monday 9 AM
|
|
627
626
|
enabled: bool = True,
|
|
628
627
|
) -> ScheduledJob:
|
|
629
|
-
"""Create a job to organize desktop files"""
|
|
628
|
+
"""Create a job to organize desktop files."""
|
|
630
629
|
cleanup_config = {"tasks": [{"type": "organize_desktop"}]}
|
|
631
630
|
|
|
632
631
|
return ScheduledJob(
|
|
@@ -646,7 +645,7 @@ def create_temp_cleanup_job(
|
|
|
646
645
|
days: int = 7,
|
|
647
646
|
enabled: bool = True,
|
|
648
647
|
) -> ScheduledJob:
|
|
649
|
-
"""Create a job to clean up old temporary files"""
|
|
648
|
+
"""Create a job to clean up old temporary files."""
|
|
650
649
|
cleanup_config = {
|
|
651
650
|
"tasks": [{"type": "delete_old_files", "path": temp_path, "days": days, "pattern": "*"}]
|
|
652
651
|
}
|
|
@@ -667,7 +666,7 @@ def create_system_backup_job(
|
|
|
667
666
|
backup_command: str = "rsync -av /home/user/ /backup/",
|
|
668
667
|
enabled: bool = True,
|
|
669
668
|
) -> ScheduledJob:
|
|
670
|
-
"""Create a system backup job"""
|
|
669
|
+
"""Create a system backup job."""
|
|
671
670
|
return ScheduledJob(
|
|
672
671
|
name=name,
|
|
673
672
|
cron_expression=cron_expression,
|
mcli/workflow/sync/test_cmd.py
CHANGED
|
@@ -31,20 +31,18 @@ C3LI_UNAME = os.environ.get("C3LI_UNAME")
|
|
|
31
31
|
# TODO: To implement / integrate ReactJS version of c3 packages
|
|
32
32
|
@click.group(name="ui")
|
|
33
33
|
def bundle():
|
|
34
|
-
"""ui utility - use this to interact with c3 ui components"""
|
|
35
|
-
pass
|
|
34
|
+
"""ui utility - use this to interact with c3 ui components."""
|
|
36
35
|
|
|
37
36
|
|
|
38
37
|
@click.command(name="provision")
|
|
39
38
|
def provision():
|
|
40
|
-
"""provision utility - use this to provision your c3 package"""
|
|
41
|
-
pass
|
|
39
|
+
"""provision utility - use this to provision your c3 package."""
|
|
42
40
|
|
|
43
41
|
|
|
44
42
|
@click.command(name="v8")
|
|
45
43
|
@click.option("--interactive", "interactive", flag_value=True, default=False)
|
|
46
44
|
def v8(interactive):
|
|
47
|
-
"""bundle utility - use this to bundle your c3 package"""
|
|
45
|
+
"""bundle utility - use this to bundle your c3 package."""
|
|
48
46
|
if interactive:
|
|
49
47
|
pass # logger.info("Bundling in interactive mode")
|
|
50
48
|
else:
|
|
@@ -55,21 +53,19 @@ def v8(interactive):
|
|
|
55
53
|
@click.command(name="v7")
|
|
56
54
|
@click.option("--interactive", "interactive", flag_value=True, default=False)
|
|
57
55
|
def v7(interactive):
|
|
58
|
-
"""bundle utility - use this to bundle your c3 package"""
|
|
56
|
+
"""bundle utility - use this to bundle your c3 package."""
|
|
59
57
|
if interactive:
|
|
60
58
|
pass # logger.info("Bundling in interactive mode")
|
|
61
|
-
pass
|
|
62
59
|
|
|
63
60
|
|
|
64
61
|
@click.command(name="sync")
|
|
65
62
|
def sync():
|
|
66
|
-
"""sync utility - use this to sync your c3 package"""
|
|
63
|
+
"""sync utility - use this to sync your c3 package."""
|
|
67
64
|
if hasattr(watcher, "watch"):
|
|
68
65
|
watcher.watch(C3LI_PACKAGES_TO_SYNC, C3LI_PATH_TO_PACKAGE_REPO)
|
|
69
66
|
else:
|
|
70
67
|
# Dummy fallback for test pass
|
|
71
68
|
pass
|
|
72
|
-
pass
|
|
73
69
|
|
|
74
70
|
|
|
75
71
|
bundle.add_command(provision)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mcli-framework
|
|
3
|
-
Version: 7.12.
|
|
3
|
+
Version: 7.12.3
|
|
4
4
|
Summary: Portable workflow framework - transform any script into a versioned, schedulable command. Store in ~/.mcli/workflows/, version with lockfile, run as daemon or cron job.
|
|
5
5
|
Author-email: Luis Fernandez de la Vara <luis@lefv.io>
|
|
6
6
|
Maintainer-email: Luis Fernandez de la Vara <luis@lefv.io>
|