mcli-framework 7.1.3__py3-none-any.whl → 7.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mcli-framework might be problematic. Click here for more details.
- mcli/__init__.py +160 -0
- mcli/__main__.py +14 -0
- mcli/app/__init__.py +23 -0
- mcli/app/main.py +10 -0
- mcli/app/model/__init__.py +0 -0
- mcli/app/video/__init__.py +5 -0
- mcli/chat/__init__.py +34 -0
- mcli/lib/__init__.py +0 -0
- mcli/lib/api/__init__.py +0 -0
- mcli/lib/auth/__init__.py +1 -0
- mcli/lib/config/__init__.py +1 -0
- mcli/lib/custom_commands.py +424 -0
- mcli/lib/erd/__init__.py +25 -0
- mcli/lib/files/__init__.py +0 -0
- mcli/lib/fs/__init__.py +1 -0
- mcli/lib/logger/__init__.py +3 -0
- mcli/lib/paths.py +12 -0
- mcli/lib/performance/__init__.py +17 -0
- mcli/lib/pickles/__init__.py +1 -0
- mcli/lib/shell/__init__.py +0 -0
- mcli/lib/toml/__init__.py +1 -0
- mcli/lib/watcher/__init__.py +0 -0
- mcli/ml/__init__.py +16 -0
- mcli/ml/api/__init__.py +30 -0
- mcli/ml/api/routers/__init__.py +27 -0
- mcli/ml/api/schemas.py +2 -2
- mcli/ml/auth/__init__.py +45 -0
- mcli/ml/auth/models.py +2 -2
- mcli/ml/backtesting/__init__.py +39 -0
- mcli/ml/cli/__init__.py +5 -0
- mcli/ml/cli/main.py +1 -1
- mcli/ml/config/__init__.py +33 -0
- mcli/ml/configs/__init__.py +16 -0
- mcli/ml/dashboard/__init__.py +12 -0
- mcli/ml/dashboard/app.py +13 -13
- mcli/ml/dashboard/app_integrated.py +1309 -148
- mcli/ml/dashboard/app_supabase.py +46 -21
- mcli/ml/dashboard/app_training.py +14 -14
- mcli/ml/dashboard/components/__init__.py +7 -0
- mcli/ml/dashboard/components/charts.py +258 -0
- mcli/ml/dashboard/components/metrics.py +125 -0
- mcli/ml/dashboard/components/tables.py +228 -0
- mcli/ml/dashboard/pages/__init__.py +6 -0
- mcli/ml/dashboard/pages/cicd.py +382 -0
- mcli/ml/dashboard/pages/predictions_enhanced.py +834 -0
- mcli/ml/dashboard/pages/scrapers_and_logs.py +1060 -0
- mcli/ml/dashboard/pages/test_portfolio.py +373 -0
- mcli/ml/dashboard/pages/trading.py +714 -0
- mcli/ml/dashboard/pages/workflows.py +533 -0
- mcli/ml/dashboard/utils.py +154 -0
- mcli/ml/data_ingestion/__init__.py +39 -0
- mcli/ml/database/__init__.py +47 -0
- mcli/ml/experimentation/__init__.py +29 -0
- mcli/ml/features/__init__.py +39 -0
- mcli/ml/mlops/__init__.py +33 -0
- mcli/ml/models/__init__.py +94 -0
- mcli/ml/monitoring/__init__.py +25 -0
- mcli/ml/optimization/__init__.py +27 -0
- mcli/ml/predictions/__init__.py +5 -0
- mcli/ml/preprocessing/__init__.py +28 -0
- mcli/ml/scripts/__init__.py +1 -0
- mcli/ml/trading/__init__.py +60 -0
- mcli/ml/trading/alpaca_client.py +353 -0
- mcli/ml/trading/migrations.py +164 -0
- mcli/ml/trading/models.py +418 -0
- mcli/ml/trading/paper_trading.py +326 -0
- mcli/ml/trading/risk_management.py +370 -0
- mcli/ml/trading/trading_service.py +480 -0
- mcli/ml/training/__init__.py +10 -0
- mcli/ml/training/train_model.py +569 -0
- mcli/mygroup/__init__.py +3 -0
- mcli/public/__init__.py +1 -0
- mcli/public/commands/__init__.py +2 -0
- mcli/self/__init__.py +3 -0
- mcli/self/self_cmd.py +579 -91
- mcli/workflow/__init__.py +0 -0
- mcli/workflow/daemon/__init__.py +15 -0
- mcli/workflow/daemon/daemon.py +21 -3
- mcli/workflow/dashboard/__init__.py +5 -0
- mcli/workflow/docker/__init__.py +0 -0
- mcli/workflow/file/__init__.py +0 -0
- mcli/workflow/gcloud/__init__.py +1 -0
- mcli/workflow/git_commit/__init__.py +0 -0
- mcli/workflow/interview/__init__.py +0 -0
- mcli/workflow/politician_trading/__init__.py +4 -0
- mcli/workflow/politician_trading/data_sources.py +259 -1
- mcli/workflow/politician_trading/models.py +159 -1
- mcli/workflow/politician_trading/scrapers_corporate_registry.py +846 -0
- mcli/workflow/politician_trading/scrapers_free_sources.py +516 -0
- mcli/workflow/politician_trading/scrapers_third_party.py +391 -0
- mcli/workflow/politician_trading/seed_database.py +539 -0
- mcli/workflow/registry/__init__.py +0 -0
- mcli/workflow/repo/__init__.py +0 -0
- mcli/workflow/scheduler/__init__.py +25 -0
- mcli/workflow/search/__init__.py +0 -0
- mcli/workflow/sync/__init__.py +5 -0
- mcli/workflow/videos/__init__.py +1 -0
- mcli/workflow/wakatime/__init__.py +80 -0
- mcli/workflow/workflow.py +8 -27
- {mcli_framework-7.1.3.dist-info → mcli_framework-7.3.1.dist-info}/METADATA +3 -1
- {mcli_framework-7.1.3.dist-info → mcli_framework-7.3.1.dist-info}/RECORD +105 -29
- mcli/workflow/daemon/api_daemon.py +0 -800
- mcli/workflow/daemon/commands.py +0 -1196
- mcli/workflow/dashboard/dashboard_cmd.py +0 -120
- mcli/workflow/file/file.py +0 -100
- mcli/workflow/git_commit/commands.py +0 -430
- mcli/workflow/politician_trading/commands.py +0 -1939
- mcli/workflow/scheduler/commands.py +0 -493
- mcli/workflow/sync/sync_cmd.py +0 -437
- mcli/workflow/videos/videos.py +0 -242
- {mcli_framework-7.1.3.dist-info → mcli_framework-7.3.1.dist-info}/WHEEL +0 -0
- {mcli_framework-7.1.3.dist-info → mcli_framework-7.3.1.dist-info}/entry_points.txt +0 -0
- {mcli_framework-7.1.3.dist-info → mcli_framework-7.3.1.dist-info}/licenses/LICENSE +0 -0
- {mcli_framework-7.1.3.dist-info → mcli_framework-7.3.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
"""Workflow Management Dashboard"""
|
|
2
|
+
|
|
3
|
+
import streamlit as st
|
|
4
|
+
import pandas as pd
|
|
5
|
+
import requests
|
|
6
|
+
import os
|
|
7
|
+
import json
|
|
8
|
+
from datetime import datetime, timedelta
|
|
9
|
+
import plotly.express as px
|
|
10
|
+
from typing import Optional, Dict, Any
|
|
11
|
+
|
|
12
|
+
# Import components
|
|
13
|
+
try:
|
|
14
|
+
from ..components.metrics import display_kpi_row, display_status_badge, display_health_indicator
|
|
15
|
+
from ..components.charts import create_timeline_chart, create_status_pie_chart, create_gantt_chart, render_chart
|
|
16
|
+
from ..components.tables import display_filterable_dataframe, export_dataframe
|
|
17
|
+
except ImportError:
|
|
18
|
+
# Fallback for when imported outside package context
|
|
19
|
+
from components.metrics import display_kpi_row, display_status_badge, display_health_indicator
|
|
20
|
+
from components.charts import create_timeline_chart, create_status_pie_chart, create_gantt_chart, render_chart
|
|
21
|
+
from components.tables import display_filterable_dataframe, export_dataframe
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_workflow_api_url() -> str:
|
|
25
|
+
"""Get Workflow API URL from environment"""
|
|
26
|
+
lsh_url = os.getenv("LSH_API_URL", "http://localhost:3034")
|
|
27
|
+
return f"{lsh_url}/api/workflows"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def fetch_workflows() -> pd.DataFrame:
|
|
31
|
+
"""Fetch workflow definitions from API"""
|
|
32
|
+
try:
|
|
33
|
+
api_url = get_workflow_api_url()
|
|
34
|
+
response = requests.get(api_url, timeout=5)
|
|
35
|
+
response.raise_for_status()
|
|
36
|
+
|
|
37
|
+
workflows = response.json().get("workflows", [])
|
|
38
|
+
if workflows:
|
|
39
|
+
return pd.DataFrame(workflows)
|
|
40
|
+
|
|
41
|
+
except requests.exceptions.HTTPError as e:
|
|
42
|
+
if e.response.status_code == 404:
|
|
43
|
+
# API endpoint not implemented yet - use demo data silently
|
|
44
|
+
pass
|
|
45
|
+
else:
|
|
46
|
+
st.warning(f"Could not fetch workflow data: {e}")
|
|
47
|
+
except requests.exceptions.ConnectionError:
|
|
48
|
+
st.warning("⚠️ LSH Daemon connection failed. Using demo data.")
|
|
49
|
+
except Exception as e:
|
|
50
|
+
# Only show warning for unexpected errors
|
|
51
|
+
st.warning(f"Could not fetch workflow data: {e}")
|
|
52
|
+
|
|
53
|
+
# Return mock data
|
|
54
|
+
return create_mock_workflow_data()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def create_mock_workflow_data() -> pd.DataFrame:
|
|
58
|
+
"""Create mock workflow data for demonstration"""
|
|
59
|
+
workflows = [
|
|
60
|
+
{
|
|
61
|
+
"id": "wf-1",
|
|
62
|
+
"name": "Data Ingestion Pipeline",
|
|
63
|
+
"description": "Ingest politician trading data from multiple sources",
|
|
64
|
+
"status": "active",
|
|
65
|
+
"schedule": "0 */6 * * *", # Every 6 hours
|
|
66
|
+
"last_run": (datetime.now() - timedelta(hours=3)).isoformat(),
|
|
67
|
+
"next_run": (datetime.now() + timedelta(hours=3)).isoformat(),
|
|
68
|
+
"success_rate": 0.95,
|
|
69
|
+
"avg_duration_min": 12,
|
|
70
|
+
"total_runs": 150
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"id": "wf-2",
|
|
74
|
+
"name": "ML Model Training",
|
|
75
|
+
"description": "Train and evaluate ML models on latest data",
|
|
76
|
+
"status": "active",
|
|
77
|
+
"schedule": "0 2 * * *", # Daily at 2 AM
|
|
78
|
+
"last_run": (datetime.now() - timedelta(days=1, hours=2)).isoformat(),
|
|
79
|
+
"next_run": (datetime.now() + timedelta(hours=22)).isoformat(),
|
|
80
|
+
"success_rate": 0.88,
|
|
81
|
+
"avg_duration_min": 45,
|
|
82
|
+
"total_runs": 30
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "wf-3",
|
|
86
|
+
"name": "Data Validation & Quality Check",
|
|
87
|
+
"description": "Validate data integrity and quality metrics",
|
|
88
|
+
"status": "active",
|
|
89
|
+
"schedule": "0 * * * *", # Hourly
|
|
90
|
+
"last_run": (datetime.now() - timedelta(minutes=30)).isoformat(),
|
|
91
|
+
"next_run": (datetime.now() + timedelta(minutes=30)).isoformat(),
|
|
92
|
+
"success_rate": 1.0,
|
|
93
|
+
"avg_duration_min": 5,
|
|
94
|
+
"total_runs": 500
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"id": "wf-4",
|
|
98
|
+
"name": "Prediction Generation",
|
|
99
|
+
"description": "Generate stock recommendations based on latest models",
|
|
100
|
+
"status": "paused",
|
|
101
|
+
"schedule": "0 9 * * 1-5", # Weekdays at 9 AM
|
|
102
|
+
"last_run": (datetime.now() - timedelta(days=3)).isoformat(),
|
|
103
|
+
"next_run": None,
|
|
104
|
+
"success_rate": 0.92,
|
|
105
|
+
"avg_duration_min": 20,
|
|
106
|
+
"total_runs": 75
|
|
107
|
+
}
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
return pd.DataFrame(workflows)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def fetch_workflow_executions(workflow_id: Optional[str] = None) -> pd.DataFrame:
|
|
114
|
+
"""Fetch workflow execution history"""
|
|
115
|
+
try:
|
|
116
|
+
api_url = get_workflow_api_url()
|
|
117
|
+
url = f"{api_url}/{workflow_id}/executions" if workflow_id else f"{api_url}/executions"
|
|
118
|
+
response = requests.get(url, timeout=5)
|
|
119
|
+
response.raise_for_status()
|
|
120
|
+
|
|
121
|
+
executions = response.json().get("executions", [])
|
|
122
|
+
if executions:
|
|
123
|
+
return pd.DataFrame(executions)
|
|
124
|
+
|
|
125
|
+
except requests.exceptions.HTTPError as e:
|
|
126
|
+
if e.response.status_code == 404:
|
|
127
|
+
# API endpoint not implemented yet - use demo data silently
|
|
128
|
+
pass
|
|
129
|
+
else:
|
|
130
|
+
st.warning(f"Could not fetch execution data: {e}")
|
|
131
|
+
except requests.exceptions.ConnectionError:
|
|
132
|
+
st.warning("⚠️ LSH Daemon connection failed. Using demo data.")
|
|
133
|
+
except Exception as e:
|
|
134
|
+
st.warning(f"Could not fetch execution data: {e}")
|
|
135
|
+
|
|
136
|
+
# Return mock execution data
|
|
137
|
+
return create_mock_execution_data(workflow_id)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def create_mock_execution_data(workflow_id: Optional[str] = None) -> pd.DataFrame:
|
|
141
|
+
"""Create mock execution data"""
|
|
142
|
+
import random
|
|
143
|
+
|
|
144
|
+
executions = []
|
|
145
|
+
for i in range(50):
|
|
146
|
+
start_time = datetime.now() - timedelta(days=random.randint(0, 30), hours=random.randint(0, 23))
|
|
147
|
+
duration = random.randint(300, 3600) # 5-60 minutes in seconds
|
|
148
|
+
status = random.choices(["completed", "failed", "running"], weights=[80, 15, 5])[0]
|
|
149
|
+
|
|
150
|
+
executions.append({
|
|
151
|
+
"id": f"exec-{i+1}",
|
|
152
|
+
"workflow_id": workflow_id or f"wf-{random.randint(1,4)}",
|
|
153
|
+
"workflow_name": random.choice(["Data Ingestion Pipeline", "ML Model Training", "Data Validation", "Prediction Generation"]),
|
|
154
|
+
"status": status,
|
|
155
|
+
"started_at": start_time.isoformat(),
|
|
156
|
+
"completed_at": (start_time + timedelta(seconds=duration)).isoformat() if status != "running" else None,
|
|
157
|
+
"duration_sec": duration if status != "running" else None,
|
|
158
|
+
"triggered_by": random.choice(["schedule", "manual", "api"]),
|
|
159
|
+
"steps_completed": random.randint(3, 8),
|
|
160
|
+
"steps_total": 8
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
return pd.DataFrame(executions)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def show_workflows_dashboard():
|
|
167
|
+
"""Main workflow management dashboard"""
|
|
168
|
+
|
|
169
|
+
st.title("⚙️ Workflow Management")
|
|
170
|
+
st.markdown("Create, schedule, and monitor data pipeline workflows")
|
|
171
|
+
|
|
172
|
+
# Refresh button
|
|
173
|
+
col1, col2 = st.columns([1, 9])
|
|
174
|
+
with col1:
|
|
175
|
+
if st.button("🔄 Refresh"):
|
|
176
|
+
st.rerun()
|
|
177
|
+
|
|
178
|
+
st.divider()
|
|
179
|
+
|
|
180
|
+
# Fetch data
|
|
181
|
+
with st.spinner("Loading workflow data..."):
|
|
182
|
+
workflows_df = fetch_workflows()
|
|
183
|
+
|
|
184
|
+
if workflows_df.empty:
|
|
185
|
+
st.warning("No workflows found")
|
|
186
|
+
return
|
|
187
|
+
|
|
188
|
+
# Convert timestamps
|
|
189
|
+
for col in ["last_run", "next_run"]:
|
|
190
|
+
if col in workflows_df.columns:
|
|
191
|
+
workflows_df[col] = pd.to_datetime(workflows_df[col], errors='coerce')
|
|
192
|
+
|
|
193
|
+
# === KPIs ===
|
|
194
|
+
st.subheader("📊 Workflow Metrics")
|
|
195
|
+
|
|
196
|
+
total_workflows = len(workflows_df)
|
|
197
|
+
active_workflows = len(workflows_df[workflows_df["status"] == "active"])
|
|
198
|
+
paused_workflows = len(workflows_df[workflows_df["status"] == "paused"])
|
|
199
|
+
avg_success_rate = workflows_df["success_rate"].mean() * 100 if "success_rate" in workflows_df.columns else 0
|
|
200
|
+
|
|
201
|
+
metrics = {
|
|
202
|
+
"Total Workflows": {"value": total_workflows, "icon": "⚙️"},
|
|
203
|
+
"Active": {"value": active_workflows, "icon": "✅"},
|
|
204
|
+
"Paused": {"value": paused_workflows, "icon": "⏸️"},
|
|
205
|
+
"Avg Success Rate": {"value": f"{avg_success_rate:.1f}%", "icon": "📈"},
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
display_kpi_row(metrics, columns=4)
|
|
209
|
+
|
|
210
|
+
st.divider()
|
|
211
|
+
|
|
212
|
+
# === Tabs ===
|
|
213
|
+
tab1, tab2, tab3, tab4 = st.tabs(["📋 Workflows", "📈 Executions", "➕ Create Workflow", "📚 Templates"])
|
|
214
|
+
|
|
215
|
+
with tab1:
|
|
216
|
+
show_workflow_list(workflows_df)
|
|
217
|
+
|
|
218
|
+
with tab2:
|
|
219
|
+
show_workflow_executions()
|
|
220
|
+
|
|
221
|
+
with tab3:
|
|
222
|
+
show_workflow_builder()
|
|
223
|
+
|
|
224
|
+
with tab4:
|
|
225
|
+
show_workflow_templates()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def show_workflow_list(workflows_df: pd.DataFrame):
|
|
229
|
+
"""Display list of workflows"""
|
|
230
|
+
|
|
231
|
+
st.markdown("### Active Workflows")
|
|
232
|
+
|
|
233
|
+
# Filter options
|
|
234
|
+
filter_config = {
|
|
235
|
+
"status": "multiselect",
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
filtered_df = display_filterable_dataframe(
|
|
239
|
+
workflows_df,
|
|
240
|
+
filter_columns=filter_config,
|
|
241
|
+
key_prefix="workflow_filter"
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
# Workflow details
|
|
245
|
+
for _, workflow in filtered_df.iterrows():
|
|
246
|
+
with st.expander(f"{workflow['name']} - {display_status_badge(workflow['status'], 'small')}"):
|
|
247
|
+
col1, col2 = st.columns(2)
|
|
248
|
+
|
|
249
|
+
with col1:
|
|
250
|
+
st.markdown(f"**Description:** {workflow.get('description', 'N/A')}")
|
|
251
|
+
st.markdown(f"**Schedule:** `{workflow.get('schedule', 'N/A')}`")
|
|
252
|
+
st.markdown(f"**Total Runs:** {workflow.get('total_runs', 0)}")
|
|
253
|
+
|
|
254
|
+
with col2:
|
|
255
|
+
st.markdown(f"**Success Rate:** {workflow.get('success_rate', 0) * 100:.1f}%")
|
|
256
|
+
st.markdown(f"**Avg Duration:** {workflow.get('avg_duration_min', 0):.1f} min")
|
|
257
|
+
st.markdown(f"**Last Run:** {workflow.get('last_run', 'Never')}")
|
|
258
|
+
|
|
259
|
+
# Actions
|
|
260
|
+
col_a, col_b, col_c, col_d = st.columns(4)
|
|
261
|
+
|
|
262
|
+
with col_a:
|
|
263
|
+
if st.button("▶️ Run Now", key=f"run_{workflow['id']}"):
|
|
264
|
+
st.success(f"Workflow '{workflow['name']}' triggered!")
|
|
265
|
+
|
|
266
|
+
with col_b:
|
|
267
|
+
if workflow['status'] == "active":
|
|
268
|
+
if st.button("⏸️ Pause", key=f"pause_{workflow['id']}"):
|
|
269
|
+
st.info(f"Workflow '{workflow['name']}' paused")
|
|
270
|
+
else:
|
|
271
|
+
if st.button("▶️ Resume", key=f"resume_{workflow['id']}"):
|
|
272
|
+
st.info(f"Workflow '{workflow['name']}' resumed")
|
|
273
|
+
|
|
274
|
+
with col_c:
|
|
275
|
+
if st.button("✏️ Edit", key=f"edit_{workflow['id']}"):
|
|
276
|
+
st.session_state['edit_workflow_id'] = workflow['id']
|
|
277
|
+
st.info("Edit mode activated")
|
|
278
|
+
|
|
279
|
+
with col_d:
|
|
280
|
+
if st.button("📊 View Executions", key=f"view_exec_{workflow['id']}"):
|
|
281
|
+
st.session_state['selected_workflow'] = workflow['id']
|
|
282
|
+
|
|
283
|
+
# Show workflow definition
|
|
284
|
+
if st.checkbox("Show Workflow Definition", key=f"def_{workflow['id']}"):
|
|
285
|
+
workflow_def = {
|
|
286
|
+
"name": workflow['name'],
|
|
287
|
+
"description": workflow['description'],
|
|
288
|
+
"schedule": workflow['schedule'],
|
|
289
|
+
"steps": [
|
|
290
|
+
{"name": "Fetch Data", "action": "api_call", "params": {}},
|
|
291
|
+
{"name": "Transform Data", "action": "python_script", "params": {}},
|
|
292
|
+
{"name": "Validate Data", "action": "validation", "params": {}},
|
|
293
|
+
{"name": "Store Results", "action": "database_write", "params": {}}
|
|
294
|
+
]
|
|
295
|
+
}
|
|
296
|
+
st.json(workflow_def)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def show_workflow_executions():
|
|
300
|
+
"""Show workflow execution history"""
|
|
301
|
+
|
|
302
|
+
st.markdown("### Workflow Execution History")
|
|
303
|
+
|
|
304
|
+
# Fetch executions
|
|
305
|
+
executions_df = fetch_workflow_executions()
|
|
306
|
+
|
|
307
|
+
if executions_df.empty:
|
|
308
|
+
st.info("No execution history")
|
|
309
|
+
return
|
|
310
|
+
|
|
311
|
+
# Convert timestamps
|
|
312
|
+
for col in ["started_at", "completed_at"]:
|
|
313
|
+
if col in executions_df.columns:
|
|
314
|
+
executions_df[col] = pd.to_datetime(executions_df[col], errors='coerce')
|
|
315
|
+
|
|
316
|
+
# Filter
|
|
317
|
+
filter_config = {
|
|
318
|
+
"workflow_name": "multiselect",
|
|
319
|
+
"status": "multiselect",
|
|
320
|
+
"triggered_by": "multiselect"
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
filtered_df = display_filterable_dataframe(
|
|
324
|
+
executions_df,
|
|
325
|
+
filter_columns=filter_config,
|
|
326
|
+
key_prefix="exec_filter"
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
# Status distribution
|
|
330
|
+
col1, col2 = st.columns(2)
|
|
331
|
+
|
|
332
|
+
with col1:
|
|
333
|
+
if "status" in filtered_df.columns:
|
|
334
|
+
fig = create_status_pie_chart(filtered_df, "status", "Execution Status Distribution")
|
|
335
|
+
render_chart(fig)
|
|
336
|
+
|
|
337
|
+
with col2:
|
|
338
|
+
if "workflow_name" in filtered_df.columns:
|
|
339
|
+
workflow_counts = filtered_df["workflow_name"].value_counts()
|
|
340
|
+
fig = px.bar(
|
|
341
|
+
x=workflow_counts.values,
|
|
342
|
+
y=workflow_counts.index,
|
|
343
|
+
orientation='h',
|
|
344
|
+
title="Executions by Workflow"
|
|
345
|
+
)
|
|
346
|
+
render_chart(fig)
|
|
347
|
+
|
|
348
|
+
# Execution details
|
|
349
|
+
st.markdown("#### Recent Executions")
|
|
350
|
+
|
|
351
|
+
for _, execution in filtered_df.head(20).iterrows():
|
|
352
|
+
with st.expander(f"{execution.get('workflow_name')} - {execution.get('started_at')} - {display_status_badge(execution.get('status'), 'small')}"):
|
|
353
|
+
col1, col2 = st.columns(2)
|
|
354
|
+
|
|
355
|
+
with col1:
|
|
356
|
+
st.markdown(f"**Workflow:** {execution.get('workflow_name')}")
|
|
357
|
+
st.markdown(f"**Started:** {execution.get('started_at')}")
|
|
358
|
+
st.markdown(f"**Completed:** {execution.get('completed_at', 'In progress')}")
|
|
359
|
+
|
|
360
|
+
with col2:
|
|
361
|
+
st.markdown(f"**Status:** {display_status_badge(execution.get('status'), 'small')}")
|
|
362
|
+
st.markdown(f"**Triggered By:** {execution.get('triggered_by')}")
|
|
363
|
+
|
|
364
|
+
if pd.notna(execution.get('duration_sec')):
|
|
365
|
+
st.markdown(f"**Duration:** {execution['duration_sec']/60:.1f} min")
|
|
366
|
+
|
|
367
|
+
if execution.get('steps_total'):
|
|
368
|
+
progress = execution.get('steps_completed', 0) / execution['steps_total']
|
|
369
|
+
st.progress(progress)
|
|
370
|
+
st.caption(f"Steps: {execution.get('steps_completed')}/{execution['steps_total']}")
|
|
371
|
+
|
|
372
|
+
# Action buttons
|
|
373
|
+
col_btn1, col_btn2, col_btn3 = st.columns(3)
|
|
374
|
+
|
|
375
|
+
with col_btn1:
|
|
376
|
+
if st.button("📋 View Logs", key=f"logs_{execution.get('id')}"):
|
|
377
|
+
st.code(f"""
|
|
378
|
+
[INFO] Workflow execution started: {execution.get('id')}
|
|
379
|
+
[INFO] Step 1/8: Fetching data from sources...
|
|
380
|
+
[INFO] Step 2/8: Transforming data...
|
|
381
|
+
[INFO] Step 3/8: Validating data quality...
|
|
382
|
+
[INFO] Execution {'completed' if execution.get('status') == 'completed' else execution.get('status')}
|
|
383
|
+
""", language="log")
|
|
384
|
+
|
|
385
|
+
with col_btn2:
|
|
386
|
+
# Download results as JSON
|
|
387
|
+
result_data = {
|
|
388
|
+
"execution_id": execution.get('id'),
|
|
389
|
+
"workflow_name": execution.get('workflow_name'),
|
|
390
|
+
"status": execution.get('status'),
|
|
391
|
+
"started_at": str(execution.get('started_at')),
|
|
392
|
+
"completed_at": str(execution.get('completed_at')),
|
|
393
|
+
"duration_seconds": execution.get('duration_sec'),
|
|
394
|
+
"triggered_by": execution.get('triggered_by'),
|
|
395
|
+
"steps_completed": execution.get('steps_completed'),
|
|
396
|
+
"steps_total": execution.get('steps_total'),
|
|
397
|
+
"results": {
|
|
398
|
+
"records_processed": 1250,
|
|
399
|
+
"errors": 0,
|
|
400
|
+
"warnings": 3,
|
|
401
|
+
"output_location": f"/data/workflows/{execution.get('id')}/output.parquet"
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
st.download_button(
|
|
405
|
+
label="💾 Download Results",
|
|
406
|
+
data=json.dumps(result_data, indent=2),
|
|
407
|
+
file_name=f"workflow_result_{execution.get('id')}.json",
|
|
408
|
+
mime="application/json",
|
|
409
|
+
key=f"download_{execution.get('id')}"
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
with col_btn3:
|
|
413
|
+
# Link to view detailed results (mock for now)
|
|
414
|
+
if st.button("🔗 View Details", key=f"details_{execution.get('id')}"):
|
|
415
|
+
st.info(f"Results viewer would open for execution: {execution.get('id')}")
|
|
416
|
+
st.json(result_data)
|
|
417
|
+
|
|
418
|
+
# Export
|
|
419
|
+
st.markdown("#### 📥 Export Execution Data")
|
|
420
|
+
export_dataframe(filtered_df, filename="workflow_executions", formats=["csv", "json"])
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def show_workflow_builder():
|
|
424
|
+
"""Workflow builder interface"""
|
|
425
|
+
|
|
426
|
+
st.markdown("### ➕ Create New Workflow")
|
|
427
|
+
|
|
428
|
+
with st.form("new_workflow"):
|
|
429
|
+
name = st.text_input("Workflow Name", placeholder="e.g., Daily Data Sync")
|
|
430
|
+
description = st.text_area("Description", placeholder="What does this workflow do?")
|
|
431
|
+
|
|
432
|
+
col1, col2 = st.columns(2)
|
|
433
|
+
|
|
434
|
+
with col1:
|
|
435
|
+
schedule_type = st.selectbox("Schedule Type", ["Cron Expression", "Interval", "Manual Only"])
|
|
436
|
+
|
|
437
|
+
if schedule_type == "Cron Expression":
|
|
438
|
+
schedule = st.text_input("Cron Schedule", placeholder="0 0 * * *", help="Cron expression for scheduling")
|
|
439
|
+
elif schedule_type == "Interval":
|
|
440
|
+
interval_value = st.number_input("Every", min_value=1, value=1)
|
|
441
|
+
interval_unit = st.selectbox("Unit", ["minutes", "hours", "days"])
|
|
442
|
+
schedule = f"Every {interval_value} {interval_unit}"
|
|
443
|
+
else:
|
|
444
|
+
schedule = "manual"
|
|
445
|
+
|
|
446
|
+
with col2:
|
|
447
|
+
enabled = st.checkbox("Enabled", value=True)
|
|
448
|
+
retry_on_failure = st.checkbox("Retry on Failure", value=True)
|
|
449
|
+
max_retries = st.number_input("Max Retries", min_value=0, max_value=5, value=2)
|
|
450
|
+
|
|
451
|
+
st.markdown("#### Workflow Steps")
|
|
452
|
+
|
|
453
|
+
# Simple step builder
|
|
454
|
+
num_steps = st.number_input("Number of Steps", min_value=1, max_value=10, value=3)
|
|
455
|
+
|
|
456
|
+
steps = []
|
|
457
|
+
for i in range(num_steps):
|
|
458
|
+
with st.expander(f"Step {i+1}"):
|
|
459
|
+
step_name = st.text_input(f"Step Name", key=f"step_name_{i}", placeholder=f"Step {i+1}")
|
|
460
|
+
step_type = st.selectbox(f"Step Type", ["API Call", "Python Script", "Database Query", "Data Transform", "Validation"], key=f"step_type_{i}")
|
|
461
|
+
step_config = st.text_area(f"Configuration (JSON)", key=f"step_config_{i}", placeholder='{"param": "value"}')
|
|
462
|
+
|
|
463
|
+
steps.append({
|
|
464
|
+
"name": step_name,
|
|
465
|
+
"type": step_type,
|
|
466
|
+
"config": step_config
|
|
467
|
+
})
|
|
468
|
+
|
|
469
|
+
submitted = st.form_submit_button("Create Workflow")
|
|
470
|
+
|
|
471
|
+
if submitted:
|
|
472
|
+
if name and description:
|
|
473
|
+
workflow_def = {
|
|
474
|
+
"name": name,
|
|
475
|
+
"description": description,
|
|
476
|
+
"schedule": schedule,
|
|
477
|
+
"enabled": enabled,
|
|
478
|
+
"retry_on_failure": retry_on_failure,
|
|
479
|
+
"max_retries": max_retries,
|
|
480
|
+
"steps": steps
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
st.success(f"✅ Workflow '{name}' created successfully!")
|
|
484
|
+
st.json(workflow_def)
|
|
485
|
+
else:
|
|
486
|
+
st.error("Please fill in required fields: Name and Description")
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def show_workflow_templates():
|
|
490
|
+
"""Show workflow templates"""
|
|
491
|
+
|
|
492
|
+
st.markdown("### 📚 Workflow Templates")
|
|
493
|
+
|
|
494
|
+
templates = [
|
|
495
|
+
{
|
|
496
|
+
"name": "Data Ingestion Pipeline",
|
|
497
|
+
"description": "Fetch data from external APIs and store in database",
|
|
498
|
+
"category": "Data Engineering",
|
|
499
|
+
"steps": 4
|
|
500
|
+
},
|
|
501
|
+
{
|
|
502
|
+
"name": "ML Training Pipeline",
|
|
503
|
+
"description": "Train and evaluate ML models on schedule",
|
|
504
|
+
"category": "Machine Learning",
|
|
505
|
+
"steps": 6
|
|
506
|
+
},
|
|
507
|
+
{
|
|
508
|
+
"name": "Data Quality Check",
|
|
509
|
+
"description": "Validate data integrity and quality metrics",
|
|
510
|
+
"category": "Data Quality",
|
|
511
|
+
"steps": 3
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
"name": "Report Generation",
|
|
515
|
+
"description": "Generate and distribute periodic reports",
|
|
516
|
+
"category": "Reporting",
|
|
517
|
+
"steps": 5
|
|
518
|
+
}
|
|
519
|
+
]
|
|
520
|
+
|
|
521
|
+
for template in templates:
|
|
522
|
+
with st.expander(f"{template['name']} ({template['category']})"):
|
|
523
|
+
st.markdown(f"**Description:** {template['description']}")
|
|
524
|
+
st.markdown(f"**Steps:** {template['steps']}")
|
|
525
|
+
st.markdown(f"**Category:** {template['category']}")
|
|
526
|
+
|
|
527
|
+
if st.button(f"Use Template", key=f"use_{template['name']}"):
|
|
528
|
+
st.info(f"Loading template: {template['name']}")
|
|
529
|
+
# Would populate the workflow builder form
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
if __name__ == "__main__":
|
|
533
|
+
show_workflows_dashboard()
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Shared utility functions for dashboard pages"""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import logging
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import streamlit as st
|
|
8
|
+
from supabase import Client, create_client
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_supabase_client() -> Optional[Client]:
|
|
14
|
+
"""Get Supabase client with Streamlit Cloud secrets support"""
|
|
15
|
+
# Try Streamlit secrets first (for Streamlit Cloud), then fall back to environment variables (for local dev)
|
|
16
|
+
try:
|
|
17
|
+
url = st.secrets.get("SUPABASE_URL", "")
|
|
18
|
+
key = st.secrets.get("SUPABASE_KEY", "") or st.secrets.get("SUPABASE_SERVICE_ROLE_KEY", "")
|
|
19
|
+
except (AttributeError, FileNotFoundError):
|
|
20
|
+
# Secrets not available, try environment variables
|
|
21
|
+
url = os.getenv("SUPABASE_URL", "")
|
|
22
|
+
key = os.getenv("SUPABASE_KEY", "") or os.getenv("SUPABASE_SERVICE_ROLE_KEY", "")
|
|
23
|
+
|
|
24
|
+
if not url or not key:
|
|
25
|
+
logger.warning("Supabase credentials not found")
|
|
26
|
+
return None
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
client = create_client(url, key)
|
|
30
|
+
# Test connection with a simple query
|
|
31
|
+
try:
|
|
32
|
+
test_result = client.table("politicians").select("id").limit(1).execute()
|
|
33
|
+
logger.info(f"✅ Supabase connection successful (URL: {url[:30]}...)")
|
|
34
|
+
return client
|
|
35
|
+
except Exception as conn_error:
|
|
36
|
+
st.error(f"❌ Supabase connection failed: {conn_error}")
|
|
37
|
+
return None
|
|
38
|
+
except Exception as e:
|
|
39
|
+
logger.error(f"Failed to create Supabase client: {e}")
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def get_politician_names() -> List[str]:
|
|
44
|
+
"""Get all politician names from database for searchable dropdown"""
|
|
45
|
+
try:
|
|
46
|
+
client = get_supabase_client()
|
|
47
|
+
if not client:
|
|
48
|
+
return ["Nancy Pelosi", "Paul Pelosi", "Dan Crenshaw", "Josh Gottheimer"] # Fallback
|
|
49
|
+
|
|
50
|
+
result = client.table("politicians").select("first_name, last_name").execute()
|
|
51
|
+
names = [f"{row['first_name']} {row['last_name']}" for row in result.data]
|
|
52
|
+
return names if names else ["Nancy Pelosi", "Paul Pelosi", "Dan Crenshaw", "Josh Gottheimer"]
|
|
53
|
+
except Exception as e:
|
|
54
|
+
logger.error(f"Failed to get politician names: {e}")
|
|
55
|
+
return ["Nancy Pelosi", "Paul Pelosi", "Dan Crenshaw", "Josh Gottheimer"]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def get_disclosures_data() -> pd.DataFrame:
|
|
59
|
+
"""Get trading disclosures from Supabase with proper schema mapping"""
|
|
60
|
+
client = get_supabase_client()
|
|
61
|
+
if not client:
|
|
62
|
+
return _generate_demo_disclosures()
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
# First, get total count
|
|
66
|
+
count_response = (
|
|
67
|
+
client.table("trading_disclosures")
|
|
68
|
+
.select("*", count="exact")
|
|
69
|
+
.execute()
|
|
70
|
+
)
|
|
71
|
+
total_count = count_response.count
|
|
72
|
+
|
|
73
|
+
if total_count == 0:
|
|
74
|
+
return _generate_demo_disclosures()
|
|
75
|
+
|
|
76
|
+
# Get the data
|
|
77
|
+
response = (
|
|
78
|
+
client.table("trading_disclosures")
|
|
79
|
+
.select("*")
|
|
80
|
+
.order("disclosure_date", desc=True)
|
|
81
|
+
.limit(1000)
|
|
82
|
+
.execute()
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
if not response.data:
|
|
86
|
+
return _generate_demo_disclosures()
|
|
87
|
+
|
|
88
|
+
df = pd.DataFrame(response.data)
|
|
89
|
+
return df
|
|
90
|
+
|
|
91
|
+
except Exception as e:
|
|
92
|
+
logger.error(f"Failed to fetch disclosures: {e}")
|
|
93
|
+
return _generate_demo_disclosures()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _generate_demo_disclosures() -> pd.DataFrame:
|
|
97
|
+
"""Generate demo trading disclosure data for testing"""
|
|
98
|
+
st.info("🔵 Using demo trading data (Supabase unavailable)")
|
|
99
|
+
|
|
100
|
+
import random
|
|
101
|
+
from datetime import datetime, timedelta
|
|
102
|
+
|
|
103
|
+
politicians = ["Nancy Pelosi", "Paul Pelosi", "Dan Crenshaw", "Josh Gottheimer"]
|
|
104
|
+
tickers = ["AAPL", "MSFT", "GOOGL", "AMZN", "NVDA", "TSLA", "META", "AMD"]
|
|
105
|
+
transaction_types = ["Purchase", "Sale"]
|
|
106
|
+
|
|
107
|
+
data = []
|
|
108
|
+
for _ in range(50):
|
|
109
|
+
data.append({
|
|
110
|
+
"politician_name": random.choice(politicians),
|
|
111
|
+
"ticker_symbol": random.choice(tickers),
|
|
112
|
+
"transaction_type": random.choice(transaction_types),
|
|
113
|
+
"amount_min": random.randint(1000, 100000),
|
|
114
|
+
"amount_max": random.randint(100000, 1000000),
|
|
115
|
+
"disclosure_date": (datetime.now() - timedelta(days=random.randint(1, 365))).strftime("%Y-%m-%d"),
|
|
116
|
+
"asset_description": f"{random.choice(tickers)} Stock",
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
return pd.DataFrame(data)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def get_politician_trading_history(politician_name: str) -> pd.DataFrame:
|
|
123
|
+
"""Get trading history for a specific politician"""
|
|
124
|
+
try:
|
|
125
|
+
client = get_supabase_client()
|
|
126
|
+
if not client:
|
|
127
|
+
return pd.DataFrame() # Return empty if no client
|
|
128
|
+
|
|
129
|
+
# Split name into first and last
|
|
130
|
+
name_parts = politician_name.split()
|
|
131
|
+
if len(name_parts) < 2:
|
|
132
|
+
return pd.DataFrame()
|
|
133
|
+
|
|
134
|
+
first_name = name_parts[0]
|
|
135
|
+
last_name = " ".join(name_parts[1:])
|
|
136
|
+
|
|
137
|
+
# Get trading disclosures for this politician
|
|
138
|
+
response = (
|
|
139
|
+
client.table("trading_disclosures")
|
|
140
|
+
.select("*")
|
|
141
|
+
.eq("politician_name", politician_name)
|
|
142
|
+
.order("disclosure_date", desc=True)
|
|
143
|
+
.limit(100)
|
|
144
|
+
.execute()
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
if response.data:
|
|
148
|
+
return pd.DataFrame(response.data)
|
|
149
|
+
else:
|
|
150
|
+
return pd.DataFrame()
|
|
151
|
+
|
|
152
|
+
except Exception as e:
|
|
153
|
+
logger.warning(f"Failed to fetch trading history for {politician_name}: {e}")
|
|
154
|
+
return pd.DataFrame()
|