mcli-framework 7.5.1__py3-none-any.whl → 7.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mcli-framework might be problematic. Click here for more details.
- mcli/app/commands_cmd.py +51 -39
- mcli/app/completion_helpers.py +4 -13
- mcli/app/main.py +21 -25
- mcli/app/model_cmd.py +119 -9
- mcli/lib/custom_commands.py +16 -11
- mcli/ml/api/app.py +1 -5
- mcli/ml/dashboard/app.py +2 -2
- mcli/ml/dashboard/app_integrated.py +168 -116
- mcli/ml/dashboard/app_supabase.py +7 -3
- mcli/ml/dashboard/app_training.py +3 -6
- mcli/ml/dashboard/components/charts.py +74 -115
- mcli/ml/dashboard/components/metrics.py +24 -44
- mcli/ml/dashboard/components/tables.py +32 -40
- mcli/ml/dashboard/overview.py +102 -78
- mcli/ml/dashboard/pages/cicd.py +103 -56
- mcli/ml/dashboard/pages/debug_dependencies.py +35 -28
- mcli/ml/dashboard/pages/gravity_viz.py +374 -313
- mcli/ml/dashboard/pages/monte_carlo_predictions.py +50 -48
- mcli/ml/dashboard/pages/predictions_enhanced.py +396 -248
- mcli/ml/dashboard/pages/scrapers_and_logs.py +299 -273
- mcli/ml/dashboard/pages/test_portfolio.py +153 -121
- mcli/ml/dashboard/pages/trading.py +238 -169
- mcli/ml/dashboard/pages/workflows.py +129 -84
- mcli/ml/dashboard/streamlit_extras_utils.py +70 -79
- mcli/ml/dashboard/utils.py +24 -21
- mcli/ml/dashboard/warning_suppression.py +6 -4
- mcli/ml/database/session.py +16 -5
- mcli/ml/mlops/pipeline_orchestrator.py +1 -3
- mcli/ml/predictions/monte_carlo.py +6 -18
- mcli/ml/trading/alpaca_client.py +95 -96
- mcli/ml/trading/migrations.py +76 -40
- mcli/ml/trading/models.py +78 -60
- mcli/ml/trading/paper_trading.py +92 -74
- mcli/ml/trading/risk_management.py +106 -85
- mcli/ml/trading/trading_service.py +155 -110
- mcli/ml/training/train_model.py +1 -3
- mcli/{app → self}/completion_cmd.py +6 -6
- mcli/self/self_cmd.py +100 -57
- mcli/test/test_cmd.py +30 -0
- mcli/workflow/daemon/daemon.py +2 -0
- mcli/workflow/model_service/openai_adapter.py +347 -0
- mcli/workflow/politician_trading/models.py +6 -2
- mcli/workflow/politician_trading/scrapers_corporate_registry.py +39 -88
- mcli/workflow/politician_trading/scrapers_free_sources.py +32 -39
- mcli/workflow/politician_trading/scrapers_third_party.py +21 -39
- mcli/workflow/politician_trading/seed_database.py +70 -89
- {mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/METADATA +1 -1
- {mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/RECORD +56 -54
- /mcli/{app → self}/logs_cmd.py +0 -0
- /mcli/{app → self}/redis_cmd.py +0 -0
- /mcli/{app → self}/visual_cmd.py +0 -0
- /mcli/{app → test}/cron_test_cmd.py +0 -0
- {mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/WHEEL +0 -0
- {mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/entry_points.txt +0 -0
- {mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/licenses/LICENSE +0 -0
- {mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/top_level.txt +0 -0
|
@@ -41,12 +41,13 @@ load_environment_variables()
|
|
|
41
41
|
# Import streamlit-extras utilities
|
|
42
42
|
try:
|
|
43
43
|
from mcli.ml.dashboard.streamlit_extras_utils import (
|
|
44
|
+
data_quality_indicators,
|
|
44
45
|
enhanced_metrics,
|
|
45
46
|
section_header,
|
|
46
|
-
vertical_space,
|
|
47
|
-
data_quality_indicators,
|
|
48
47
|
trading_status_card,
|
|
48
|
+
vertical_space,
|
|
49
49
|
)
|
|
50
|
+
|
|
50
51
|
HAS_STREAMLIT_EXTRAS = True
|
|
51
52
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
52
53
|
HAS_STREAMLIT_EXTRAS = False
|
|
@@ -101,18 +102,21 @@ show_debug_dependencies = None
|
|
|
101
102
|
# Import Overview page
|
|
102
103
|
try:
|
|
103
104
|
from mcli.ml.dashboard.overview import show_overview
|
|
105
|
+
|
|
104
106
|
HAS_OVERVIEW_PAGE = True
|
|
105
107
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
106
108
|
st.warning(f"Overview page not available: {e}")
|
|
107
109
|
|
|
108
110
|
try:
|
|
109
111
|
from mcli.ml.dashboard.pages.predictions_enhanced import show_predictions_enhanced
|
|
112
|
+
|
|
110
113
|
HAS_PREDICTIONS_ENHANCED = True
|
|
111
114
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
112
115
|
st.warning(f"Predictions Enhanced page not available: {e}")
|
|
113
116
|
|
|
114
117
|
try:
|
|
115
118
|
from mcli.ml.dashboard.pages.scrapers_and_logs import show_scrapers_and_logs
|
|
119
|
+
|
|
116
120
|
HAS_SCRAPERS_PAGE = True
|
|
117
121
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
118
122
|
st.warning(f"Scrapers & Logs page not available: {e}")
|
|
@@ -127,6 +131,7 @@ try:
|
|
|
127
131
|
# First, try importing alpaca directly to see the specific error
|
|
128
132
|
try:
|
|
129
133
|
import alpaca
|
|
134
|
+
|
|
130
135
|
st.success(f"✅ alpaca module imported successfully")
|
|
131
136
|
if hasattr(alpaca, "__version__"):
|
|
132
137
|
st.info(f"Alpaca version: {alpaca.__version__}")
|
|
@@ -139,16 +144,21 @@ try:
|
|
|
139
144
|
|
|
140
145
|
# Try to provide diagnostic info
|
|
141
146
|
st.warning("💡 Troubleshooting tips:")
|
|
142
|
-
st.markdown(
|
|
147
|
+
st.markdown(
|
|
148
|
+
"""
|
|
143
149
|
- Check that `alpaca-py>=0.20.0` is in requirements.txt
|
|
144
150
|
- Verify Python version is 3.8+ (current: {}.{})
|
|
145
151
|
- Check Streamlit Cloud deployment logs for installation errors
|
|
146
152
|
- Visit the **Debug Dependencies** page for detailed diagnostics
|
|
147
|
-
""".format(
|
|
153
|
+
""".format(
|
|
154
|
+
sys.version_info.major, sys.version_info.minor
|
|
155
|
+
)
|
|
156
|
+
)
|
|
148
157
|
|
|
149
158
|
# Now try importing the trading pages
|
|
150
|
-
from mcli.ml.dashboard.pages.trading import show_trading_dashboard
|
|
151
159
|
from mcli.ml.dashboard.pages.test_portfolio import show_test_portfolio
|
|
160
|
+
from mcli.ml.dashboard.pages.trading import show_trading_dashboard
|
|
161
|
+
|
|
152
162
|
HAS_TRADING_PAGES = True
|
|
153
163
|
st.success("✅ Trading pages imported successfully!")
|
|
154
164
|
|
|
@@ -160,12 +170,8 @@ except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
|
160
170
|
# Show installed packages related to alpaca
|
|
161
171
|
try:
|
|
162
172
|
import subprocess
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
capture_output=True,
|
|
166
|
-
text=True,
|
|
167
|
-
timeout=5
|
|
168
|
-
)
|
|
173
|
+
|
|
174
|
+
result = subprocess.run(["pip", "list"], capture_output=True, text=True, timeout=5)
|
|
169
175
|
alpaca_packages = [line for line in result.stdout.split("\n") if "alpaca" in line.lower()]
|
|
170
176
|
if alpaca_packages:
|
|
171
177
|
st.info("📦 Found alpaca-related packages:")
|
|
@@ -182,6 +188,7 @@ except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
|
182
188
|
|
|
183
189
|
try:
|
|
184
190
|
from mcli.ml.dashboard.pages.monte_carlo_predictions import show_monte_carlo_predictions
|
|
191
|
+
|
|
185
192
|
HAS_MONTE_CARLO_PAGE = True
|
|
186
193
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
187
194
|
HAS_MONTE_CARLO_PAGE = False
|
|
@@ -189,12 +196,14 @@ except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
|
189
196
|
# Import CI/CD and Workflows pages
|
|
190
197
|
try:
|
|
191
198
|
from mcli.ml.dashboard.pages.cicd import show_cicd_dashboard
|
|
199
|
+
|
|
192
200
|
HAS_CICD_PAGE = True
|
|
193
201
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
194
202
|
st.warning(f"CI/CD page not available: {e}")
|
|
195
203
|
|
|
196
204
|
try:
|
|
197
205
|
from mcli.ml.dashboard.pages.workflows import show_workflows_dashboard
|
|
206
|
+
|
|
198
207
|
HAS_WORKFLOWS_PAGE = True
|
|
199
208
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
200
209
|
st.warning(f"Workflows page not available: {e}")
|
|
@@ -202,15 +211,13 @@ except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
|
202
211
|
# Import Debug Dependencies page (always available for troubleshooting)
|
|
203
212
|
try:
|
|
204
213
|
from mcli.ml.dashboard.pages.debug_dependencies import show_debug_dependencies
|
|
214
|
+
|
|
205
215
|
HAS_DEBUG_PAGE = True
|
|
206
216
|
except (ImportError, KeyError, ModuleNotFoundError) as e:
|
|
207
217
|
st.warning(f"Debug Dependencies page not available: {e}")
|
|
208
218
|
|
|
209
219
|
# Page config - must be before other st commands
|
|
210
|
-
setup_page_config(
|
|
211
|
-
page_title="Politician Trading Tracker - MCLI",
|
|
212
|
-
page_icon="📊"
|
|
213
|
-
)
|
|
220
|
+
setup_page_config(page_title="Politician Trading Tracker - MCLI", page_icon="📊")
|
|
214
221
|
|
|
215
222
|
# Apply standard dashboard styles (includes metric-card, alert boxes)
|
|
216
223
|
apply_dashboard_styles()
|
|
@@ -631,7 +638,9 @@ def _generate_fallback_predictions(processed_data):
|
|
|
631
638
|
n_tickers = len(tickers)
|
|
632
639
|
else:
|
|
633
640
|
# Generate demo predictions with realistic tickers
|
|
634
|
-
tickers = np.array(
|
|
641
|
+
tickers = np.array(
|
|
642
|
+
["AAPL", "GOOGL", "MSFT", "TSLA", "AMZN", "NVDA", "META", "NFLX", "AMD", "INTC"]
|
|
643
|
+
)
|
|
635
644
|
n_tickers = len(tickers)
|
|
636
645
|
st.info("🔵 Showing demo predictions (Supabase connection unavailable)")
|
|
637
646
|
|
|
@@ -709,11 +718,7 @@ def get_disclosures_data(limit: int = 1000, offset: int = 0, for_training: bool
|
|
|
709
718
|
|
|
710
719
|
try:
|
|
711
720
|
# First, get total count
|
|
712
|
-
count_response = (
|
|
713
|
-
client.table("trading_disclosures")
|
|
714
|
-
.select("*", count="exact")
|
|
715
|
-
.execute()
|
|
716
|
-
)
|
|
721
|
+
count_response = client.table("trading_disclosures").select("*", count="exact").execute()
|
|
717
722
|
total_count = count_response.count
|
|
718
723
|
|
|
719
724
|
# Fetch data with appropriate limit
|
|
@@ -754,55 +759,53 @@ def get_disclosures_data(limit: int = 1000, offset: int = 0, for_training: bool
|
|
|
754
759
|
|
|
755
760
|
# Map Supabase schema to dashboard expected columns
|
|
756
761
|
# Extract politician info from nested dict
|
|
757
|
-
if
|
|
758
|
-
df[
|
|
759
|
-
lambda x: x.get(
|
|
762
|
+
if "politicians" in df.columns:
|
|
763
|
+
df["politician_name"] = df["politicians"].apply(
|
|
764
|
+
lambda x: x.get("full_name", "") if isinstance(x, dict) else ""
|
|
760
765
|
)
|
|
761
|
-
df[
|
|
762
|
-
lambda x: x.get(
|
|
766
|
+
df["party"] = df["politicians"].apply(
|
|
767
|
+
lambda x: x.get("party", "") if isinstance(x, dict) else ""
|
|
763
768
|
)
|
|
764
|
-
df[
|
|
765
|
-
lambda x: x.get(
|
|
769
|
+
df["state"] = df["politicians"].apply(
|
|
770
|
+
lambda x: x.get("state_or_country", "") if isinstance(x, dict) else ""
|
|
766
771
|
)
|
|
767
772
|
|
|
768
773
|
# Map asset_ticker to ticker_symbol (dashboard expects this)
|
|
769
774
|
# Note: Most disclosures don't have stock tickers (funds, real estate, bonds)
|
|
770
775
|
# Use asset_type as categorical identifier for non-stock assets
|
|
771
|
-
if
|
|
776
|
+
if "asset_ticker" in df.columns:
|
|
772
777
|
# Use real ticker when available
|
|
773
|
-
df[
|
|
778
|
+
df["ticker_symbol"] = df["asset_ticker"]
|
|
774
779
|
|
|
775
780
|
# For None/null values, use asset_type as category
|
|
776
|
-
if
|
|
777
|
-
df[
|
|
778
|
-
df[
|
|
781
|
+
if "asset_type" in df.columns:
|
|
782
|
+
df["ticker_symbol"] = df["ticker_symbol"].fillna(
|
|
783
|
+
df["asset_type"].str.upper().str.replace("_", "-")
|
|
779
784
|
)
|
|
780
785
|
else:
|
|
781
|
-
df[
|
|
782
|
-
elif
|
|
786
|
+
df["ticker_symbol"] = df["ticker_symbol"].fillna("NON-STOCK")
|
|
787
|
+
elif "asset_type" in df.columns:
|
|
783
788
|
# No ticker column - use asset type as category
|
|
784
|
-
df[
|
|
789
|
+
df["ticker_symbol"] = df["asset_type"].str.upper().str.replace("_", "-")
|
|
785
790
|
else:
|
|
786
|
-
df[
|
|
791
|
+
df["ticker_symbol"] = "UNKNOWN"
|
|
787
792
|
|
|
788
793
|
# Calculate amount from range (use midpoint)
|
|
789
|
-
if
|
|
790
|
-
df[
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
elif 'amount_exact' in df.columns:
|
|
794
|
-
df['amount'] = df['amount_exact']
|
|
794
|
+
if "amount_range_min" in df.columns and "amount_range_max" in df.columns:
|
|
795
|
+
df["amount"] = (df["amount_range_min"].fillna(0) + df["amount_range_max"].fillna(0)) / 2
|
|
796
|
+
elif "amount_exact" in df.columns:
|
|
797
|
+
df["amount"] = df["amount_exact"]
|
|
795
798
|
else:
|
|
796
|
-
df[
|
|
799
|
+
df["amount"] = 0
|
|
797
800
|
|
|
798
801
|
# Add asset_description if not exists
|
|
799
|
-
if
|
|
800
|
-
df[
|
|
802
|
+
if "asset_description" not in df.columns and "asset_name" in df.columns:
|
|
803
|
+
df["asset_description"] = df["asset_name"]
|
|
801
804
|
|
|
802
805
|
# Convert dates to datetime with ISO8601 format
|
|
803
|
-
for date_col in [
|
|
806
|
+
for date_col in ["disclosure_date", "transaction_date", "created_at", "updated_at"]:
|
|
804
807
|
if date_col in df.columns:
|
|
805
|
-
df[date_col] = pd.to_datetime(df[date_col], format=
|
|
808
|
+
df[date_col] = pd.to_datetime(df[date_col], format="ISO8601", errors="coerce")
|
|
806
809
|
|
|
807
810
|
# Convert any remaining dict/list columns to JSON strings
|
|
808
811
|
for col in df.columns:
|
|
@@ -827,7 +830,13 @@ def _generate_demo_disclosures():
|
|
|
827
830
|
np.random.seed(42)
|
|
828
831
|
n_records = 100
|
|
829
832
|
|
|
830
|
-
politicians = [
|
|
833
|
+
politicians = [
|
|
834
|
+
"Nancy Pelosi",
|
|
835
|
+
"Paul Pelosi",
|
|
836
|
+
"Dan Crenshaw",
|
|
837
|
+
"Josh Gottheimer",
|
|
838
|
+
"Tommy Tuberville",
|
|
839
|
+
]
|
|
831
840
|
tickers = ["AAPL", "GOOGL", "MSFT", "TSLA", "AMZN", "NVDA", "META", "NFLX", "AMD", "INTC"]
|
|
832
841
|
transaction_types = ["purchase", "sale", "exchange"]
|
|
833
842
|
|
|
@@ -836,18 +845,22 @@ def _generate_demo_disclosures():
|
|
|
836
845
|
start_date = end_date - pd.Timedelta(days=180)
|
|
837
846
|
dates = pd.date_range(start=start_date, end=end_date, periods=n_records)
|
|
838
847
|
|
|
839
|
-
return pd.DataFrame(
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
848
|
+
return pd.DataFrame(
|
|
849
|
+
{
|
|
850
|
+
"id": range(1, n_records + 1),
|
|
851
|
+
"politician_name": np.random.choice(politicians, n_records),
|
|
852
|
+
"ticker_symbol": np.random.choice(tickers, n_records),
|
|
853
|
+
"transaction_type": np.random.choice(transaction_types, n_records),
|
|
854
|
+
"amount": np.random.uniform(15000, 500000, n_records),
|
|
855
|
+
"disclosure_date": dates,
|
|
856
|
+
"transaction_date": dates - pd.Timedelta(days=np.random.randint(1, 45)),
|
|
857
|
+
"asset_description": [
|
|
858
|
+
f"Common Stock - {t}" for t in np.random.choice(tickers, n_records)
|
|
859
|
+
],
|
|
860
|
+
"party": np.random.choice(["Democrat", "Republican"], n_records),
|
|
861
|
+
"state": np.random.choice(["CA", "TX", "NY", "FL", "AL"], n_records),
|
|
862
|
+
}
|
|
863
|
+
)
|
|
851
864
|
|
|
852
865
|
|
|
853
866
|
@st.cache_data(ttl=30)
|
|
@@ -883,15 +896,15 @@ def get_model_metrics():
|
|
|
883
896
|
|
|
884
897
|
def main():
|
|
885
898
|
"""Main dashboard function"""
|
|
886
|
-
|
|
899
|
+
|
|
887
900
|
# Clear any problematic session state that might cause media file errors
|
|
888
901
|
try:
|
|
889
902
|
# Remove any file-related session state that might be causing issues
|
|
890
903
|
keys_to_remove = []
|
|
891
904
|
for key in st.session_state.keys():
|
|
892
|
-
if
|
|
905
|
+
if "file" in key.lower() or "download" in key.lower() or "media" in key.lower():
|
|
893
906
|
keys_to_remove.append(key)
|
|
894
|
-
|
|
907
|
+
|
|
895
908
|
for key in keys_to_remove:
|
|
896
909
|
if key in st.session_state:
|
|
897
910
|
del st.session_state[key]
|
|
@@ -913,17 +926,19 @@ def main():
|
|
|
913
926
|
pages.append("Overview")
|
|
914
927
|
|
|
915
928
|
# Add other pages
|
|
916
|
-
pages.extend(
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
929
|
+
pages.extend(
|
|
930
|
+
[
|
|
931
|
+
"Pipeline Overview",
|
|
932
|
+
"ML Processing",
|
|
933
|
+
"Model Performance",
|
|
934
|
+
"Model Training & Evaluation",
|
|
935
|
+
"Predictions",
|
|
936
|
+
"Trading Dashboard",
|
|
937
|
+
"Test Portfolio",
|
|
938
|
+
"LSH Jobs",
|
|
939
|
+
"System Health",
|
|
940
|
+
]
|
|
941
|
+
)
|
|
927
942
|
|
|
928
943
|
# Add scrapers and logs page
|
|
929
944
|
if HAS_SCRAPERS_PAGE:
|
|
@@ -946,10 +961,7 @@ def main():
|
|
|
946
961
|
pages.append("Debug Dependencies")
|
|
947
962
|
|
|
948
963
|
page = st.sidebar.selectbox(
|
|
949
|
-
"Choose a page",
|
|
950
|
-
pages,
|
|
951
|
-
index=0, # Default to Pipeline Overview
|
|
952
|
-
key="main_page_selector"
|
|
964
|
+
"Choose a page", pages, index=0, key="main_page_selector" # Default to Pipeline Overview
|
|
953
965
|
)
|
|
954
966
|
|
|
955
967
|
# Auto-refresh toggle (default off to prevent blocking)
|
|
@@ -1006,6 +1018,7 @@ def main():
|
|
|
1006
1018
|
except Exception as e:
|
|
1007
1019
|
st.error(f"❌ Error in Trading Dashboard page: {e}")
|
|
1008
1020
|
import traceback
|
|
1021
|
+
|
|
1009
1022
|
st.code(traceback.format_exc())
|
|
1010
1023
|
else:
|
|
1011
1024
|
st.warning("Trading dashboard not available")
|
|
@@ -1016,6 +1029,7 @@ def main():
|
|
|
1016
1029
|
except Exception as e:
|
|
1017
1030
|
st.error(f"❌ Error in Test Portfolio page: {e}")
|
|
1018
1031
|
import traceback
|
|
1032
|
+
|
|
1019
1033
|
st.code(traceback.format_exc())
|
|
1020
1034
|
else:
|
|
1021
1035
|
st.warning("Test portfolio not available")
|
|
@@ -1026,6 +1040,7 @@ def main():
|
|
|
1026
1040
|
except Exception as e:
|
|
1027
1041
|
st.error(f"❌ Error in Monte Carlo Predictions page: {e}")
|
|
1028
1042
|
import traceback
|
|
1043
|
+
|
|
1029
1044
|
st.code(traceback.format_exc())
|
|
1030
1045
|
else:
|
|
1031
1046
|
st.warning("Monte Carlo predictions not available")
|
|
@@ -1039,6 +1054,7 @@ def main():
|
|
|
1039
1054
|
except Exception as e:
|
|
1040
1055
|
st.error(f"❌ Error in Scrapers & Logs page: {e}")
|
|
1041
1056
|
import traceback
|
|
1057
|
+
|
|
1042
1058
|
st.code(traceback.format_exc())
|
|
1043
1059
|
elif page == "CI/CD Pipelines":
|
|
1044
1060
|
if show_cicd_dashboard is not None:
|
|
@@ -1047,9 +1063,12 @@ def main():
|
|
|
1047
1063
|
except Exception as e:
|
|
1048
1064
|
st.error(f"❌ Error in CI/CD Pipelines page: {e}")
|
|
1049
1065
|
import traceback
|
|
1066
|
+
|
|
1050
1067
|
st.code(traceback.format_exc())
|
|
1051
1068
|
else:
|
|
1052
|
-
st.warning(
|
|
1069
|
+
st.warning(
|
|
1070
|
+
"CI/CD Pipelines page is not available. This page requires additional dependencies."
|
|
1071
|
+
)
|
|
1053
1072
|
elif page == "Workflows":
|
|
1054
1073
|
if show_workflows_dashboard is not None:
|
|
1055
1074
|
try:
|
|
@@ -1057,9 +1076,12 @@ def main():
|
|
|
1057
1076
|
except Exception as e:
|
|
1058
1077
|
st.error(f"❌ Error in Workflows page: {e}")
|
|
1059
1078
|
import traceback
|
|
1079
|
+
|
|
1060
1080
|
st.code(traceback.format_exc())
|
|
1061
1081
|
else:
|
|
1062
|
-
st.warning(
|
|
1082
|
+
st.warning(
|
|
1083
|
+
"Workflows page is not available. This page requires additional dependencies."
|
|
1084
|
+
)
|
|
1063
1085
|
|
|
1064
1086
|
elif page == "Debug Dependencies":
|
|
1065
1087
|
if show_debug_dependencies is not None:
|
|
@@ -1068,6 +1090,7 @@ def main():
|
|
|
1068
1090
|
except Exception as e:
|
|
1069
1091
|
st.error(f"❌ Error in Debug Dependencies page: {e}")
|
|
1070
1092
|
import traceback
|
|
1093
|
+
|
|
1071
1094
|
st.code(traceback.format_exc())
|
|
1072
1095
|
else:
|
|
1073
1096
|
st.warning("Debug Dependencies page is not available.")
|
|
@@ -1100,13 +1123,15 @@ def show_pipeline_overview():
|
|
|
1100
1123
|
st.markdown("### 📄 Data Pagination")
|
|
1101
1124
|
|
|
1102
1125
|
# Initialize session state for page number
|
|
1103
|
-
if
|
|
1126
|
+
if "page_number" not in st.session_state:
|
|
1104
1127
|
st.session_state.page_number = 1
|
|
1105
1128
|
|
|
1106
1129
|
col_size, col_page_input, col_nav = st.columns([1, 2, 2])
|
|
1107
1130
|
|
|
1108
1131
|
with col_size:
|
|
1109
|
-
page_size = st.selectbox(
|
|
1132
|
+
page_size = st.selectbox(
|
|
1133
|
+
"Records per page", [100, 500, 1000, 2000], index=2, key="page_size_select"
|
|
1134
|
+
)
|
|
1110
1135
|
|
|
1111
1136
|
# Get total count first
|
|
1112
1137
|
client = get_supabase_client()
|
|
@@ -1126,7 +1151,7 @@ def show_pipeline_overview():
|
|
|
1126
1151
|
max_value=max(1, total_pages),
|
|
1127
1152
|
value=st.session_state.page_number,
|
|
1128
1153
|
step=1,
|
|
1129
|
-
key="page_number_input"
|
|
1154
|
+
key="page_number_input",
|
|
1130
1155
|
)
|
|
1131
1156
|
st.session_state.page_number = page_input
|
|
1132
1157
|
|
|
@@ -1323,8 +1348,14 @@ def train_model_with_feedback():
|
|
|
1323
1348
|
noise_level = 0.1 / batch_smoothness # Larger batch = less noise
|
|
1324
1349
|
|
|
1325
1350
|
# Calculate metrics with parameter effects
|
|
1326
|
-
train_loss = (0.5 + np.random.uniform(0, 0.3 * stability)) * np.exp(
|
|
1327
|
-
|
|
1351
|
+
train_loss = (0.5 + np.random.uniform(0, 0.3 * stability)) * np.exp(
|
|
1352
|
+
-(epoch / epochs) * convergence_speed
|
|
1353
|
+
) + np.random.uniform(-noise_level, noise_level)
|
|
1354
|
+
train_acc = (
|
|
1355
|
+
0.5
|
|
1356
|
+
+ (0.4 * (epoch / epochs) * convergence_speed)
|
|
1357
|
+
+ np.random.uniform(-noise_level * stability, noise_level * stability)
|
|
1358
|
+
)
|
|
1328
1359
|
val_loss = train_loss * (1 + np.random.uniform(-0.05 * stability, 0.15 * stability))
|
|
1329
1360
|
val_acc = train_acc * (1 + np.random.uniform(-0.1 * stability, 0.1 * stability))
|
|
1330
1361
|
|
|
@@ -1504,45 +1535,55 @@ def show_ml_processing():
|
|
|
1504
1535
|
|
|
1505
1536
|
# Select and reorder columns for better display
|
|
1506
1537
|
display_columns = [
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1538
|
+
"transaction_date",
|
|
1539
|
+
(
|
|
1540
|
+
"politician_name"
|
|
1541
|
+
if "politician_name" in disclosures.columns
|
|
1542
|
+
else "politician_id"
|
|
1543
|
+
),
|
|
1544
|
+
"transaction_type",
|
|
1545
|
+
"asset_name", # The actual stock/asset name
|
|
1546
|
+
"asset_ticker", # The stock ticker (e.g., AAPL, TSLA)
|
|
1547
|
+
"asset_type", # Type (Stock, Fund, etc.)
|
|
1548
|
+
"amount_range_min",
|
|
1549
|
+
"amount_range_max",
|
|
1515
1550
|
]
|
|
1516
1551
|
|
|
1517
1552
|
# Only include columns that exist in the DataFrame
|
|
1518
|
-
available_display_cols = [
|
|
1553
|
+
available_display_cols = [
|
|
1554
|
+
col for col in display_columns if col in disclosures.columns
|
|
1555
|
+
]
|
|
1519
1556
|
|
|
1520
1557
|
# Display the data with selected columns
|
|
1521
1558
|
display_df = disclosures[available_display_cols].head(100).copy()
|
|
1522
1559
|
|
|
1523
1560
|
# Rename columns for better readability
|
|
1524
1561
|
column_renames = {
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1562
|
+
"transaction_date": "Date",
|
|
1563
|
+
"politician_name": "Politician",
|
|
1564
|
+
"politician_id": "Politician ID",
|
|
1565
|
+
"transaction_type": "Type",
|
|
1566
|
+
"asset_name": "Asset Name",
|
|
1567
|
+
"asset_ticker": "Ticker",
|
|
1568
|
+
"asset_type": "Asset Type",
|
|
1569
|
+
"amount_range_min": "Min Amount",
|
|
1570
|
+
"amount_range_max": "Max Amount",
|
|
1534
1571
|
}
|
|
1535
1572
|
display_df.rename(columns=column_renames, inplace=True)
|
|
1536
1573
|
|
|
1537
1574
|
# Show info about record counts
|
|
1538
|
-
st.info(
|
|
1575
|
+
st.info(
|
|
1576
|
+
f"📊 Processing **{len(disclosures):,} total records** (showing first 100 for preview)"
|
|
1577
|
+
)
|
|
1539
1578
|
|
|
1540
1579
|
st.dataframe(display_df, width="stretch")
|
|
1541
1580
|
st.metric("Total Records Being Processed", len(disclosures))
|
|
1542
1581
|
|
|
1543
1582
|
with tabs[1]:
|
|
1544
1583
|
st.subheader("Preprocessed Data")
|
|
1545
|
-
st.info(
|
|
1584
|
+
st.info(
|
|
1585
|
+
f"📊 Processing **{len(processed_data):,} total records** (showing first 100 for preview)"
|
|
1586
|
+
)
|
|
1546
1587
|
st.dataframe(processed_data.head(100), width="stretch")
|
|
1547
1588
|
|
|
1548
1589
|
# Data quality metrics
|
|
@@ -1580,7 +1621,9 @@ def show_ml_processing():
|
|
|
1580
1621
|
)
|
|
1581
1622
|
st.plotly_chart(fig, width="stretch", config={"responsive": True})
|
|
1582
1623
|
|
|
1583
|
-
st.info(
|
|
1624
|
+
st.info(
|
|
1625
|
+
f"📊 Generated features for **{len(features):,} total records** (showing first 100 for preview)"
|
|
1626
|
+
)
|
|
1584
1627
|
st.dataframe(features.head(100), width="stretch")
|
|
1585
1628
|
|
|
1586
1629
|
with tabs[3]:
|
|
@@ -1626,7 +1669,8 @@ def show_ml_processing():
|
|
|
1626
1669
|
|
|
1627
1670
|
elif predictions is None:
|
|
1628
1671
|
st.error("❌ ML Pipeline Error: No predictions generated")
|
|
1629
|
-
st.info(
|
|
1672
|
+
st.info(
|
|
1673
|
+
"""
|
|
1630
1674
|
**Possible causes:**
|
|
1631
1675
|
- No trained model available
|
|
1632
1676
|
- Insufficient training data
|
|
@@ -1637,19 +1681,25 @@ def show_ml_processing():
|
|
|
1637
1681
|
2. Check 'Preprocessed' tab - verify data preprocessing works
|
|
1638
1682
|
3. Go to 'Model Training & Evaluation' page to train a model
|
|
1639
1683
|
4. Check Supabase connection in 'System Health' page
|
|
1640
|
-
"""
|
|
1684
|
+
"""
|
|
1685
|
+
)
|
|
1641
1686
|
|
|
1642
1687
|
# Debug info
|
|
1643
1688
|
with st.expander("🔍 Debug Information"):
|
|
1644
1689
|
st.write("**Data Status:**")
|
|
1645
1690
|
st.write(f"- Raw records: {len(disclosures)}")
|
|
1646
|
-
st.write(
|
|
1647
|
-
|
|
1691
|
+
st.write(
|
|
1692
|
+
f"- Processed records: {len(processed_data) if processed_data is not None else 'N/A'}"
|
|
1693
|
+
)
|
|
1694
|
+
st.write(
|
|
1695
|
+
f"- Features generated: {len(features.columns) if features is not None else 'N/A'}"
|
|
1696
|
+
)
|
|
1648
1697
|
st.write(f"- Predictions: None")
|
|
1649
1698
|
|
|
1650
1699
|
else:
|
|
1651
1700
|
st.warning("⚠️ No predictions generated (empty results)")
|
|
1652
|
-
st.info(
|
|
1701
|
+
st.info(
|
|
1702
|
+
"""
|
|
1653
1703
|
**This usually means:**
|
|
1654
1704
|
- Not enough data to generate predictions
|
|
1655
1705
|
- All data was filtered out during feature engineering
|
|
@@ -1660,10 +1710,11 @@ def show_ml_processing():
|
|
|
1660
1710
|
- Processed records: {}
|
|
1661
1711
|
- Features: {}
|
|
1662
1712
|
""".format(
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1713
|
+
len(disclosures),
|
|
1714
|
+
len(processed_data) if processed_data is not None else 0,
|
|
1715
|
+
len(features) if features is not None else 0,
|
|
1716
|
+
)
|
|
1717
|
+
)
|
|
1667
1718
|
else:
|
|
1668
1719
|
st.error("Failed to process data through pipeline")
|
|
1669
1720
|
else:
|
|
@@ -2975,5 +3026,6 @@ except Exception as e:
|
|
|
2975
3026
|
st.error(f"❌ Dashboard error: {e}")
|
|
2976
3027
|
st.info("🔄 Please refresh the page to try again")
|
|
2977
3028
|
import traceback
|
|
3029
|
+
|
|
2978
3030
|
with st.expander("Error details"):
|
|
2979
3031
|
st.code(traceback.format_exc())
|
|
@@ -10,7 +10,11 @@ import plotly.graph_objects as go
|
|
|
10
10
|
import streamlit as st
|
|
11
11
|
from plotly.subplots import make_subplots
|
|
12
12
|
|
|
13
|
-
from mcli.ml.dashboard.common import
|
|
13
|
+
from mcli.ml.dashboard.common import (
|
|
14
|
+
get_supabase_client,
|
|
15
|
+
load_environment_variables,
|
|
16
|
+
setup_page_config,
|
|
17
|
+
)
|
|
14
18
|
from mcli.ml.dashboard.styles import apply_dashboard_styles
|
|
15
19
|
|
|
16
20
|
# Page config must come first
|
|
@@ -61,10 +65,10 @@ def get_disclosures_data():
|
|
|
61
65
|
df = pd.DataFrame(response.data)
|
|
62
66
|
|
|
63
67
|
# Convert datetime columns to proper datetime format
|
|
64
|
-
date_columns = [
|
|
68
|
+
date_columns = ["transaction_date", "disclosure_date", "created_at", "updated_at"]
|
|
65
69
|
for col in date_columns:
|
|
66
70
|
if col in df.columns:
|
|
67
|
-
df[col] = pd.to_datetime(df[col], format=
|
|
71
|
+
df[col] = pd.to_datetime(df[col], format="ISO8601", errors="coerce")
|
|
68
72
|
|
|
69
73
|
return df
|
|
70
74
|
except Exception as e:
|
|
@@ -10,16 +10,13 @@ import streamlit as st
|
|
|
10
10
|
from plotly.subplots import make_subplots
|
|
11
11
|
from scipy import stats
|
|
12
12
|
|
|
13
|
-
from mcli.ml.database.models import Experiment, Model, ModelStatus
|
|
14
|
-
from mcli.ml.database.session import SessionLocal
|
|
15
13
|
from mcli.ml.dashboard.common import setup_page_config
|
|
16
14
|
from mcli.ml.dashboard.styles import apply_dashboard_styles
|
|
15
|
+
from mcli.ml.database.models import Experiment, Model, ModelStatus
|
|
16
|
+
from mcli.ml.database.session import SessionLocal
|
|
17
17
|
|
|
18
18
|
# Page config - must be first
|
|
19
|
-
setup_page_config(
|
|
20
|
-
page_title="MCLI Training Dashboard",
|
|
21
|
-
page_icon="🔬"
|
|
22
|
-
)
|
|
19
|
+
setup_page_config(page_title="MCLI Training Dashboard", page_icon="🔬")
|
|
23
20
|
|
|
24
21
|
# Apply standard dashboard styles (includes metric-card)
|
|
25
22
|
apply_dashboard_styles()
|