featrixsphere 0.2.5978__tar.gz → 0.2.6379__tar.gz

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.5978
+Version: 0.2.6379
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-0.2.6379/VERSION

@@ -0,0 +1 @@
+0.2.6379

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/featrixsphere/__init__.py

@@ -57,7 +57,7 @@ TWO API OPTIONS:
     >>> print(result['prediction'])
 """
 
-__version__ = "0.2.5978"
+__version__ = "0.2.6379"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/featrixsphere/client.py

@@ -137,22 +137,24 @@ class SessionInfo:
 class PredictionBatch:
     """
     Cached prediction batch that allows instant lookups after initial batch processing.
-    
+
     Usage:
         # First run - populate cache
         batch = client.predict_batch(session_id, records)
-        
+
         # Second run - instant cache lookups
         for i in values1:
             for j in values2:
                 record = {"param1": i, "param2": j}
                 result = batch.predict(record)  # Instant!
     """
-    
-    def __init__(self, session_id: str, client: 'FeatrixSphereClient', target_column: str = None):
+
+    def __init__(self, session_id: str, client: 'FeatrixSphereClient', target_column: str = None,
+                 best_metric_preference: str = None):
         self.session_id = session_id
         self.client = client
         self.target_column = target_column
+        self.best_metric_preference = best_metric_preference
         self._cache = {}  # record_hash -> prediction_result
         self._stats = {'hits': 0, 'misses': 0, 'populated': 0}
         
@@ -203,14 +205,15 @@ class PredictionBatch:
         """Populate the cache with batch predictions."""
         if not records:
             return {'summary': {'total_records': 0, 'successful': 0, 'failed': 0}}
-        
+
         print(f"🚀 Creating prediction batch for {len(records)} records...")
-        
+
         # Use existing batch prediction system
         batch_results = self.client.predict_records(
             session_id=self.session_id,
             records=records,
             target_column=self.target_column,
+            best_metric_preference=self.best_metric_preference,
             show_progress_bar=True
         )
         
@@ -1958,7 +1961,7 @@ class FeatrixSphereClient:
             ax.text(0.5, 0.5, f'Error plotting embedding: {e}', 
                    transform=ax.transAxes, ha='center', va='center')
 
-    def create_embedding_space(self, name: str, s3_training_dataset: str, s3_validation_dataset: str, s3_visualization_dataset: str = None, webhooks: Dict[str, str] = None, user_metadata: Dict[str, Any] = None) -> SessionInfo:
+    def create_embedding_space(self, name: str, s3_training_dataset: str, s3_validation_dataset: str, s3_visualization_dataset: str = None, webhooks: Dict[str, str] = None, user_metadata: Dict[str, Any] = None, foundation_mode: bool = None) -> SessionInfo:
         """
         Create a new embedding space from S3 training and validation datasets.
 
@@ -1969,6 +1972,9 @@ class FeatrixSphereClient:
             s3_visualization_dataset: Optional S3 URL for visualization dataset for epoch projection animations (must start with 's3://')
             webhooks: Optional dict with webhook configuration keys (webhook_callback_secret, s3_backup_url, model_id_update_url)
             user_metadata: Optional user metadata for ES/SP identification (max 32KB)
+            foundation_mode: Force foundation training mode for large datasets. If True, uses foundation
+                           training (chunked iteration, SQLite-backed splits). If False, uses standard
+                           training. If None (default), auto-detects based on dataset size (>=100k rows).
 
         Returns:
             SessionInfo for the newly created embedding space session
@@ -1989,6 +1995,8 @@ class FeatrixSphereClient:
         print(f"  Validation: {s3_validation_dataset}")
         if s3_visualization_dataset:
             print(f"  Visualization: {s3_visualization_dataset}")
+        if foundation_mode is not None:
+            print(f"  Foundation mode: {'enabled' if foundation_mode else 'disabled'}")
 
         data = {
             "name": name,
@@ -2003,6 +2011,8 @@ class FeatrixSphereClient:
             import json
             data['user_metadata'] = json.dumps(user_metadata)
             print(f"User metadata: {user_metadata}")
+        if foundation_mode is not None:
+            data['foundation_mode'] = foundation_mode
         
         response_data = self._post_json("/compute/create-embedding-space", data)
         
@@ -7737,23 +7747,25 @@ class FeatrixSphereClient:
         else:
             return data
     
-    def predict_csv_file(self, session_id: str, file_path: Path) -> Dict[str, Any]:
+    def predict_csv_file(self, session_id: str, file_path: Path,
+                          best_metric_preference: str = None) -> Dict[str, Any]:
         """
         Make batch predictions on a CSV file.
-        
+
         Args:
             session_id: ID of session with trained predictor
             file_path: Path to CSV file
-            
+            best_metric_preference: Which metric checkpoint to use: "roc_auc", "pr_auc", or None (default)
+
         Returns:
             Batch prediction results
         """
         import pandas as pd
         from jsontables import JSONTablesEncoder
-        
+
         if not file_path.exists():
             raise FileNotFoundError(f"File not found: {file_path}")
-        
+
         # Support CSV, Parquet, JSON, and JSONL files
         file_path_str = str(file_path).lower()
         if file_path_str.endswith('.parquet'):
@@ -7772,29 +7784,31 @@ class FeatrixSphereClient:
             df = pd.read_json(file_path)
         else:
             df = pd.read_csv(file_path)
-        
+
         # Convert to JSON Tables format and clean NaNs
         table_data = JSONTablesEncoder.from_dataframe(df)
         cleaned_table_data = self.replace_nans_with_nulls(table_data)
-        
-        return self.predict_table(session_id, cleaned_table_data)
 
-    def run_predictions(self, session_id: str, records: List[Dict[str, Any]]) -> Dict[str, Any]:
+        return self.predict_table(session_id, cleaned_table_data, best_metric_preference=best_metric_preference)
+
+    def run_predictions(self, session_id: str, records: List[Dict[str, Any]],
+                         best_metric_preference: str = None) -> Dict[str, Any]:
         """
         Run predictions on provided records. Clean and fast for production use.
-        
+
         Args:
             session_id: ID of session with trained predictor
             records: List of record dictionaries
-            
+            best_metric_preference: Which metric checkpoint to use: "roc_auc", "pr_auc", or None (default)
+
         Returns:
             Dictionary with prediction results
         """
         # Clean NaNs for JSON encoding
         cleaned_records = self.replace_nans_with_nulls(records)
-        
+
         # Make batch predictions
-        batch_results = self.predict_records(session_id, cleaned_records)
+        batch_results = self.predict_records(session_id, cleaned_records, best_metric_preference=best_metric_preference)
         predictions = batch_results['predictions']
         
         # Process predictions into clean format
@@ -8510,32 +8524,33 @@ class FeatrixSphereClient:
         
         return cleared_counts
 
-    def predict_batch(self, session_id: str, records: List[Dict[str, Any]], 
-                     target_column: str = None) -> PredictionBatch:
+    def predict_batch(self, session_id: str, records: List[Dict[str, Any]],
+                       target_column: str = None, best_metric_preference: str = None) -> PredictionBatch:
         """
         Create a prediction batch for instant cached lookups.
-        
+
         Perfect for parameter sweeps, grid searches, and exploring prediction surfaces.
         Run your loops twice with identical code - first populates cache, second gets instant results.
-        
+
         Args:
             session_id: ID of session with trained predictor
             records: List of all records you'll want to predict on
             target_column: Specific target column predictor to use
-            
+            best_metric_preference: Which metric checkpoint to use: "roc_auc", "pr_auc", or None (default)
+
         Returns:
             PredictionBatch object with instant predict() method
-            
+
         Example:
             # Generate all combinations you'll need
             records = []
             for i in range(10):
                 for j in range(10):
                     records.append({"param1": i, "param2": j})
-            
+
             # First run - populate cache with batch processing
             batch = client.predict_batch(session_id, records)
-            
+
             # Second run - same loops but instant cache lookups
             results = []
             for i in range(10):
@@ -8545,50 +8560,52 @@ class FeatrixSphereClient:
                     results.append(result)
         """
         # Create batch object
-        batch = PredictionBatch(session_id, self, target_column)
-        
+        batch = PredictionBatch(session_id, self, target_column, best_metric_preference)
+
         # Populate cache with batch predictions
         batch._populate_cache(records)
-        
+
         return batch
 
-    def predict_grid(self, session_id: str, degrees_of_freedom: int, 
-                    grid_shape: tuple = None, target_column: str = None) -> 'PredictionGrid':
+    def predict_grid(self, session_id: str, degrees_of_freedom: int,
+                      grid_shape: tuple = None, target_column: str = None,
+                      best_metric_preference: str = None) -> 'PredictionGrid':
         """
         Create a prediction grid for exploring parameter surfaces with automatic visualization.
-        
+
         Perfect for 1D curves, 2D heatmaps, and 3D surfaces with built-in plotting functions.
-        
+
         Args:
             session_id: ID of session with trained predictor
             degrees_of_freedom: Number of dimensions (1, 2, or 3)
             grid_shape: Custom grid shape tuple (default: auto-sized)
             target_column: Specific target column predictor to use
-            
+            best_metric_preference: Which metric checkpoint to use: "roc_auc", "pr_auc", or None (default)
+
         Returns:
             PredictionGrid object with predict() and plotting methods
-            
+
         Example:
             # 2D parameter sweep with automatic plotting
             grid = client.predict_grid(session_id, degrees_of_freedom=2)
             grid.set_axis_labels(["Spend", "Campaign Type"])
             grid.set_axis_values(0, [100, 250, 500])
             grid.set_axis_values(1, ["search", "display", "social"])
-            
+
             for i, spend in enumerate([100, 250, 500]):
                 for j, campaign in enumerate(["search", "display", "social"]):
                     record = {"spend": spend, "campaign_type": campaign}
                     grid.predict(record, grid_position=(i, j))
-            
+
             # Automatic visualization
             grid.plot_heatmap()  # 2D heatmap
             grid.plot_3d()       # 3D surface
-            
+
             # Find optimal parameters
             optimal_pos = grid.get_optimal_position()
             print(f"Optimal parameters at grid position: {optimal_pos}")
         """
-        return PredictionGrid(session_id, self, degrees_of_freedom, grid_shape, target_column)
+        return PredictionGrid(session_id, self, degrees_of_freedom, grid_shape, target_column, best_metric_preference)
 
     def get_embedding_space_columns(self, session_id: str) -> Dict[str, Any]:
         """
@@ -8665,12 +8682,13 @@ class PredictionGrid:
         grid.plot_3d()       # 3D surface plot
     """
     
-    def __init__(self, session_id: str, client: 'FeatrixSphereClient', degrees_of_freedom: int, 
-                 grid_shape: tuple = None, target_column: str = None):
+    def __init__(self, session_id: str, client: 'FeatrixSphereClient', degrees_of_freedom: int,
+                 grid_shape: tuple = None, target_column: str = None, best_metric_preference: str = None):
         self.session_id = session_id
         self.client = client
         self.degrees_of_freedom = degrees_of_freedom
         self.target_column = target_column
+        self.best_metric_preference = best_metric_preference
         
         # Initialize grid matrix based on degrees of freedom
         if grid_shape:
@@ -8762,6 +8780,7 @@ class PredictionGrid:
                 session_id=self.session_id,
                 records=records_list,
                 target_column=self.target_column,
+                best_metric_preference=self.best_metric_preference,
                 show_progress_bar=show_progress
             )
             

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/featrixsphere.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.5978
+Version: 0.2.6379
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/featrixsphere.egg-info/SOURCES.txt

@@ -112,22 +112,34 @@ src/lib/vector_db.py
 src/lib/webhook_helpers.py
 src/lib/weightwatcher_tracking.py
 src/lib/featrix/__init__.py
+src/lib/featrix/auto_calibrate.py
+src/lib/featrix/download-data/build_geo_weather_db.py
+src/lib/featrix/download-data/download_geographic_data.py
+src/lib/featrix/download-data/download_road_data.py
+src/lib/featrix/download-data/download_weather_data.py
+src/lib/featrix/download-data/install-data.sh
 src/lib/featrix/neural/__init__.py
 src/lib/featrix/neural/adaptive_event_log.py
+src/lib/featrix/neural/auc_animation.py
 src/lib/featrix/neural/calibration_utils.py
 src/lib/featrix/neural/causal_relationship_scorer.py
 src/lib/featrix/neural/charting.py
 src/lib/featrix/neural/classification_metrics.py
+src/lib/featrix/neural/cluster_cohesion_tracker.py
 src/lib/featrix/neural/config.py
 src/lib/featrix/neural/curve_plots_example.py
 src/lib/featrix/neural/customer_quality_tracker.py
 src/lib/featrix/neural/data_frame_data_set.py
 src/lib/featrix/neural/dataloader_utils.py
+src/lib/featrix/neural/default_training_rules.py
 src/lib/featrix/neural/detect.py
 src/lib/featrix/neural/dimension_validator.py
 src/lib/featrix/neural/domain_codec.py
 src/lib/featrix/neural/dropout_scheduler.py
+src/lib/featrix/neural/duration_ops.py
 src/lib/featrix/neural/dynamic_relationship_extractor.py
+src/lib/featrix/neural/email_codec.py
+src/lib/featrix/neural/email_domain_ops.py
 src/lib/featrix/neural/embedded_space.py
 src/lib/featrix/neural/embedding_lr_scheduler.py
 src/lib/featrix/neural/embedding_quality.py
@@ -136,6 +148,7 @@ src/lib/featrix/neural/embedding_utils.py
 src/lib/featrix/neural/encoders.py
 src/lib/featrix/neural/enrich.py
 src/lib/featrix/neural/es_projection.py
+src/lib/featrix/neural/es_training_callbacks.py
 src/lib/featrix/neural/exceptions.py
 src/lib/featrix/neural/featrix_csv.py
 src/lib/featrix/neural/featrix_json.py
@@ -144,6 +157,9 @@ src/lib/featrix/neural/featrix_token.py
 src/lib/featrix/neural/feature_effectiveness_tracker.py
 src/lib/featrix/neural/feature_engineer.py
 src/lib/featrix/neural/feature_suggestion_tracker.py
+src/lib/featrix/neural/foundation_input_data.py
+src/lib/featrix/neural/geo_foundation.py
+src/lib/featrix/neural/geo_ops.py
 src/lib/featrix/neural/gpu_utils.py
 src/lib/featrix/neural/gradient_flow.py
 src/lib/featrix/neural/graph_encoder.py
@@ -158,10 +174,14 @@ src/lib/featrix/neural/input_data_file.py
 src/lib/featrix/neural/input_data_set.py
 src/lib/featrix/neural/integrity.py
 src/lib/featrix/neural/io_utils.py
+src/lib/featrix/neural/ip_address_ops.py
 src/lib/featrix/neural/json_cache.py
 src/lib/featrix/neural/json_codec.py
+src/lib/featrix/neural/junction_adapter.py
 src/lib/featrix/neural/local_string_cache.py
 src/lib/featrix/neural/logging_config.py
+src/lib/featrix/neural/loss_framework.py
+src/lib/featrix/neural/loss_short_embedding.py
 src/lib/featrix/neural/lr_timeline.py
 src/lib/featrix/neural/mask_bias_tracker.py
 src/lib/featrix/neural/mask_tracker.py
@@ -175,7 +195,6 @@ src/lib/featrix/neural/pair_scorer.py
 src/lib/featrix/neural/per_epoch_quality.py
 src/lib/featrix/neural/platform_utils.py
 src/lib/featrix/neural/prng_control.py
-src/lib/featrix/neural/random_utils.py
 src/lib/featrix/neural/relationship_complexity.py
 src/lib/featrix/neural/relationship_estimator.py
 src/lib/featrix/neural/relationship_extractor.py
@@ -199,32 +218,47 @@ src/lib/featrix/neural/single_predictor_mlp.py
 src/lib/featrix/neural/sphere_config.py
 src/lib/featrix/neural/sqlite_utils.py
 src/lib/featrix/neural/stopwatch.py
+src/lib/featrix/neural/strategy_scalar_scalar_ops.py
+src/lib/featrix/neural/strategy_set_scalar_ops.py
+src/lib/featrix/neural/strategy_string_scalar_ops.py
 src/lib/featrix/neural/string_analysis.py
 src/lib/featrix/neural/string_cache.py
 src/lib/featrix/neural/string_codec.py
 src/lib/featrix/neural/string_list_codec.py
+src/lib/featrix/neural/string_scalar_ops.py
+src/lib/featrix/neural/string_set_ops.py
 src/lib/featrix/neural/temporal_relationship_ops.py
 src/lib/featrix/neural/timeline_events.py
 src/lib/featrix/neural/timestamp_codec.py
 src/lib/featrix/neural/training_banner.py
 src/lib/featrix/neural/training_context_manager.py
+src/lib/featrix/neural/training_data_timeline.py
 src/lib/featrix/neural/training_event.py
 src/lib/featrix/neural/training_exceptions.py
 src/lib/featrix/neural/training_history_db.py
 src/lib/featrix/neural/training_logger.py
 src/lib/featrix/neural/training_movie_writer.py
+src/lib/featrix/neural/training_rules.py
 src/lib/featrix/neural/training_timeline.py
 src/lib/featrix/neural/transformer_encoder.py
 src/lib/featrix/neural/type_aware_ops_config.py
 src/lib/featrix/neural/url_codec.py
+src/lib/featrix/neural/url_ops.py
 src/lib/featrix/neural/url_parser.py
+src/lib/featrix/neural/us_holidays.py
 src/lib/featrix/neural/utils.py
 src/lib/featrix/neural/vector_codec.py
 src/lib/featrix/neural/webhooks.py
 src/lib/featrix/neural/weight_timeline.py
 src/lib/featrix/neural/world_data.py
+src/lib/featrix/neural/xgboost_classifier.py
 src/lib/featrix/neural/llm/__init__.py
 src/lib/featrix/neural/llm/schema_analyzer.py
+src/lib/featrix/neural/loss_functions/__init__.py
+src/lib/featrix/neural/loss_functions/list_versions.py
+src/lib/featrix/neural/loss_functions/loss_functions_01Jan2026.py
+src/lib/featrix/neural/loss_functions/loss_functions_01Jul2025.py
+src/lib/featrix/neural/loss_functions/loss_functions_21Jan2026.py
 src/lib/featrix/neural/qa/demo_advisor_decisions.py
 src/lib/featrix/neural/qa/example_complete_workflow.py
 src/lib/featrix/neural/qa/generate_focal_report.py
@@ -232,7 +266,9 @@ src/lib/featrix/neural/qa/model_advisor.py
 src/lib/featrix/neural/qa/show_results.py
 src/lib/featrix/neural/qa/test_adaptive_loss_benchmark.py
 src/lib/featrix/neural/qa/test_adaptive_training.py
+src/lib/featrix/neural/qa/test_all_codecs.py
 src/lib/featrix/neural/qa/test_checkpoint_dict_reconstruction.py
+src/lib/featrix/neural/qa/test_cls_vs_mean_pooling.py
 src/lib/featrix/neural/qa/test_confusion_matrix_metadata.py
 src/lib/featrix/neural/qa/test_data_timestamps.py
 src/lib/featrix/neural/qa/test_embedding_quality.py
@@ -257,8 +293,10 @@ src/lib/featrix/neural/qa/test_relationship_extractor_mixed_types.py
 src/lib/featrix/neural/qa/test_set_encoder_strategies.py
 src/lib/featrix/neural/qa/test_temporal_relationships.py
 src/lib/featrix/neural/qa/test_timeline_quick.py
+src/lib/featrix/neural/qa/test_training_data_timeline.py
 src/lib/featrix/neural/qa/test_training_monitor.py
 src/lib/featrix/neural/qa/test_warning_tracking.py
 src/lib/featrix/neural/qa/visualize_training_timeline.py
 src/lib/featrix/neural/tools/__init__.py
-src/lib/featrix/neural/tools/compare_clusters.py
+src/lib/featrix/neural/tools/compare_clusters.py
+tests/test_foundation_mode_local.py

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/src/api.py

@@ -3031,10 +3031,11 @@ def create_app() -> FastAPI:
                 predictor_dir = Path(predictor_path).parent
                 if predictor_dir.exists():
                     # Look for best checkpoint files
+                    # Filename format: {timestamp}_best_single_predictor_auc_roc_{value}_epoch_{N}.pickle
                     if best_metric_preference.lower() in ['roc_auc', 'roc-auc', 'auc']:
                         # Look for ROC-AUC best checkpoint
                         roc_auc_checkpoints = sorted(
-                            predictor_dir.glob("best_single_predictor_roc_auc*.pickle"),
+                            predictor_dir.glob("*best_single_predictor_auc_roc*.pickle"),
                             key=lambda p: p.stat().st_mtime,
                             reverse=True
                         )
@@ -3046,7 +3047,7 @@ def create_app() -> FastAPI:
                     elif best_metric_preference.lower() in ['pr_auc', 'pr-auc', 'precision_recall']:
                         # Look for PR-AUC best checkpoint
                         pr_auc_checkpoints = sorted(
-                            predictor_dir.glob("best_single_predictor_pr_auc*.pickle"),
+                            predictor_dir.glob("*best_single_predictor_auc_pr*.pickle"),
                             key=lambda p: p.stat().st_mtime,
                             reverse=True
                         )
@@ -4550,6 +4551,7 @@ def create_app() -> FastAPI:
         s3_file_data_set_validation: str
         s3_file_data_set_visualization: str | None = None  # Optional separate dataset for training animation projections
         user_metadata: dict | None = None  # User metadata for identification (max 32KB)
+        foundation_mode: bool | None = None  # Force foundation training mode (auto-detected if None)
 
     class CloneEmbeddingSpaceRequest(BaseModel):
         model_config = ConfigDict(
@@ -4620,6 +4622,7 @@ def create_app() -> FastAPI:
                 s3_validation_path=request.s3_file_data_set_validation,
                 s3_visualization_path=request.s3_file_data_set_visualization,
                 user_metadata=request.user_metadata,
+                foundation_mode=request.foundation_mode,
             )
 
             # CRITICAL: Save session to Redis BEFORE dispatching jobs

{featrixsphere-0.2.5978 → featrixsphere-0.2.6379}/src/celery_app.py

@@ -2122,7 +2122,46 @@ def create_structured_data(self, job_spec: dict, job_id: str, data_file: str = N
                 
                 logger.info(f"📊 Processing data file (reading, parsing, cleaning, expanding JSON columns, writing to SQLite)...")
                 logger.info(f"   This may take a while for large files - watch for progress updates below...")
-                c2d.csv_to_sqlite(csv_path=str(data_file_path))
+
+                # Check if we should use foundation mode (large datasets >= 50k rows)
+                # Foundation mode creates train/validation/test/warmup splits directly in SQLite
+                # This enables chunked iteration, SQLite-backed splits, and better memory efficiency
+                FOUNDATION_THRESHOLD = 50_000
+                foundation_mode_setting = job_spec.get("foundation_mode")  # None = auto-detect, True = force, False = disable
+                use_foundation_mode = foundation_mode_setting is True  # Explicitly True means force it
+
+                # Check row count to auto-enable foundation mode (only if not explicitly disabled)
+                if foundation_mode_setting is None:  # Auto-detect mode
+                    try:
+                        # Quick row count check based on file type
+                        file_ext = data_file_path.suffix.lower()
+                        if file_ext == '.parquet':
+                            import pyarrow.parquet as pq
+                            parquet_file = pq.ParquetFile(str(data_file_path))
+                            row_count = parquet_file.metadata.num_rows
+                        elif file_ext in ['.csv', '.gz']:
+                            # For CSV, estimate row count from file size (rough estimate)
+                            # More accurate: count newlines, but that's slow for large files
+                            # Use file size heuristic: ~100 bytes per row on average
+                            estimated_rows = int(file_size_mb * 1024 * 1024 / 100)
+                            row_count = estimated_rows
+                        else:
+                            # For JSON/JSONL, estimate similarly
+                            estimated_rows = int(file_size_mb * 1024 * 1024 / 200)
+                            row_count = estimated_rows
+
+                        logger.info(f"   Estimated row count: {row_count:,}")
+                        if row_count >= FOUNDATION_THRESHOLD:
+                            logger.info(f"🏛️  Large dataset detected ({row_count:,} rows >= {FOUNDATION_THRESHOLD:,}) - enabling foundation mode")
+                            use_foundation_mode = True
+                    except Exception as row_count_err:
+                        logger.warning(f"   Could not estimate row count: {row_count_err} - using standard mode")
+
+                if use_foundation_mode:
+                    logger.info(f"🏛️  Using FOUNDATION MODE for large dataset processing")
+                    c2d.csv_to_foundation_sqlite(csv_path=str(data_file_path))
+                else:
+                    c2d.csv_to_sqlite(csv_path=str(data_file_path))
 
                 logger.info(f"✅ STEP 2/3 COMPLETE: CSV to SQLite conversion finished")