dslighting 1.7.11__py3-none-any.whl → 1.7.20__py3-none-any.whl

dslighting/__init__.py

@@ -26,7 +26,7 @@ Advanced Usage:
 For more information, see: https://github.com/usail-hkust/dslighting
 """
 
-__version__ = "1.7.11"
+__version__ = "1.7.20"
 __author__ = "DSLighting Team"
 
 # Core API classes

dslighting/core/agent.py

@@ -393,17 +393,134 @@ class Agent:
 
                             # Create simple wrapper class
                             class SimpleMLEBenchmark:
-                                def __init__(self, registry_instance):
+                                def __init__(self, registry_instance, logger, registry_dir, data_dir, task_id):
                                     self.registry = registry_instance
                                     self.problems = [{"competition_id": task_id}]
+                                    self.logger = logger
+                                    self.registry_dir = registry_dir
+                                    self.data_dir = data_dir
+                                    self.task_id = task_id
 
                                 async def grade(self, submission_path):
                                     """Grade submission using custom registry."""
                                     try:
-                                        competition = self.registry.get_competition(task_id)
+                                        # Read config directly from registry
+                                        import yaml
+                                        config_path = self.registry_dir / self.task_id / "config.yaml"
+
+                                        if not config_path.exists():
+                                            self.logger.warning(f"  Config not found: {config_path}")
+                                            return 0.0
+
+                                        with open(config_path) as f:
+                                            config = yaml.safe_load(f)
+
+                                        # Resolve paths relative to data_dir (from config.yaml)
+                                        # self.data_dir is the parent directory (e.g., /path/to/competitions)
+                                        # config["dataset"]["answers"] is relative path like "bike-sharing-demand/prepared/private/test_answer.csv
+                                        answers_rel_path = config.get("dataset", {}).get("answers", "")
+                                        answers_path = self.data_dir / answers_rel_path
+
+                                        # **MANDATORY**: Check for prepared/public and prepared/private structure
+                                        competition_dir = self.data_dir / self.task_id
+                                        prepared_public_dir = competition_dir / "prepared" / "public"
+                                        prepared_private_dir = competition_dir / "prepared" / "private"
+
+                                        if not prepared_public_dir.exists():
+                                            self.logger.error(f"  ❌ Required directory not found: {prepared_public_dir}")
+                                            self.logger.error(f"  ❌ Tasks must have prepared/public/ directory structure")
+                                            self.logger.error(f"  See: https://github.com/usail-hkust/dslighting for setup instructions")
+                                            return 0.0
+
+                                        if not prepared_private_dir.exists():
+                                            self.logger.error(f"  ❌ Required directory not found: {prepared_private_dir}")
+                                            self.logger.error(f"  ❌ Tasks must have prepared/private/ directory structure")
+                                            self.logger.error(f"  See: https://github.com/usail-hkust/dslighting for setup instructions")
+                                            return 0.0
+
+                                        self.logger.info(f"  ✓ Required structure verified:")
+                                        self.logger.info(f"    - prepared/public: {prepared_public_dir}")
+                                        self.logger.info(f"    - prepared/private: {prepared_private_dir}")
+
+                                        if not answers_path.exists():
+                                            self.logger.warning(f"  Answers file not found: {answers_path}")
+                                            self.logger.warning(f"  Looking for: {answers_path}")
+                                            return 0.0
+
+                                        self.logger.info(f"  ✓ Found answers file: {answers_path}")
+
+                                        # Import the actual Competition class from mlebench
+                                        from mlebench.registry import Competition
+                                        from mlebench.grade_helpers import Grader
+
+                                        # Load grader
+                                        grader_config = config.get("grader", {})
+                                        grader_name = grader_config.get("name", "rmsle")
+
+                                        # Import grade function if specified
+                                        grade_fn = None
+                                        if "grade_fn" in grader_config:
+                                            # Parse grade_fn format: mlebench.competitions.bike_sharing_demand.grade:grade
+                                            fn_str = grader_config["grade_fn"]
+                                            if ":" in fn_str:
+                                                module_path, fn_name = fn_str.rsplit(":", 1)
+                                                # Convert to file import if needed
+                                                if not module_path.startswith("file:"):
+                                                    fn_file = self.registry_dir / self.task_id / "grade.py"
+                                                    if fn_file.exists():
+                                                        fn_str = f"file:{fn_file}:{fn_name}"
+                                                else:
+                                                    # Try to import from mlebench
+                                                    try:
+                                                        import importlib
+                                                        importlib.import_module(module_path)
+                                                    except:
+                                                        pass
+
+                                        # Create a simple grader
+                                        if grade_fn or fn_str:
+                                            grader = Grader(
+                                                name=grader_name,
+                                                grade_fn=fn_str if fn_str else grade_fn,
+                                            )
+                                        else:
+                                            # Default RMSLE grader
+                                            grader = Grader(name="rmsle", grade_fn=None)
+
+                                        # Resolve paths - use actual prepared directories (already verified above)
+                                        raw_dir = competition_dir / "raw"
+                                        checksums = competition_dir / "checksums.txt"
+                                        leaderboard = competition_dir / "leaderboard.csv"
+                                        # Use the actual prepared directories that we verified exist
+                                        private_dir = prepared_private_dir
+                                        public_dir = prepared_public_dir
+
+                                        # Create placeholder prepare_fn
+                                        def dummy_prepare_fn(a, b, c):
+                                            return private_dir
+
+                                        # Create actual Competition object with all required fields
+                                        simple_comp = Competition(
+                                            id=config["id"],
+                                            name=config["name"],
+                                            description=config.get("description", ""),
+                                            grader=grader,
+                                            answers=answers_path,
+                                            gold_submission=answers_path,  # Use same as answers for grading
+                                            sample_submission=public_dir / "sampleSubmission.csv",
+                                            competition_type=config.get("competition_type", "standard"),
+                                            prepare_fn=dummy_prepare_fn,
+                                            raw_dir=raw_dir,
+                                            private_dir=private_dir,
+                                            public_dir=public_dir,
+                                            checksums=checksums,
+                                            leaderboard=leaderboard,
+                                        )
+
+                                        # Grade using mlebench's grade_csv
                                         report = grade_csv(
                                             submission_path,
-                                            competition,
+                                            simple_comp,
                                         )
                                         # Return the score (float), not the entire report
                                         score = report.score if report.score is not None else 0.0
@@ -411,9 +528,11 @@ class Agent:
                                         return score
                                     except Exception as e:
                                         self.logger.warning(f"  Grading failed: {e}")
+                                        import traceback
+                                        self.logger.warning(f"  Traceback: {traceback.format_exc()}")
                                         return 0.0
 
-                            benchmark = SimpleMLEBenchmark(custom_registry)
+                            benchmark = SimpleMLEBenchmark(custom_registry, self.logger, registry_dir_path, data_dir_path, task_id)
                             runner = self.get_runner()
                             runner.benchmark = benchmark
                             print(f"[DEBUG 13] ✓ Benchmark set successfully for task: {task_id}")

dslighting/datasets/__init__.py

@@ -44,6 +44,8 @@ def load_bike_sharing_demand_raw() -> dict:
     This returns the raw data files as DataFrames in a dictionary.
     Use this if you want to access the data directly.
 
+    Note: Data files are in prepared/public and prepared/private structure.
+
     Returns:
         Dictionary with keys:
             - 'train': Training DataFrame
@@ -67,12 +69,16 @@ def load_bike_sharing_demand_raw() -> dict:
         )
 
     data = {}
-    data['train'] = pd.read_csv(data_path / "train.csv")
-    data['test'] = pd.read_csv(data_path / "test.csv")
-    data['sample_submission'] = pd.read_csv(data_path / "sampleSubmission.csv")
+    # Load from prepared/public directory
+    public_dir = data_path / "prepared" / "public"
+    private_dir = data_path / "prepared" / "private"
+
+    data['train'] = pd.read_csv(public_dir / "train.csv")
+    data['test'] = pd.read_csv(public_dir / "test.csv")
+    data['sample_submission'] = pd.read_csv(public_dir / "sampleSubmission.csv")
 
-    # Load test_answer if available
-    test_answer_path = data_path / "test_answer.csv"
+    # Load test_answer from prepared/private
+    test_answer_path = private_dir / "test_answer.csv"
     if test_answer_path.exists():
         data['test_answer'] = pd.read_csv(test_answer_path)
 
@@ -83,11 +89,11 @@ def load_bike_sharing_demand(data_dir: Optional[str] = None) -> dict:
     """
     Load the Bike Sharing Demand dataset as a ready-to-use competition.
 
-    This creates the proper MLE-Bench directory structure (prepared/public/)
-    in the specified or a temporary directory, and returns the path information.
+    The dataset already follows the MLE-Bench standard structure with
+    prepared/public and prepared/private directories.
 
     Args:
-        data_dir: Base data directory. If None, uses a temporary directory.
+        data_dir: Base data directory. If None, uses the built-in dataset location.
 
     Returns:
         Dictionary with keys:
@@ -108,30 +114,33 @@ def load_bike_sharing_demand(data_dir: Optional[str] = None) -> dict:
         ...     data_dir=str(info['data_dir'].parent)
         ... )
     """
-    # Load raw data
+    # Load raw data (from prepared/public and prepared/private structure)
     raw_data = load_bike_sharing_demand_raw()
 
-    # Create directory structure
+    # Determine data directory
     if data_dir is None:
-        # Create a temporary directory
-        data_dir = Path(tempfile.mkdtemp(prefix="bike_sharing_"))
+        # Use built-in dataset location
+        competition_dir = get_data_path() / "bike-sharing-demand"
     else:
+        # Create directory structure in specified location
         data_dir = Path(data_dir)
+        competition_dir = data_dir / "bike-sharing-demand"
 
-    # Create competition structure
-    competition_dir = data_dir / "bike-sharing-demand"
-    prepared_dir = competition_dir / "prepared" / "public"
-    prepared_dir.mkdir(parents=True, exist_ok=True)
+        # Create prepared/public and prepared/private structure
+        prepared_dir = competition_dir / "prepared" / "public"
+        prepared_dir.mkdir(parents=True, exist_ok=True)
+
+        # Copy data files
+        raw_data['train'].to_csv(prepared_dir / "train.csv", index=False)
+        raw_data['test'].to_csv(prepared_dir / "test.csv", index=False)
+        raw_data['sample_submission'].to_csv(prepared_dir / "sampleSubmission.csv", index=False)
 
-    # Save data files
-    raw_data['train'].to_csv(prepared_dir / "train.csv", index=False)
-    raw_data['test'].to_csv(prepared_dir / "test.csv", index=False)
-    raw_data['sample_submission'].to_csv(prepared_dir / "sampleSubmission.csv", index=False)
+        # Create private directory with answers
+        private_dir = competition_dir / "prepared" / "private"
+        private_dir.mkdir(parents=True, exist_ok=True)
+        raw_data['test_answer'].to_csv(private_dir / "test_answer.csv", index=False)
 
-    # Create private directory with answers
-    private_dir = competition_dir / "prepared" / "private"
-    private_dir.mkdir(parents=True, exist_ok=True)
-    raw_data['test_answer'].to_csv(private_dir / "test_answer.csv", index=False)
+    prepared_dir = competition_dir / "prepared" / "public"
 
     return {
         'task_id': 'bike-sharing-demand',

{dslighting-1.7.11.dist-info → dslighting-1.7.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dslighting
-Version: 1.7.11
+Version: 1.7.20
 Summary: Simplified API for Data Science Agent Automation
 Author: DSLighting Team
 License: AGPL-3.0

{dslighting-1.7.11.dist-info → dslighting-1.7.20.dist-info}/RECORD

@@ -66,17 +66,17 @@ dsat/workflows/search/aide_workflow.py,sha256=mxIGXcueZGXpv1RXsQJ0YPWtvzICaFQeJo
 dsat/workflows/search/automind_workflow.py,sha256=b2JzqUDnDOt_SQdtAvC0fBCJzgTadLylbpgmpaS63Ls,12573
 dsat/workflows/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dsat/workflows/templates/basic_kaggle_loop.py,sha256=e6YLEpCArgWfKViwoti7SdygHsHp43sqP6VyMqnOJaA,3128
-dslighting/__init__.py,sha256=1dOz7QKnMou-NnK_6tHiXgcdkh82rZR3aVsw0iokoVY,5153
+dslighting/__init__.py,sha256=joE_kvnmYWLrHWSePpyTJKwbLy641BO2xPKN_3P_qMA,5153
 dslighting/core/__init__.py,sha256=T4yYs0RQoz6DBarjOk12PeZq9YoPYrfl3Os0CPlzcB0,252
-dslighting/core/agent.py,sha256=sUbbLaScP3lqQRMuiqbRn_v_zFzcy9q9Ot3khEyR9dY,31530
+dslighting/core/agent.py,sha256=40f4D33Jba4_GOhDbKT9GfyOmQOz7XjB9lIzLTqp_LE,40009
 dslighting/core/config_builder.py,sha256=JMDW0JFj6PRCeP70D1Td8FYXC1bIupe4H6m8J1caO7Y,10294
 dslighting/core/data_loader.py,sha256=HdcDN-10FYfViu8Z0RSYjA2ne6VXVbfKEOZO7EpxLIc,25003
 dslighting/core/task_detector.py,sha256=xOYAV9yiboC8lDeDIEtxvucaGi6fENfeycrowWs-kP0,16300
-dslighting/datasets/__init__.py,sha256=I0Ctrt9wGhHnjzTAp8mVqoo5IQIAq0qaLWYMMOWsXzg,5355
-dslighting/datasets/bike-sharing-demand/sampleSubmission.csv,sha256=WMJ5URg7EEN9Z1LPW6xwiun4BJDZkcJP923ilvccsvI,142861
-dslighting/datasets/bike-sharing-demand/test.csv,sha256=-paigmJ0767Po8ANQlbRFpQlaZB0Xg_OwqEE6Fq1e-M,109461
-dslighting/datasets/bike-sharing-demand/test_answer.csv,sha256=A-tTMmqn6094FzXHn4bv73xurV5rZD4GKstCpVh8LSk,51199
-dslighting/datasets/bike-sharing-demand/train.csv,sha256=SIaNazhqAdVtsRZmMzLdLOw8IDAs5H_FeLSHhwU2vdE,521358
+dslighting/datasets/__init__.py,sha256=Imn7lnXzrkDkNeL6xrSZziUC9Z7X_3uwvemTJEXeBqc,5812
+dslighting/datasets/bike-sharing-demand/prepared/private/test_answer.csv,sha256=A-tTMmqn6094FzXHn4bv73xurV5rZD4GKstCpVh8LSk,51199
+dslighting/datasets/bike-sharing-demand/prepared/public/sampleSubmission.csv,sha256=WMJ5URg7EEN9Z1LPW6xwiun4BJDZkcJP923ilvccsvI,142861
+dslighting/datasets/bike-sharing-demand/prepared/public/test.csv,sha256=-paigmJ0767Po8ANQlbRFpQlaZB0Xg_OwqEE6Fq1e-M,109461
+dslighting/datasets/bike-sharing-demand/prepared/public/train.csv,sha256=SIaNazhqAdVtsRZmMzLdLOw8IDAs5H_FeLSHhwU2vdE,521358
 dslighting/registry/README.md,sha256=SFAmvPqFyl2dm1mLd3-r94qW9DaIq84OZeQd8wIsw04,12208
 dslighting/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dslighting/registry/utils.py,sha256=cPkDpfTrO9RQD0As2YF64GuFjJyjMsYHzdc1v8sJ5go,9808
@@ -2105,7 +2105,7 @@ mlebench/README.md,sha256=tyV4Y8FWJ1ZhJYcEoNYhP2jcndJ9dTbQNyIV6Ubf7TU,3027
 mlebench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mlebench/cli.py,sha256=aJzQRGExprtwRGW5l2WVnmYST7ZiEXehxd6IOMf3pYg,7877
 mlebench/data.py,sha256=IUMRgf9zD-A4fQqH4jPR-4d2KPBiUsVM0I9f2FgubKQ,14462
-mlebench/grade.py,sha256=ykG_Ga6dJHtTEoMVX9lVhyPySe2b5iER5MzQ7zgonlQ,8877
+mlebench/grade.py,sha256=TAfw4-IsGHDtKwaYvti697UXiGH0yPkx5FHXJpwRDvk,10826
 mlebench/grade_helpers.py,sha256=ILRjLFBXnRylsY8bxpFns_RsNRwX52qQ90qdQs0hn7Q,9380
 mlebench/metrics.py,sha256=s0Om2rKXJ9hyQYWnh_G8gLAGwKBZxHM01VdP3ZaVH54,2494
 mlebench/registry.py,sha256=8ZLBULzX6p4DrRH3SsWfUIP_Cdk8E_olIJDpHlbkaWw,14335
@@ -2446,8 +2446,8 @@ mlebench/competitions/vinbigdata-chest-xray-abnormalities-detection/prepare_val.
 mlebench/competitions/whale-categorization-playground/grade.py,sha256=Wl1fNvpapAmc_Cdy2Cp68nEqeHB2XKnN91U9t-YLlXQ,1562
 mlebench/competitions/whale-categorization-playground/prepare.py,sha256=huo8fDBcGR413JF8m8Js8l8gkFCZpWL1st7Yd57Rjg8,4199
 mlebench/competitions/whale-categorization-playground/prepare_val.py,sha256=XvdnOvEJyTxovXQsoKLMKA_J6EMdxkpOe_SIgEdoOx8,7486
-dslighting-1.7.11.dist-info/METADATA,sha256=9A-hFT5_ik3lAppLTl3zcRtnzgMVeMYdjiRECbPqFL8,18326
-dslighting-1.7.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dslighting-1.7.11.dist-info/entry_points.txt,sha256=1xqIWL9-EG9t7nkERVeNRtMoRyIQe-6CJZJN-rdCYFQ,91
-dslighting-1.7.11.dist-info/top_level.txt,sha256=cmZU3ri0tz1IjiTKrz85Ih9614QlCrYqSqzlPMgaSM0,25
-dslighting-1.7.11.dist-info/RECORD,,
+dslighting-1.7.20.dist-info/METADATA,sha256=zr0RDfx3rAYi-9fgMzscOXLkG4cp_AB_WcKkkmiySBQ,18326
+dslighting-1.7.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dslighting-1.7.20.dist-info/entry_points.txt,sha256=1xqIWL9-EG9t7nkERVeNRtMoRyIQe-6CJZJN-rdCYFQ,91
+dslighting-1.7.20.dist-info/top_level.txt,sha256=cmZU3ri0tz1IjiTKrz85Ih9614QlCrYqSqzlPMgaSM0,25
+dslighting-1.7.20.dist-info/RECORD,,

mlebench/grade.py

@@ -73,9 +73,61 @@ def grade_csv(path_to_submission: Path, competition: Competition) -> Competition
         )
 
     valid_submission = score is not None
-    competition_leaderboard = get_leaderboard(competition)
-    rank_info = competition.grader.rank_score(score, competition_leaderboard)
-    is_lower_better = competition.grader.is_lower_better(competition_leaderboard)
+
+    # Helper function to determine is_lower_better from metric name
+    def infer_is_lower_better(metric_name: str) -> bool:
+        """
+        Infer whether lower is better based on metric name.
+        Returns True if lower is better (error metrics), False if higher is better (accuracy metrics).
+        """
+        metric_name_lower = metric_name.lower()
+
+        # Error/loss metrics (lower is better)
+        error_metrics = [
+            'rmse', 'rmsle', 'mae', 'mse', 'mape',
+            'log_loss', 'crossentropy', 'kld', 'error',
+            'loss', 'distance', 'deviation'
+        ]
+
+        # Accuracy/score metrics (higher is better)
+        accuracy_metrics = [
+            'accuracy', 'precision', 'recall', 'f1',
+            'auc', 'roc', 'score', 'r2', 'correlation',
+            'iou', 'dice', 'map', 'ndcg'
+        ]
+
+        for error_metric in error_metrics:
+            if error_metric in metric_name_lower:
+                return True
+
+        for accuracy_metric in accuracy_metrics:
+            if accuracy_metric in metric_name_lower:
+                return False
+
+        # Default: assume lower is better for safety
+        return True
+
+    # Try to get leaderboard, but make it optional
+    try:
+        competition_leaderboard = get_leaderboard(competition)
+        rank_info = competition.grader.rank_score(score, competition_leaderboard)
+        is_lower_better = competition.grader.is_lower_better(competition_leaderboard)
+    except (AssertionError, FileNotFoundError):
+        # Leaderboard not found, infer is_lower_better from grader name
+        logger.warning(f"Leaderboard not found for competition {competition.id}, skipping rank calculation")
+        competition_leaderboard = None
+        rank_info = {
+            "gold_threshold": None,
+            "silver_threshold": None,
+            "bronze_threshold": None,
+            "median_threshold": None,
+            "gold_medal": False,
+            "silver_medal": False,
+            "bronze_medal": False,
+            "above_median": False,
+        }
+        # Infer is_lower_better from grader name
+        is_lower_better = infer_is_lower_better(competition.grader.name)
 
     return CompetitionReport(
         competition_id=competition.id,