dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/multi-modal-gesture-recognition/prepare_val.py

@@ -0,0 +1,139 @@
+import random
+import shutil
+from pathlib import Path
+
+import pandas as pd
+
+
+def _create_split(
+    source_df: pd.DataFrame,
+    test_tar_name: str,
+    train_tar_names: list[str],
+    raw_dir: Path,
+    public_dir: Path,
+    private_dir: Path,
+) -> pd.DataFrame:
+    """
+    Helper function to perform a data split based on specified tar files.
+
+    It unpacks a given test tarball to identify test sample IDs, splits the
+    source dataframe into train/test sets, creates all necessary public and private
+    CSV files, copies the relevant data tarballs, and returns the newly created
+    training dataframe for potential subsequent splits.
+    """
+    # Unpack the test file to get test IDs
+    # The ID is the last 4 digits of the sample filename (e.g., "0300" from "Sample00300.zip")
+    test_data_dir_name = test_tar_name.replace(".tar.gz", "")
+    shutil.unpack_archive(raw_dir / test_tar_name, raw_dir / test_data_dir_name)
+    test_ids = sorted([fp.stem[-4:] for fp in (raw_dir / test_data_dir_name).glob("*.zip")])
+
+    # Create the new training dataframe for this split
+    new_training_df = source_df[~source_df["Id"].isin(test_ids)]
+    new_training_df.to_csv(public_dir / "training.csv", index=False)
+    assert len(new_training_df) == len(source_df) - len(test_ids)
+
+    # Make private answers
+    answers_df = source_df[source_df["Id"].isin(test_ids)]
+    answers_df.to_csv(private_dir / "test.csv", index=False)
+    assert len(answers_df) == len(test_ids)
+
+    # Make new public test.csv (IDs only)
+    test_df = pd.DataFrame({"Id": test_ids})
+    test_df.to_csv(public_dir / "test.csv", index=False)
+    assert len(test_df) == len(test_ids)
+
+    # Make new public randomPredictions.csv
+    # predictions are random shufflings of numbers 1-20 (no repeats)
+    random.seed(0)
+    preds = []
+    for _ in range(len(test_ids)):
+        pred = " ".join(str(x) for x in random.sample(range(1, 21), 20))
+        preds.append(pred)
+    random_predictions_df = pd.DataFrame({"Id": test_ids, "Sequence": preds})
+    random_predictions_df.to_csv(public_dir / "randomPredictions.csv", index=False)
+    assert len(random_predictions_df) == len(test_ids)
+
+    # Copy over the designated test set tarball
+    shutil.copyfile(src=raw_dir / test_tar_name, dst=public_dir / "test.tar.gz")
+
+    # Copy over the designated training tarballs for this split
+    for file in train_tar_names:
+        shutil.copyfile(src=raw_dir / file, dst=public_dir / file)
+
+    return new_training_df
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+
+    Raw dataset has:
+    - Train: training1, training2, training3, training4
+    - Val: validation1, validation2, validation3 (no labels)
+    - Test: (not available)
+
+    New prepared dataset has:
+    - Train: training1, training2, training3
+    - Val: validation1, validation2, validation3 (no labels)
+    - Test: training4 (renamed to `test.tar.gz`)
+
+    Furthermore,
+    - We modify the `training.csv` to remove training4 samples
+    - We modify the `test.csv` and `randomPredictions.csv` to include only training4 IDs
+
+    No other changes. We copy over the remaining files (devel01-40.7z, valid_all_files_combined.7z, sample_code_mmrgc.zip) as-is.
+    """
+    # --- Setup new directories for the validation split ---
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+
+    # Load the complete training data manifest
+    full_training_df = pd.read_csv(raw / "training.csv", dtype={"Id": str, "Sequence": str})
+
+    # --- 1. Create the original public/private split ---
+    # This split is identical to the original script's behavior.
+    # Train set: training1, 2, 3. Test set: training4.
+    original_train_tars = ["training1.tar.gz", "training2.tar.gz", "training3.tar.gz"]
+    original_test_tar = "training4.tar.gz"
+
+    train_df_for_val_split = _create_split(
+        source_df=full_training_df,
+        test_tar_name=original_test_tar,
+        train_tar_names=original_train_tars,
+        raw_dir=raw,
+        public_dir=public,
+        private_dir=private,
+    )
+
+    # Copy over validation and other miscellaneous files to the 'public' directory
+    files_to_copy = [
+        "validation1.tar.gz",
+        "validation2.tar.gz",
+        "validation3.tar.gz",
+        "devel01-40.7z",
+        "valid_all_files_combined.7z",
+        "sample_code_mmrgc.zip",
+    ]
+    for file in files_to_copy:
+        shutil.copyfile(src=raw / file, dst=public / file)
+
+    # --- 2. Create the new public_val/private_val split ---
+    # This second split uses the training data from the first split as its source.
+    # New train set: training1, 2. New test (validation) set: training3.
+    val_train_tars = ["training1.tar.gz", "training2.tar.gz"]
+    val_test_tar = "training3.tar.gz"
+
+    _create_split(
+        source_df=train_df_for_val_split,
+        test_tar_name=val_test_tar,
+        train_tar_names=val_train_tars,
+        raw_dir=raw,
+        public_dir=public_val,
+        private_dir=private_val,
+    )
+
+    # Copy over validation and other files to 'public_val' to mirror the 'public' structure
+    for file in files_to_copy:
+        shutil.copyfile(src=raw / file, dst=public_val / file)

dslighting/registry/my-custom-task-01/config.yaml

@@ -0,0 +1,15 @@
+competition_type: generic
+dataset:
+  answers: my-custom-task-01/prepared/private/answer.csv
+  sample_submission: my-custom-task-01/prepared/public/sample.csv
+description: description.md
+grader:
+  grade_fn: mlebench.grade:simple_accuracy_grader
+id: my-custom-task-01
+metric: RMSE
+name: Bike Sharing Demand Prediction
+preparer: file:/Users/liufan/Applications/Github/dslighting/benchmarks/mlebench/competitions/my-custom-task-01/prepare.py:prepare
+private_dir: my-custom-task-01/prepared/private
+public_dir: my-custom-task-01/prepared/public
+raw_dir: my-custom-task-01/raw
+task_type: regression

dslighting/registry/my-custom-task-01/description.md

@@ -0,0 +1,51 @@
+# Bike Sharing Demand Prediction
+
+## Description
+Predict bike rental counts based on temporal, weather, and seasonal features using historical rental data from a bike sharing system. The goal is to forecast the total number of bikes rented (`count`) during each hour period, which is crucial for inventory management, station planning, and optimizing bike redistribution strategies.
+
+## Evaluation
+The competition uses **Root Mean Squared Logarithmic Error (RMSLE)** as the evaluation metric:
+
+```
+RMSLE = sqrt(1/n * Σ(log(p_i + 1) - log(a_i + 1))²)
+```
+
+Where:
+- `p_i` is the predicted count
+- `a_i` is the actual count
+- `n` is the number of observations
+- `log` is the natural logarithm
+
+This metric is robust to outliers and penalizes underestimates more than overestimates, which is important for business planning.
+
+## Submission Format
+Submissions should be a CSV file with two columns: `datetime` and `count`. The file should contain predictions for all instances in the test set.
+
+```csv
+datetime,count
+2011-07-19 11:00:00,127
+2011-07-19 12:00:00,89
+2012-01-16 00:00:00,13
+...
+```
+
+## Dataset Description
+The dataset contains hourly bike rental data split into training and test sets:
+- **Training set**: 5,000 hourly records with complete features and target variables
+- **Test set**: 2,178 hourly records with features only (targets provided separately for validation)
+
+## Data Fields
+- `datetime`: Hourly date + timestamp (format: YYYY-MM-DD HH:MM:SS)
+- `season`: Season (1: spring, 2: summer, 3: fall, 4: winter)
+- `holiday`: Whether day is holiday (1: holiday, 0: not holiday)
+- `workingday`: Whether day is a working day (1: working day, 0: weekend/holiday)
+- `weather`: Weather situation (1: clear, 2: mist, 3: light rain/snow, 4: heavy rain/snow)
+- `temp`: Temperature in Celsius
+- `atemp`: "Feels like" temperature in Celsius
+- `humidity`: Relative humidity (%)
+- `windspeed`: Wind speed
+- `casual`: Number of non-registered user rentals (training set only)
+- `registered`: Number of registered user rentals (training set only)
+- `count`: Total number of bike rentals (target variable)
+
+The training set contains all fields, while the test set excludes `casual`, `registered`, and `count` columns.

dslighting/registry/my-custom-task-01/prepare.py

@@ -0,0 +1,2 @@
+from pathlib import Path
+def prepare(r, pub, priv): return pub

dslighting/registry/my-custom-task-01/report.md

@@ -0,0 +1,74 @@
+# Report: Bike Sharing Demand Prediction
+
+
+
+--- COMPREHENSIVE DATA REPORT ---
+
+## Directory Structure (Current Working Directory)
+```text
+./
+├── sample.csv
+├── sampleSubmission.csv
+├── test.csv
+├── test_answer.csv
+└── train.csv
+```
+
+## Data Schema Analysis
+### Analysis of `sample.csv`
+
+```text
+         Data Type  Missing (%)  Cardinality                             Sample Values
+datetime    object          0.0         2178  ['2011-07-19 11:00:00', '2012-01-16 0...
+count        int64          0.0            1                                    [0, 0]
+```
+
+### Analysis of `sampleSubmission.csv`
+
+```text
+         Data Type  Missing (%)  Cardinality                             Sample Values
+datetime    object          0.0         5000  ['2011-01-20 00:00:00', '2011-01-20 0...
+count        int64          0.0            1                                    [0, 0]
+```
+
+### Analysis of `test.csv`
+
+```text
+           Data Type  Missing (%)  Cardinality                             Sample Values
+datetime      object          0.0         2178  ['2011-07-19 11:00:00', '2012-01-16 0...
+season         int64          0.0            4                                    [3, 1]
+holiday        int64          0.0            2                                    [0, 1]
+workingday     int64          0.0            2                                    [1, 0]
+weather        int64          0.0            3                                    [1, 1]
+temp         float64          0.0           47                              [33.62, 4.1]
+atemp        float64          0.0           57                             [40.15, 6.82]
+humidity       int64          0.0           80                                  [59, 54]
+windspeed    float64          0.0           24                             [0.0, 6.0032]
+```
+
+### Analysis of `test_answer.csv`
+
+```text
+         Data Type  Missing (%)  Cardinality                             Sample Values
+datetime    object          0.0         2178  ['2011-07-19 11:00:00', '2012-01-16 0...
+count        int64          0.0          582                                 [127, 13]
+```
+
+### Analysis of `train.csv`
+
+```text
+           Data Type  Missing (%)  Cardinality                             Sample Values
+datetime      object          0.0         5000  ['2011-07-06 05:00:00', '2012-08-04 1...
+season         int64          0.0            4                                    [3, 3]
+holiday        int64          0.0            2                                    [0, 0]
+workingday     int64          0.0            2                                    [1, 0]
+weather        int64          0.0            3                                    [1, 1]
+temp         float64          0.0           48                             [27.88, 36.9]
+atemp        float64          0.0           59                            [31.82, 40.91]
+humidity       int64          0.0           82                                  [83, 39]
+windspeed    float64          0.0           26                         [6.0032, 19.9995]
+casual         int64          0.0          258                                  [5, 197]
+registered     int64          0.0          625                                 [30, 253]
+count          int64          0.0          715                                 [35, 450]
+```
+

dslighting/registry/new-my-task-01/config.yaml

@@ -0,0 +1,15 @@
+competition_type: generic
+dataset:
+  answers: new-my-task-01/prepared/private/answer.csv
+  sample_submission: new-my-task-01/prepared/public/sample.csv
+description: description.md
+grader:
+  grade_fn: mlebench.grade:simple_accuracy_grader
+id: new-my-task-01
+metric: RMSE
+name: Bike Sharing Demand Prediction
+preparer: file:/Users/liufan/Applications/Github/dslighting/benchmarks/mlebench/competitions/new-my-task-01/prepare.py:prepare
+private_dir: new-my-task-01/prepared/private
+public_dir: new-my-task-01/prepared/public
+raw_dir: new-my-task-01/raw
+task_type: regression

dslighting/registry/new-my-task-01/description.md

@@ -0,0 +1,5 @@
+# Bike Sharing Demand Prediction
+
+Predict bike rental counts based on temporal, weather, and seasonal features using training data with datetime, weather conditions, and rental counts.
+
+**Goal:** Predict `count`.

dslighting/registry/new-my-task-01/prepare.py

@@ -0,0 +1,2 @@
+from pathlib import Path
+def prepare(r, pub, priv): return pub

dslighting/registry/new-my-task-01/report.md

@@ -0,0 +1,120 @@
+# Comprehensive Analysis Report: Bike Sharing Demand Prediction
+
+## Executive Summary
+
+This analysis examines bike rental patterns using historical data containing temporal, weather, and seasonal features. The dataset reveals strong cyclical patterns driven by commuting behavior, weather conditions, and seasonal variations. Understanding these patterns is crucial for optimizing bike inventory management, maintenance scheduling, and marketing strategies for bike-sharing services.
+
+## Key Findings
+
+### 1. Temporal Patterns Drive Usage Behavior
+
+![Hourly Rental Pattern](http://localhost:8002/outputs/dsat_run_new_my_task_01_4a56206a/sandbox/visualizations/hourly_trend.png)
+**Hourly Rental Pattern Analysis**
+The data reveals a clear bimodal distribution with distinct peaks during morning (8 AM) and evening (5-6 PM) commute hours, indicating that bike sharing serves primarily as a transportation solution for daily commuters. The lowest usage occurs during early morning hours (1-4 AM), suggesting limited recreational or late-night usage. This pattern highlights the importance of positioning bike-sharing as a reliable commuting alternative.
+
+### 2. Day Type Influences Rental Volume
+
+![Day Type Comparison](http://localhost:8002/outputs/dsat_run_new_my_task_01_4a56206a/sandbox/visualizations/day_type_comparison.png)
+**Day Type Comparison**
+Working days show slightly higher average rentals than holidays and weekends, reinforcing the commuting hypothesis. This suggests that bike-sharing services should focus on reliability and availability during weekday peak hours while potentially developing weekend-specific promotions to boost off-peak usage.
+
+### 3. Weather Conditions Significantly Impact Demand
+
+![Weather Impact Analysis](http://localhost:8002/outputs/dsat_run_new_my_task_01_4a56206a/sandbox/visualizations/weather_impact.png)
+**Weather Impact Analysis**
+Clear weather conditions significantly increase bike rentals, while adverse weather (rain/snow) dramatically reduces rental activity by up to 50-70%. This finding underscores the need for dynamic pricing strategies and proactive maintenance scheduling based on weather forecasts to optimize revenue and resource allocation.
+
+### 4. Environmental Factors Show Strong Correlations
+
+![Correlation Analysis](http://localhost:8002/outputs/dsat_run_new_my_task_01_4a56206a/sandbox/visualizations/correlation_matrix.png)
+**Correlation Analysis**
+Temperature demonstrates the strongest positive correlation with rental counts (r ≈ 0.40), indicating that warmer weather encourages bike usage. Humidity shows moderate negative correlation, suggesting discomfort discourages riding. The high correlation between temperature and "feels like" temperature (atemp) indicates potential multicollinearity that should be addressed in modeling.
+
+### 5. Seasonal Variations Reveal Clear Patterns
+
+![Seasonal Patterns Analysis](http://localhost:8002/outputs/dsat_run_new_my_task_01_4a56206a/sandbox/visualizations/seasonal_patterns.png)
+**Seasonal Patterns Analysis**
+Monthly trends show highest usage in summer months (June-August), with seasonal analysis confirming that summer and fall have the highest rental activity. This seasonal pattern suggests the need for strategic inventory management, with increased bike availability during peak seasons and maintenance scheduling during off-peak periods.
+
+### 6. Target Variable Distribution
+
+![Target Variable Distribution Analysis](http://localhost:8002/outputs/dsat_run_new_my_task_01_4a56206a/sandbox/visualizations/dist_target.png)
+**Target Variable Distribution Analysis**
+The original distribution shows a right-skewed pattern, which is common for count data. The log-transformed distribution demonstrates better modeling properties, suggesting that logarithmic transformations or Poisson-based models may yield superior predictive performance.
+
+## Business Implications
+
+1. **Operational Planning**: The strong commuting patterns indicate that bike availability should be optimized around peak hours (7-9 AM and 4-7 PM) to maximize utilization.
+2. **Revenue Management**: Weather-dependent pricing strategies could help mitigate revenue loss during adverse conditions while maximizing profits during favorable weather.
+3. **Marketing Strategy**: Targeted promotions during weekends and off-peak hours could help balance demand throughout the day and week.
+4. **Maintenance Scheduling**: Seasonal patterns suggest optimal maintenance windows during winter months when demand is lowest.
+
+## Recommendations
+
+1. **Model Development**: Use time-series aware models that capture hourly, daily, and seasonal patterns while accounting for weather conditions.
+2. **Feature Engineering**: Create derived features such as time-of-day indicators, weather severity indices, and holiday proximity metrics.
+3. **Evaluation Strategy**: Implement time-based cross-validation to ensure model robustness across different seasons and conditions.
+4. **Business Applications**: Develop a demand forecasting system that integrates weather forecasts to optimize bike redistribution and staffing.
+
+The analysis confirms that bike-sharing demand is highly predictable when considering temporal, weather, and seasonal factors, providing a solid foundation for developing accurate forecasting models and data-driven business strategies.
+
+
+
+## 🏁 Final Run Summary
+
+### 📊 Performance
+- **Score**: 0.0000
+- **Cost**: $0.0172
+
+### 💡 Methodology Summary
+The solution employs a Random Forest Regressor to predict bike rental counts. The methodology begins with crucial data preprocessing, where the raw datetime string is converted into a datetime object. This allows for effective feature engineering by extracting multiple temporal components: hour, day, month, and year. These new features are created to help the model capture patterns related to time, such as daily commuting cycles, seasonal variations, and yearly trends, which are critical for accurate demand forecasting.
+
+The model selection centers on a Random Forest Regressor, chosen for its ability to handle complex, non-linear relationships between the engineered features and the target variable. After training the model on all features, predictions are post-processed to ensure they are non-negative, as negative rental counts are impossible. Finally, the model's performance is evaluated using the Root Mean Squared Log Error (RMSLE) on the training data, a metric well-suited for this task as it penalizes underestimates more heavily than overestimates and handles large value ranges effectively.
+
+### 🧩 Final Solution Code
+
+```python
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_squared_log_error
+import numpy as np
+
+# Load data
+train_df = pd.read_csv('train.csv')
+test_df = pd.read_csv('test.csv')
+
+# Parse datetime and extract temporal features
+for df in [train_df, test_df]:
+    df['datetime'] = pd.to_datetime(df['datetime'])
+    df['hour'] = df['datetime'].dt.hour
+    df['day'] = df['datetime'].dt.day
+    df['month'] = df['datetime'].dt.month
+    df['year'] = df['datetime'].dt.year
+
+# Prepare features and target
+feature_cols = ['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp', 'humidity', 'windspeed', 'hour', 'day', 'month', 'year']
+X_train = train_df[feature_cols]
+y_train = train_df['count']
+X_test = test_df[feature_cols]
+
+# Train model
+model = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
+model.fit(X_train, y_train)
+
+# Make predictions
+y_pred = model.predict(X_test)
+y_pred = np.where(y_pred < 0, 0, y_pred)  # Ensure non-negative predictions
+
+# Create submission file
+submission_df = pd.DataFrame({
+    'datetime': test_df['datetime'],
+    'count': y_pred
+})
+submission_df.to_csv('submission_new-my-task-01_17ff77.csv', index=False)
+
+# Print validation score on training data
+train_pred = model.predict(X_train)
+train_pred = np.where(train_pred < 0, 0, train_pred)
+rmsle = np.sqrt(mean_squared_log_error(y_train, train_pred))
+print(f"Training RMSLE: {rmsle:.4f}")
+```

dslighting/registry/new-my-task-02/config.yaml

@@ -0,0 +1,9 @@
+competition_type: standard_ml
+description: description.md
+id: new-my-task-02
+metric: Mean Squared Error
+name: Tennis Match Flow and Performance Analysis
+private_dir: new-my-task-02/prepared/private
+public_dir: new-my-task-02/prepared/public
+raw_dir: new-my-task-02/raw
+task_type: Time Series Analysis

dslighting/registry/new-my-task-02/description.md

@@ -0,0 +1,3 @@
+# Task Initialization: new-my-task-02
+
+- `[22:01:15]` Upload complete. Ready for EDA.

dslighting/registry/new-my-task-02/report.md

@@ -0,0 +1,76 @@
+# Initial Data Scan: Tennis Match Flow and Performance Analysis
+
+
+
+--- COMPREHENSIVE DATA REPORT ---
+
+## Directory Structure (Current Working Directory)
+```text
+./
+├── Wimbledon_featured_matches.csv
+├── data_dictionary.csv
+└── sampleSubmission.csv
+```
+
+## Data Schema Analysis
+### Analysis of `data_dictionary.csv`
+
+```text
+            Data Type  Missing (%)  Cardinality                             Sample Values
+variables      object          0.0           46                   ['match_id', 'player1']
+explanation    object          0.0           46  ['match identification', 'first and l...
+example        object          0.0           24  ['2023-wimbledon-1701 ("7" is the rou...
+```
+
+### Analysis of `sampleSubmission.csv`
+
+```text
+                   Data Type  Missing (%)  Cardinality                             Sample Values
+match_id              object          0.0            1  ['2023-wimbledon-1301', '2023-wimbled...
+player1               object          0.0            1      ['Carlos Alcaraz', 'Carlos Alcaraz']
+player2               object          0.0            1        ['Nicolas Jarry', 'Nicolas Jarry']
+elapsed_time          object          0.0            5                  ['00:00:00', '00:00:38']
+set_no                 int64          0.0            1                                    [1, 1]
+game_no                int64          0.0            1                                    [1, 1]
+point_no               int64          0.0            5                                    [1, 2]
+p1_sets                int64          0.0            1                                    [0, 0]
+p2_sets                int64          0.0            1                                    [0, 0]
+p1_games               int64          0.0            1                                    [0, 0]
+p2_games               int64          0.0            1                                    [0, 0]
+p1_score               int64          0.0            3                                    [0, 0]
+p2_score               int64          0.0            3                                   [0, 15]
+server                 int64          0.0            1                                    [1, 1]
+serve_no               int64          0.0            2                                    [2, 1]
+point_victor           int64          0.0            2                                    [2, 1]
+p1_points_won          int64          0.0            4                                    [0, 1]
+p2_points_won          int64          0.0            2                                    [1, 1]
+game_victor            int64          0.0            1                                    [0, 0]
+set_victor             int64          0.0            1                                    [0, 0]
+p1_ace                 int64          0.0            2                                    [0, 0]
+p2_ace                 int64          0.0            1                                    [0, 0]
+p1_winner              int64          0.0            2                                    [0, 0]
+p2_winner              int64          0.0            1                                    [0, 0]
+winner_shot_type      object          0.0            2                                ['0', '0']
+p1_double_fault        int64          0.0            1                                    [0, 0]
+p2_double_fault        int64          0.0            1                                    [0, 0]
+p1_unf_err             int64          0.0            2                                    [1, 0]
+p2_unf_err             int64          0.0            1                                    [0, 0]
+p1_net_pt              int64          0.0            1                                    [0, 0]
+p2_net_pt              int64          0.0            2                                    [0, 0]
+p1_net_pt_won          int64          0.0            1                                    [0, 0]
+p2_net_pt_won          int64          0.0            1                                    [0, 0]
+p1_break_pt            int64          0.0            1                                    [0, 0]
+p2_break_pt            int64          0.0            1                                    [0, 0]
+p1_break_pt_won        int64          0.0            1                                    [0, 0]
+p2_break_pt_won        int64          0.0            1                                    [0, 0]
+p1_break_pt_missed     int64          0.0            1                                    [0, 0]
+p2_break_pt_missed     int64          0.0            1                                    [0, 0]
+p1_distance_run      float64          0.0            5                              [6.0, 5.253]
+p2_distance_run      float64          0.0            5                             [7.84, 7.094]
+rally_count            int64          0.0            4                                    [2, 1]
+speed_mph              int64          0.0            5                                 [95, 118]
+serve_width           object          0.0            4                               ['BC', 'B']
+serve_depth           object          0.0            2                           ['NCTL', 'CTL']
+return_depth          object         20.0            2                              ['ND', 'ND']
+```
+

dslighting/registry/new-my-task-03/config.yaml

@@ -0,0 +1,16 @@
+competition_type: open_ended
+dataset:
+  answers: new-my-task-03/raw/answer.csv
+  gold_submission: new-my-task-03/raw/answer.csv
+  sample_submission: new-my-task-03/raw/sampleSubmission.csv
+description: description.md
+grader:
+  grade_fn: file:/Users/liufan/Applications/Github/dslighting/benchmarks/mlebench/competitions/new-my-task-03/grade.py:grade
+  name: LLMJudge
+id: new-my-task-03
+metric: LLM Score
+name: Tennis Match Momentum and Performance Flow Analysis
+preparer: file:/Users/liufan/Applications/Github/dslighting/benchmarks/mlebench/competitions/new-my-task-03/prepare.py:prepare
+private_dir: new-my-task-03/prepared/private
+public_dir: new-my-task-03/prepared/public
+raw_dir: new-my-task-03/raw

dslighting/registry/new-my-task-03/description.md

@@ -0,0 +1 @@
+Develop a model that captures the flow of play as points occur and apply it to one or more of the matches. Your model should identify which player is performing better at a given time in the match, as well as how much better they are performing. Provide a visualization based on your model to depict the match flow. Note: in tennis, the player serving has a much higher probability of winning the point/game. You may wish to factor this into your model in some way.

dslighting/registry/new-my-task-03/grade.py

@@ -0,0 +1,107 @@
+import os
+from pathlib import Path
+from typing import Any
+import pandas as pd
+
+# This is a generic LLM-based grader for open-ended tasks.
+# It reads 'rubric.md' from the task directory and evaluates the submission.
+
+try:
+    from dsat.services.llm import LLMService
+    from dsat.config import LLMConfig
+except ImportError:
+    # Fallback for when running outside of dsat package context
+    import sys
+    sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent))
+    from dsat.services.llm import LLMService
+    from dsat.config import LLMConfig
+
+class Report:
+    def __init__(self, score, feedback):
+        self.score = score
+        self.feedback = feedback
+        # Standard fields expected by the framework
+        self.is_lower_better = False
+        self.submission_exists = True
+        self.valid_submission = True
+        self.gold_medal = score >= 0.9
+        self.silver_medal = score >= 0.7
+        self.bronze_medal = score >= 0.5
+        self.above_median = score >= 0.5
+        self.submission_path = ""
+        self.competition_id = "open_ended_task"
+
+def grade(submission_path: Path, competition: Any) -> Report:
+    """
+    Grades the submission using an LLM Judge against rubric.md.
+    """
+    # 1. Load the Rubric
+    task_dir = competition.raw_dir.parent
+    rubric_path = task_dir / "rubric.md"
+    
+    if not rubric_path.exists():
+        # Fallback if no rubric exists
+        print(f"Warning: Rubric not found at {rubric_path}. Returning default score.")
+        return Report(0.5, "No grading rubric defined.")
+        
+    rubric_content = rubric_path.read_text(encoding="utf-8")
+    
+    # 2. Load the Submission Content (Preview)
+    # Since it's open-ended, the 'submission_path' might be a CSV, code, or just a marker.
+    # We'll try to peek at the output artifacts if possible, or assume the agent's recent work 
+    # is what we are grading. Ideally, AIDE produces a submission file.
+    
+    submission_content = "No submission content readable."
+    if submission_path.exists():
+        try:
+            if submission_path.suffix == '.csv':
+                df = pd.read_csv(submission_path)
+                submission_content = f"CSV Submission Preview:\n{df.head().to_markdown()}"
+            else:
+                submission_content = submission_path.read_text(encoding="utf-8")[:2000]
+        except Exception as e:
+            submission_content = f"Error reading submission: {e}"
+
+    # 3. Setup LLM for Judging
+    # Note: In a real run, we might want to inject the API key securely.
+    # Here we assume environment variables are set (which they are in DSATRunner).
+    try:
+        api_key = os.getenv("API_KEY", "EMPTY")
+        base_url = os.getenv("API_BASE", "https://api.openai.com/v1")
+        model = os.getenv("LLM_MODEL", "gpt-4o")
+        
+        llm = LLMService(LLMConfig(api_key=api_key, api_base=base_url, model=model))
+        
+        prompt = f"""You are an impartial Judge. Evaluate the following submission against the provided Rubric.
+
+# RUBRIC
+{rubric_content}
+
+# SUBMISSION CONTENT
+{submission_content}
+
+# INSTRUCTION
+Assess the submission. 
+Output ONLY a float number between 0.0 and 1.0 on the first line.
+On subsequent lines, provide brief feedback.
+"""
+        # Synchronous call wrapper or direct call if possible. 
+        # Since grade() is synchronous in standard mlebench, we need a way to run async code.
+        import asyncio
+        response = asyncio.run(llm.achat([{"role": "user", "content": prompt}]))
+        
+        lines = response.strip().split('\n')
+        try:
+            score = float(lines[0].strip())
+        except ValueError:
+            # Fallback if LLM is chatty
+            import re
+            match = re.search(r"(\d+(\.\d+)?)", lines[0])
+            score = float(match.group(1)) if match else 0.5
+            
+        feedback = "\n".join(lines[1:])
+        return Report(score, feedback)
+
+    except Exception as e:
+        print(f"LLM Judging failed: {e}")
+        return Report(0.0, f"Judging failed: {e}")

dslighting/registry/new-my-task-03/prepare.py

@@ -0,0 +1,2 @@
+def prepare(raw, public, private):
+    pass