hkjc 0.3.15__py3-none-any.whl → 0.3.17__py3-none-any.whl

hkjc/analysis.py

@@ -0,0 +1,3 @@
+# TODO:
+
+# Generate filtered live odds, fav run style, dr, current rating, season start rating, track record

hkjc/historical.py

@@ -10,7 +10,10 @@ from cachetools.func import ttl_cache
 from .utils import _parse_html_table
 
 HKJC_RACE_URL_TEMPLATE = "https://racing.hkjc.com/racing/information/English/Racing/LocalResults.aspx?RaceDate={date}&Racecourse={venue_code}&RaceNo={race_number}"
-HKJC_HORSE_URL_TEMPLATE = "https://racing.hkjc.com/racing/information/English/Horse/Horse.aspx?HorseId={horse_id}"
+HKJC_HORSE_URL_TEMPLATE = "https://racing.hkjc.com/racing/information/English/Horse/Horse.aspx?HorseNo={horse_no}"
+
+incidents = ['DISQ', 'DNF', 'FE', 'ML', 'PU', 'TNP', 'TO',
+             'UR', 'VOID', 'WR', 'WV', 'WV-A', 'WX', 'WX-A', 'WXNR']
 
 
 @ttl_cache(maxsize=100, ttl=3600)
@@ -30,14 +33,13 @@ def _soupify_race_page(date: str, venue_code: str, race_number: int) -> Beautifu
     return _soupify(url)
 
 
-def _soupify_horse_page(horse_id: str) -> BeautifulSoup:
+def _soupify_horse_page(horse_no: str) -> BeautifulSoup:
     """Fetch and parse HKJC race results page and return BeautifulSoup object
     """
-    url = HKJC_HORSE_URL_TEMPLATE.format(horse_id=horse_id)
+    url = HKJC_HORSE_URL_TEMPLATE.format(horse_no=horse_no)
     return _soupify(url)
 
 
-
 def _classify_running_style(df: pl.DataFrame, running_pos_col="RunningPosition") -> pl.DataFrame:
     """Classify running style based on RunningPosition column
     """
@@ -50,25 +52,27 @@ def _classify_running_style(df: pl.DataFrame, running_pos_col="RunningPosition")
         .alias("split_data").cast(pl.Int64, strict=False)
     ).unnest("split_data")
 
+    df = df.with_columns(pl.col('FinishPosition').fill_null(pl.col('Position3')))
+
     df = df.with_columns([
         (pl.col("StartPosition")-pl.col("FinishPosition")).alias("PositionChange"),
         pl.mean_horizontal("StartPosition", "Position2",
                            "Position3", "FinishPosition").alias("AvgPosition"),
     ]).with_columns(pl.when(pl.col("StartPosition").is_null()).then(pl.lit("--"))
-                    .when((pl.col("PositionChange") <= 0) & pl.col("StartPosition") <= 3).then(pl.lit("FrontRunner"))
+                    .when((pl.col("AvgPosition") <= 3.5) & (pl.col("StartPosition") <= 3)).then(pl.lit("FrontRunner"))
                     .when((pl.col("PositionChange") >= 1) & (pl.col("StartPosition") >= 6)).then(pl.lit("Closer"))
                     .otherwise(pl.lit("Pacer")).alias("RunningStyle"))
 
-    recent_style = df['RunningStyle'][:10].mode()[0]
+    recent_style = df['RunningStyle'][:5].mode()[0]
     df = df.with_columns(pl.lit(recent_style).alias("FavoriteRunningStyle"))
 
     return df
 
 
-def _extract_horse_data(horse_id: str) -> pl.DataFrame:
+def _extract_horse_data(horse_no: str) -> pl.DataFrame:
     """Extract horse info and history from horse page
     """
-    soup = _soupify_horse_page(horse_id)
+    soup = _soupify_horse_page(horse_no)
     table = soup.find('table', class_='bigborder')
     horse_data = _parse_html_table(table).filter(
         pl.col('Date') != '')  # Remove empty rows
@@ -78,35 +82,113 @@ def _extract_horse_data(horse_id: str) -> pl.DataFrame:
     table = soup.find_all('table', class_='table_eng_text')
     profile_data = _parse_html_table(table[0], skip_header=True)
     profile_data = _parse_html_table(table[1], skip_header=True)
-    current_rating = profile_data.filter(pl.col("column_0").str.starts_with("Current Rating"))['column_2'].item(0)
-    season_start_rating = profile_data.filter(pl.col("column_0").str.starts_with("Start of Season Rating"))['column_2'].item(0)
+
+    try:
+        current_rating = int(profile_data.filter(
+            pl.col("column_0").str.starts_with("Current Rating"))['column_2'].item(0))
+        season_start_rating = int(profile_data.filter(pl.col(
+            "column_0").str.starts_with("Start of Season Rating"))['column_2'].item(0))
+    except:
+        current_rating, season_start_rating = 0, 0
+
+    try:
+        last_rating = int(profile_data.filter(
+            pl.col("column_0").str.starts_with("Last Rating"))['column_2'].item(0))
+    except:
+        last_rating = 0
 
     horse_info = {
-        'HorseID': horse_id,
-        'CurrentRating': int(current_rating),
-        'SeasonStartRating': int(season_start_rating)
+        'HorseID': horse_no,
+        'CurrentRating': current_rating,
+        'SeasonStartRating': season_start_rating,
+        'LastRating': last_rating if current_rating == 0 else current_rating
     }
     horse_data = (horse_data.with_columns([
         pl.lit(value).alias(key) for key, value in horse_info.items()
     ])
     )
+
     return horse_data
 
 
+def _clean_horse_data(df: pl.DataFrame) -> pl.DataFrame:
+    """ Clean and convert horse data to suitable data types
+    """
+    df = df.with_columns(
+        pl.col('Pla').str.split(' ').list.first().alias('Pla')
+    ).filter(~pl.col('Pla').is_in(incidents))
+
+    df = df.with_columns([
+        pl.col('Pla').cast(pl.Int64, strict=False),
+        pl.col('ActWt').cast(pl.Int64, strict=False),
+        pl.col('DeclarHorseWt').cast(pl.Int64, strict=False),
+        pl.col('Dr').cast(pl.Int64, strict=False),
+        pl.col('Rtg').cast(pl.Int64, strict=False),
+        pl.col('Dist').cast(pl.Int64, strict=False),
+        pl.col('WinOdds').cast(pl.Float64, strict=False),
+        pl.col('RaceIndex').cast(pl.Int64, strict=False)
+    ])
+
+    df = df.with_columns(
+        (
+            pl.col("FinishTime").str.split_exact(".", 1).struct.field("field_0").cast(pl.Int64) * 60 +
+            pl.col("FinishTime").str.split_exact(".", 1).struct.field("field_1").cast(pl.Int64)
+        ).cast(pl.Float64).alias("FinishTime")
+    )
+
+    df = df.with_columns(
+        pl.col('RCTrackCourse').str.split_exact(' / ', 2)
+        .struct.rename_fields(['Venue', 'Track', 'Course'])
+        .alias('RCTrackCourse')
+    ).unnest('RCTrackCourse')
+
+    return df
+
+def get_horse_data(horse_no: str) -> pl.DataFrame:
+    df = _extract_horse_data(horse_no)
+    return _clean_horse_data(df)
+
+def _clean_race_data(df: pl.DataFrame) -> pl.DataFrame:
+    """ Clean and convert horse data to suitable data types
+    """
+    df = df.with_columns(
+        pl.col('Pla').str.split(' ').list.first().alias('Pla')
+    ).filter(~pl.col('Pla').is_in(incidents))
+
+    df = df.with_columns([
+        pl.col('Pla').cast(pl.Int64, strict=False),
+        pl.col('HorseNo').cast(pl.Int64, strict=False),
+        pl.col('ActWt').cast(pl.Int64, strict=False),
+        pl.col('DeclarHorseWt').cast(pl.Int64, strict=False),
+        pl.col('Dr').cast(pl.Int64, strict=False),
+        pl.col('WinOdds').cast(pl.Float64, strict=False)
+    ])
+
+    df = df.with_columns(
+        (
+            pl.col("FinishTime").str.split_exact(":", 1).struct.field("field_0").cast(pl.Int64) * 60 +
+            pl.col("FinishTime").str.split_exact(":", 1).struct.field("field_1").cast(pl.Int64)
+        ).cast(pl.Float64).alias("FinishTime")
+    )
+
+    return df
+
 def _extract_race_data(date: str, venue_code: str, race_number: int) -> pl.DataFrame:
     soup = _soupify_race_page(date, venue_code, race_number)
     table = soup.find('div', class_='race_tab').find('table')
     race_data = _parse_html_table(table)
 
     # Extract the relevant race information
+    race_id = race_data.columns[0].replace(f'RACE{race_number}', '')
     race_class = race_data.item(1, 0).split('-')[0].strip()
     race_dist = race_data.item(1, 0).split('-')[1].strip().rstrip('M')
     race_name = race_data.item(2, 0).strip()
     going = race_data.item(1, 2).strip()
     course = race_data.item(2, 2).strip()
 
-    race_info = {'RaceDate': date,
+    race_info = {'Date': date,
                  'Venue': venue_code,
+                 'RaceIndex': int(race_id),
                  'RaceNumber': race_number,
                  'RaceClass': race_class,
                  'RaceDistance': race_dist,
@@ -120,25 +202,15 @@ def _extract_race_data(date: str, venue_code: str, race_number: int) -> pl.DataF
                  .with_columns([
                      pl.lit(value).alias(key) for key, value in race_info.items()
                  ])
-                 )
-
-    # Extract horse IDs from links
-    horse_ids = []
-    rows = table.find_all('tr')[1:]  # Skip header row
-    for row in rows:
-        horse_id = 'UNKNOWN'  # Horse link not found
-        links = row.find_all('a')
-        for link in links:
-            if 'href' in link.attrs and 'HorseId=' in link['href']:
-                horse_id = link['href'].split('HorseId=')[1]
-                break
-        horse_ids.append(horse_id)
-
-    race_data = race_data.with_columns(pl.Series('HorseID', horse_ids))
-
-    # Join with horse data
-    horse_data_list = [_extract_horse_data(horse_id) for horse_id in horse_ids]
-    horse_data_df = pl.concat(horse_data_list).unique(subset=['HorseID'])
-    race_data = race_data.join(horse_data_df, on='HorseID', how='left')
-
-    return race_data
+                 .with_columns(
+                     pl.col("Horse").str.extract(r"\((.*?)\)")
+                     .alias("HorseID")
+    )
+    )
+
+    return race_data
+
+
+def get_race_data(date: str, venue_code: str, race_number: int) -> pl.DataFrame:
+    df = _extract_race_data(date,venue_code,race_number)
+    return _clean_race_data(df)

hkjc/processing.py

@@ -6,7 +6,7 @@ from typing import Tuple, List, Union
 from .live_odds import live_odds
 from .strategy import qpbanker, place_only
 from .harville_model import fit_harville_to_odds
-from .historical import _extract_race_data
+from .historical import _extract_horse_data, _extract_race_data, _clean_horse_data
 from .utils import _validate_date
 
 import polars as pl
@@ -14,7 +14,6 @@ import numpy as np
 from itertools import combinations
 from tqdm import tqdm
 from datetime import datetime as dt
-from joblib import delayed, Parallel
 
 
 def _all_subsets(lst): return [list(x) for r in range(
@@ -24,17 +23,17 @@ def _all_subsets(lst): return [list(x) for r in range(
 # ======================================
 # Historical data processing functions
 # ======================================
-incidents = ['DISQ', 'DNF', 'FE', 'ML', 'PU', 'TNP', 'TO',
-             'UR', 'VOID', 'WR', 'WV', 'WV-A', 'WX', 'WX-A', 'WXNR']
+
 
 
 def _historical_process_single_date_venue(date: str, venue_code: str) -> List[pl.DataFrame]:
     dfs = []
-    iter_date = tqdm(range(1, 12), desc=f"Processing {date} {venue_code} ...", leave=False)
+    iter_date = tqdm(
+        range(1, 12), desc=f"Processing {date} {venue_code} ...", leave=False)
     for race_number in iter_date:
         try:
             dfs.append(_extract_race_data(date.strftime('%Y/%m/%d'),
-                                          venue_code, race_number))
+                                     venue_code, race_number))
         except:
             if race_number == 1:
                 iter_date.close()
@@ -51,38 +50,20 @@ def generate_historical_data(start_date: str, end_date: str) -> pl.DataFrame:
 
     dfs = []
 
-    for date in tqdm(pl.date_range(start_dt, end_dt, interval='1d', eager=True)):
+    for date in tqdm(pl.date_range(start_dt, end_dt, interval='1d', eager=True), leave=False, desc='Scanning for horse IDs ...'):
         for venue_code in ['ST', 'HV']:
             dfs += _historical_process_single_date_venue(date, venue_code)
 
     if dfs == []:
-        raise ValueError("Failed to obtain any race data. This could be due to invalid date range, or server requests limit. Please try again later.")
-
-    df = (pl.concat(dfs)
-          .filter(~pl.col('Pla').is_in(incidents))
-          .with_columns(
-        pl.col('Pla').str.split(' ').list.first().alias('Pla')
-    )
-    )
-
-    df = df.with_columns([
-        pl.col('Pla').cast(pl.Int64, strict=False),
-        pl.col('HorseNo').cast(pl.Int64, strict=False),
-        pl.col('ActWt').cast(pl.Int64, strict=False),
-        pl.col('DeclarHorseWt').cast(pl.Int64, strict=False),
-        pl.col('Dr').cast(pl.Int64, strict=False),
-        pl.col('RaceDistance').cast(pl.Int64, strict=False),
-        pl.col('WinOdds').cast(pl.Float64, strict=False)
-    ])
-
-    df = df.with_columns(
-        (
-            pl.col("FinishTime").str.split(":").list.get(0).cast(pl.Int64) * 60 +
-            pl.col("FinishTime").str.split(":").list.get(1).cast(pl.Float64)
-        ).cast(pl.Float64).alias("FinishTime")
-    )
+        raise ValueError(
+            "Failed to obtain any race data. This could be due to invalid date range, or server requests limit. Please try again later.")
 
-    return df
+    horse_ids = pl.concat(dfs)['HorseID'].unique()
+
+    # Use horse track records
+    dfs = [_extract_horse_data(horse_id) for horse_id in tqdm(horse_ids, desc='Processing horses ...', leave=False)]
+    df = pl.concat(dfs)
+    return _clean_horse_data(df)
 
 
 # ==========================

{hkjc-0.3.15.dist-info → hkjc-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hkjc
-Version: 0.3.15
+Version: 0.3.17
 Summary: Library for scrapping HKJC data and perform basic analysis
 Requires-Python: >=3.11
 Requires-Dist: beautifulsoup4>=4.14.2

{hkjc-0.3.15.dist-info → hkjc-0.3.17.dist-info}/RECORD

@@ -1,13 +1,14 @@
 hkjc/__init__.py,sha256=TI7PVhmoWSvYX-xdTEdaT3jfY99LiYQFRQZaIwBhJd8,785
+hkjc/analysis.py,sha256=0042_NMIkQCl0J6B0P4TFfrBDCnm2B6jsCZKOEO30yI,108
 hkjc/harville_model.py,sha256=MZjPLS-1nbEhp1d4Syuq13DtraKnd7TlNqBmOOCwxgc,15976
-hkjc/historical.py,sha256=FRECc4pmozjFKkFeWN0vTzECF9QOS7URyJoCfTt5hlw,5805
+hkjc/historical.py,sha256=yQQAx8vlr2EqcPazpYp1x2ku7dy3imQoDWImHCRv1QA,8330
 hkjc/live_odds.py,sha256=G4ELBBp1d2prxye9kKzu2pwtS4vSfRPOmEuT7-Nd-3A,4741
-hkjc/processing.py,sha256=XeVrF5KKkU3Oy-vqPvMgM22QHVTCVCuml2IsIGdRbYw,7483
+hkjc/processing.py,sha256=xrvEUgu_jz8ZxevOsRsYz0T7pWyNtSCMI6LUYByOLOw,6812
 hkjc/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hkjc/speedpro.py,sha256=Y2Z3GYGeePc4sM-ZnCHXCI1N7L-_j9nrMqS3CC5BBSo,2031
 hkjc/utils.py,sha256=4CA_FPf_U3GvzoLkqBX0qDPZgrSvKJKvbP7VWqd5FiA,6323
 hkjc/strategy/place_only.py,sha256=lHPjTSj8PzghxncNBg8FI4T4HJigekB9a3bV7l7VtPA,2079
 hkjc/strategy/qpbanker.py,sha256=MQxjwsfhllKZroKS8w8Q3bi3HMjGc1DAyBIjNZAp3yQ,4805
-hkjc-0.3.15.dist-info/METADATA,sha256=2nQL1EImJ0hXnWFdGnpORIknMdaaFham-Pw9cgjUiO4,481
-hkjc-0.3.15.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hkjc-0.3.15.dist-info/RECORD,,
+hkjc-0.3.17.dist-info/METADATA,sha256=gKSkXKYo_HCg2S4ZeAjnqZniWV0V2kGpRH_g25K9Rmo,481
+hkjc-0.3.17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hkjc-0.3.17.dist-info/RECORD,,