hkjc 0.3.16__py3-none-any.whl → 0.3.17__py3-none-any.whl

hkjc/historical.py

@@ -12,6 +12,9 @@ from .utils import _parse_html_table
 HKJC_RACE_URL_TEMPLATE = "https://racing.hkjc.com/racing/information/English/Racing/LocalResults.aspx?RaceDate={date}&Racecourse={venue_code}&RaceNo={race_number}"
 HKJC_HORSE_URL_TEMPLATE = "https://racing.hkjc.com/racing/information/English/Horse/Horse.aspx?HorseNo={horse_no}"
 
+incidents = ['DISQ', 'DNF', 'FE', 'ML', 'PU', 'TNP', 'TO',
+             'UR', 'VOID', 'WR', 'WV', 'WV-A', 'WX', 'WX-A', 'WXNR']
+
 
 @ttl_cache(maxsize=100, ttl=3600)
 def _soupify(url: str) -> BeautifulSoup:
@@ -37,7 +40,6 @@ def _soupify_horse_page(horse_no: str) -> BeautifulSoup:
     return _soupify(url)
 
 
-
 def _classify_running_style(df: pl.DataFrame, running_pos_col="RunningPosition") -> pl.DataFrame:
     """Classify running style based on RunningPosition column
     """
@@ -50,14 +52,14 @@ def _classify_running_style(df: pl.DataFrame, running_pos_col="RunningPosition")
         .alias("split_data").cast(pl.Int64, strict=False)
     ).unnest("split_data")
 
-    df.with_columns(pl.col('FinishPosition').fill_null(pl.col('Position3')))
+    df = df.with_columns(pl.col('FinishPosition').fill_null(pl.col('Position3')))
 
     df = df.with_columns([
         (pl.col("StartPosition")-pl.col("FinishPosition")).alias("PositionChange"),
         pl.mean_horizontal("StartPosition", "Position2",
                            "Position3", "FinishPosition").alias("AvgPosition"),
     ]).with_columns(pl.when(pl.col("StartPosition").is_null()).then(pl.lit("--"))
-                    .when((pl.col("PositionChange") <= 0) & pl.col("StartPosition") <= 3).then(pl.lit("FrontRunner"))
+                    .when((pl.col("AvgPosition") <= 3.5) & (pl.col("StartPosition") <= 3)).then(pl.lit("FrontRunner"))
                     .when((pl.col("PositionChange") >= 1) & (pl.col("StartPosition") >= 6)).then(pl.lit("Closer"))
                     .otherwise(pl.lit("Pacer")).alias("RunningStyle"))
 
@@ -67,7 +69,7 @@ def _classify_running_style(df: pl.DataFrame, running_pos_col="RunningPosition")
     return df
 
 
-def get_horse_data(horse_no: str) -> pl.DataFrame:
+def _extract_horse_data(horse_no: str) -> pl.DataFrame:
     """Extract horse info and history from horse page
     """
     soup = _soupify_horse_page(horse_no)
@@ -82,13 +84,16 @@ def get_horse_data(horse_no: str) -> pl.DataFrame:
     profile_data = _parse_html_table(table[1], skip_header=True)
 
     try:
-        current_rating = int(profile_data.filter(pl.col("column_0").str.starts_with("Current Rating"))['column_2'].item(0))
-        season_start_rating = int(profile_data.filter(pl.col("column_0").str.starts_with("Start of Season Rating"))['column_2'].item(0))
+        current_rating = int(profile_data.filter(
+            pl.col("column_0").str.starts_with("Current Rating"))['column_2'].item(0))
+        season_start_rating = int(profile_data.filter(pl.col(
+            "column_0").str.starts_with("Start of Season Rating"))['column_2'].item(0))
     except:
         current_rating, season_start_rating = 0, 0
-    
+
     try:
-        last_rating = int(profile_data.filter(pl.col("column_0").str.starts_with("Last Rating"))['column_2'].item(0))
+        last_rating = int(profile_data.filter(
+            pl.col("column_0").str.starts_with("Last Rating"))['column_2'].item(0))
     except:
         last_rating = 0
 
@@ -96,47 +101,85 @@ def get_horse_data(horse_no: str) -> pl.DataFrame:
         'HorseID': horse_no,
         'CurrentRating': current_rating,
         'SeasonStartRating': season_start_rating,
-        'LastRating' : last_rating if current_rating==0 else current_rating
+        'LastRating': last_rating if current_rating == 0 else current_rating
     }
     horse_data = (horse_data.with_columns([
         pl.lit(value).alias(key) for key, value in horse_info.items()
     ])
     )
 
-    horse_data = horse_data.with_columns([
+    return horse_data
+
+
+def _clean_horse_data(df: pl.DataFrame) -> pl.DataFrame:
+    """ Clean and convert horse data to suitable data types
+    """
+    df = df.with_columns(
+        pl.col('Pla').str.split(' ').list.first().alias('Pla')
+    ).filter(~pl.col('Pla').is_in(incidents))
+
+    df = df.with_columns([
         pl.col('Pla').cast(pl.Int64, strict=False),
-        pl.col('WinOdds').cast(pl.Int64, strict=False),
         pl.col('ActWt').cast(pl.Int64, strict=False),
         pl.col('DeclarHorseWt').cast(pl.Int64, strict=False),
         pl.col('Dr').cast(pl.Int64, strict=False),
         pl.col('Rtg').cast(pl.Int64, strict=False),
-        pl.col('RaceIndex').cast(pl.Int64, strict=False),
-        pl.col('Dist').cast(pl.Int64, strict=False)
+        pl.col('Dist').cast(pl.Int64, strict=False),
+        pl.col('WinOdds').cast(pl.Float64, strict=False),
+        pl.col('RaceIndex').cast(pl.Int64, strict=False)
     ])
 
-    horse_data = horse_data.with_columns(
+    df = df.with_columns(
         (
-            pl.col("FinishTime").str.split(":").list.get(0).cast(pl.Int64) * 60 +
-            pl.col("FinishTime").str.split(":").list.get(1).cast(pl.Float64)
+            pl.col("FinishTime").str.split_exact(".", 1).struct.field("field_0").cast(pl.Int64) * 60 +
+            pl.col("FinishTime").str.split_exact(".", 1).struct.field("field_1").cast(pl.Int64)
         ).cast(pl.Float64).alias("FinishTime")
     )
 
-    horse_data = horse_data.with_columns(
+    df = df.with_columns(
         pl.col('RCTrackCourse').str.split_exact(' / ', 2)
         .struct.rename_fields(['Venue', 'Track', 'Course'])
         .alias('RCTrackCourse')
     ).unnest('RCTrackCourse')
 
-    return horse_data
+    return df
 
+def get_horse_data(horse_no: str) -> pl.DataFrame:
+    df = _extract_horse_data(horse_no)
+    return _clean_horse_data(df)
 
-def get_race_data(date: str, venue_code: str, race_number: int) -> pl.DataFrame:
+def _clean_race_data(df: pl.DataFrame) -> pl.DataFrame:
+    """ Clean and convert horse data to suitable data types
+    """
+    df = df.with_columns(
+        pl.col('Pla').str.split(' ').list.first().alias('Pla')
+    ).filter(~pl.col('Pla').is_in(incidents))
+
+    df = df.with_columns([
+        pl.col('Pla').cast(pl.Int64, strict=False),
+        pl.col('HorseNo').cast(pl.Int64, strict=False),
+        pl.col('ActWt').cast(pl.Int64, strict=False),
+        pl.col('DeclarHorseWt').cast(pl.Int64, strict=False),
+        pl.col('Dr').cast(pl.Int64, strict=False),
+        pl.col('WinOdds').cast(pl.Float64, strict=False)
+    ])
+
+    df = df.with_columns(
+        (
+            pl.col("FinishTime").str.split_exact(":", 1).struct.field("field_0").cast(pl.Int64) * 60 +
+            pl.col("FinishTime").str.split_exact(":", 1).struct.field("field_1").cast(pl.Int64)
+        ).cast(pl.Float64).alias("FinishTime")
+    )
+
+    return df
+
+def _extract_race_data(date: str, venue_code: str, race_number: int) -> pl.DataFrame:
     soup = _soupify_race_page(date, venue_code, race_number)
     table = soup.find('div', class_='race_tab').find('table')
     race_data = _parse_html_table(table)
 
     # Extract the relevant race information
-    race_id = race_data.columns[0].replace(f'RACE{race_number}','')
+    race_id = race_data.columns[0].replace(f'RACE{race_number}', '')
     race_class = race_data.item(1, 0).split('-')[0].strip()
     race_dist = race_data.item(1, 0).split('-')[1].strip().rstrip('M')
     race_name = race_data.item(2, 0).strip()
@@ -162,7 +205,12 @@ def get_race_data(date: str, venue_code: str, race_number: int) -> pl.DataFrame:
                  .with_columns(
                      pl.col("Horse").str.extract(r"\((.*?)\)")
                      .alias("HorseID")
-                 )
-                 )
+    )
+    )
+
+    return race_data
 
-    return race_data
+
+def get_race_data(date: str, venue_code: str, race_number: int) -> pl.DataFrame:
+    df = _extract_race_data(date,venue_code,race_number)
+    return _clean_race_data(df)

hkjc/processing.py

@@ -6,7 +6,7 @@ from typing import Tuple, List, Union
 from .live_odds import live_odds
 from .strategy import qpbanker, place_only
 from .harville_model import fit_harville_to_odds
-from .historical import get_race_data, get_horse_data
+from .historical import _extract_horse_data, _extract_race_data, _clean_horse_data
 from .utils import _validate_date
 
 import polars as pl
@@ -23,8 +23,7 @@ def _all_subsets(lst): return [list(x) for r in range(
 # ======================================
 # Historical data processing functions
 # ======================================
-incidents = ['DISQ', 'DNF', 'FE', 'ML', 'PU', 'TNP', 'TO',
-             'UR', 'VOID', 'WR', 'WV', 'WV-A', 'WX', 'WX-A', 'WXNR']
+
 
 
 def _historical_process_single_date_venue(date: str, venue_code: str) -> List[pl.DataFrame]:
@@ -33,7 +32,7 @@ def _historical_process_single_date_venue(date: str, venue_code: str) -> List[pl
         range(1, 12), desc=f"Processing {date} {venue_code} ...", leave=False)
     for race_number in iter_date:
         try:
-            dfs.append(get_race_data(date.strftime('%Y/%m/%d'),
+            dfs.append(_extract_race_data(date.strftime('%Y/%m/%d'),
                                      venue_code, race_number))
         except:
             if race_number == 1:
@@ -51,7 +50,7 @@ def generate_historical_data(start_date: str, end_date: str) -> pl.DataFrame:
 
     dfs = []
 
-    for date in tqdm(pl.date_range(start_dt, end_dt, interval='1d', eager=True)):
+    for date in tqdm(pl.date_range(start_dt, end_dt, interval='1d', eager=True), leave=False, desc='Scanning for horse IDs ...'):
         for venue_code in ['ST', 'HV']:
             dfs += _historical_process_single_date_venue(date, venue_code)
 
@@ -62,35 +61,9 @@ def generate_historical_data(start_date: str, end_date: str) -> pl.DataFrame:
     horse_ids = pl.concat(dfs)['HorseID'].unique()
 
     # Use horse track records
-    dfs = [get_horse_data(horse_id) for horse_id in horse_ids]
-    df = (
-        pl.concat(dfs).with_columns(
-            pl.col('Date').str.strptime(pl.Date, '%m/%d/%y')
-        ).filter(pl.col('Date').is_between(start_dt, end_dt))
-        .filter(~pl.col('Pla').is_in(incidents))
-        .with_columns(
-            pl.col('Pla').str.split(' ').list.first().alias('Pla')
-        )
-    )
-
-    df = df.with_columns([
-        pl.col('Pla').cast(pl.Int64, strict=False),
-        pl.col('HorseNo').cast(pl.Int64, strict=False),
-        pl.col('ActWt').cast(pl.Int64, strict=False),
-        pl.col('DeclarHorseWt').cast(pl.Int64, strict=False),
-        pl.col('Dr').cast(pl.Int64, strict=False),
-        pl.col('RaceDistance').cast(pl.Int64, strict=False),
-        pl.col('WinOdds').cast(pl.Float64, strict=False)
-    ])
-
-    df = df.with_columns(
-        (
-            pl.col("FinishTime").str.split(":").list.get(0).cast(pl.Int64) * 60 +
-            pl.col("FinishTime").str.split(":").list.get(1).cast(pl.Float64)
-        ).cast(pl.Float64).alias("FinishTime")
-    )
-
-    return df
+    dfs = [_extract_horse_data(horse_id) for horse_id in tqdm(horse_ids, desc='Processing horses ...', leave=False)]
+    df = pl.concat(dfs)
+    return _clean_horse_data(df)
 
 
 # ==========================

{hkjc-0.3.16.dist-info → hkjc-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hkjc
-Version: 0.3.16
+Version: 0.3.17
 Summary: Library for scrapping HKJC data and perform basic analysis
 Requires-Python: >=3.11
 Requires-Dist: beautifulsoup4>=4.14.2

{hkjc-0.3.16.dist-info → hkjc-0.3.17.dist-info}/RECORD

@@ -1,14 +1,14 @@
 hkjc/__init__.py,sha256=TI7PVhmoWSvYX-xdTEdaT3jfY99LiYQFRQZaIwBhJd8,785
 hkjc/analysis.py,sha256=0042_NMIkQCl0J6B0P4TFfrBDCnm2B6jsCZKOEO30yI,108
 hkjc/harville_model.py,sha256=MZjPLS-1nbEhp1d4Syuq13DtraKnd7TlNqBmOOCwxgc,15976
-hkjc/historical.py,sha256=R_7z0yqLDj57G5JgvuPYDxKLBoqlhfpRnpTER6aeluM,6678
+hkjc/historical.py,sha256=yQQAx8vlr2EqcPazpYp1x2ku7dy3imQoDWImHCRv1QA,8330
 hkjc/live_odds.py,sha256=G4ELBBp1d2prxye9kKzu2pwtS4vSfRPOmEuT7-Nd-3A,4741
-hkjc/processing.py,sha256=KZFrGuCdCEJ5OI54PnrWhy-c9qx7mcWm12chc3HuDO8,7764
+hkjc/processing.py,sha256=xrvEUgu_jz8ZxevOsRsYz0T7pWyNtSCMI6LUYByOLOw,6812
 hkjc/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hkjc/speedpro.py,sha256=Y2Z3GYGeePc4sM-ZnCHXCI1N7L-_j9nrMqS3CC5BBSo,2031
 hkjc/utils.py,sha256=4CA_FPf_U3GvzoLkqBX0qDPZgrSvKJKvbP7VWqd5FiA,6323
 hkjc/strategy/place_only.py,sha256=lHPjTSj8PzghxncNBg8FI4T4HJigekB9a3bV7l7VtPA,2079
 hkjc/strategy/qpbanker.py,sha256=MQxjwsfhllKZroKS8w8Q3bi3HMjGc1DAyBIjNZAp3yQ,4805
-hkjc-0.3.16.dist-info/METADATA,sha256=yusLSNb82ebKoie5iUxlO5XE9mBBfPiIBHmRepLXA9c,481
-hkjc-0.3.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hkjc-0.3.16.dist-info/RECORD,,
+hkjc-0.3.17.dist-info/METADATA,sha256=gKSkXKYo_HCg2S4ZeAjnqZniWV0V2kGpRH_g25K9Rmo,481
+hkjc-0.3.17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hkjc-0.3.17.dist-info/RECORD,,