hockey-blast-common-lib 0.1.29__py3-none-any.whl → 0.1.31__py3-none-any.whl

hockey_blast_common_lib/aggregate_skater_stats.py

@@ -6,17 +6,28 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from datetime import datetime, timedelta
 import sqlalchemy
 
-from hockey_blast_common_lib.models import Game, Goal, Penalty, GameRoster, Organization, Division
+from hockey_blast_common_lib.models import Game, Goal, Penalty, GameRoster, Organization, Division, Human, Level
 from hockey_blast_common_lib.stats_models import OrgStatsSkater, DivisionStatsSkater, OrgStatsWeeklySkater, OrgStatsDailySkater, DivisionStatsWeeklySkater, DivisionStatsDailySkater, LevelStatsSkater
 from hockey_blast_common_lib.db_connection import create_session
 from sqlalchemy.sql import func, case
 from hockey_blast_common_lib.options import not_human_names, parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, MIN_GAMES_FOR_LEVEL_STATS
 from hockey_blast_common_lib.utils import get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues, get_all_division_ids_for_org
 from sqlalchemy import func, case, and_
+from collections import defaultdict
 
-def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_filter_out, filter_human_id=None, aggregation_window=None):
+def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_filter_out, debug_human_id=None, aggregation_window=None):
     human_ids_to_filter = get_human_ids_by_names(session, names_to_filter_out)
 
+    # Get the name of the aggregation, for debug purposes
+    if aggregation_type == 'org':
+        aggregation_name = session.query(Organization).filter(Organization.id == aggregation_id).first().organization_name
+    elif aggregation_type == 'division':
+        aggregation_name = session.query(Division).filter(Division.id == aggregation_id).first().level
+    elif aggregation_type == 'level':
+        aggregation_name = session.query(Level).filter(Level.id == aggregation_id).first().level_name
+    else:
+        aggregation_name = "Unknown"
+
     if aggregation_type == 'org':
         if aggregation_window == 'Daily':
             StatsModel = OrgStatsDailySkater
@@ -39,6 +50,10 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         StatsModel = LevelStatsSkater
         min_games = MIN_GAMES_FOR_LEVEL_STATS
         filter_condition = Division.level_id == aggregation_id
+        # Add filter to only include games for the last 5 years
+        # five_years_ago = datetime.now() - timedelta(days=5*365)
+        # level_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP) >= five_years_ago
+        # filter_condition = filter_condition & level_window_filter
     else:
         raise ValueError("Invalid aggregation type")
 
@@ -63,8 +78,8 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
 
     # Filter for specific human_id if provided
     human_filter = []
-    if filter_human_id:
-        human_filter = [GameRoster.human_id == filter_human_id]
+    # if debug_human_id:
+    #     human_filter = [GameRoster.human_id == debug_human_id]
 
     # Aggregate games played for each human in each division, excluding goalies
     games_played_stats = session.query(
@@ -72,7 +87,7 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         GameRoster.human_id,
         func.count(Game.id).label('games_played'),
         func.array_agg(Game.id).label('game_ids')
-    ).join(GameRoster, Game.id == GameRoster.game_id).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, GameRoster.human_id).all()
+    ).join(GameRoster, Game.id == GameRoster.game_id).join(Division, Game.division_id == Division.id).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, GameRoster.human_id).all()
 
     # Aggregate goals for each human in each division, excluding goalies
     goals_stats = session.query(
@@ -80,7 +95,7 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         Goal.goal_scorer_id.label('human_id'),
         func.count(Goal.id).label('goals'),
         func.array_agg(Goal.game_id).label('goal_game_ids')
-    ).join(Game, Game.id == Goal.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Goal.goal_scorer_id == GameRoster.human_id)).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Goal.goal_scorer_id).all()
+    ).join(Game, Game.id == Goal.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Goal.goal_scorer_id == GameRoster.human_id)).join(Division, Game.division_id == Division.id).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Goal.goal_scorer_id).all()
 
     # Aggregate assists for each human in each division, excluding goalies
     assists_stats = session.query(
@@ -88,14 +103,14 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         Goal.assist_1_id.label('human_id'),
         func.count(Goal.id).label('assists'),
         func.array_agg(Goal.game_id).label('assist_game_ids')
-    ).join(Game, Game.id == Goal.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Goal.assist_1_id == GameRoster.human_id)).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Goal.assist_1_id).all()
+    ).join(Game, Game.id == Goal.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Goal.assist_1_id == GameRoster.human_id)).join(Division, Game.division_id == Division.id).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Goal.assist_1_id).all()
 
     assists_stats_2 = session.query(
         Game.org_id,
         Goal.assist_2_id.label('human_id'),
         func.count(Goal.id).label('assists'),
         func.array_agg(Goal.game_id).label('assist_2_game_ids')
-    ).join(Game, Game.id == Goal.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Goal.assist_2_id == GameRoster.human_id)).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Goal.assist_2_id).all()
+    ).join(Game, Game.id == Goal.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Goal.assist_2_id == GameRoster.human_id)).join(Division, Game.division_id == Division.id).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Goal.assist_2_id).all()
 
     # Aggregate penalties for each human in each division, excluding goalies
     penalties_stats = session.query(
@@ -103,7 +118,7 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         Penalty.penalized_player_id.label('human_id'),
         func.count(Penalty.id).label('penalties'),
         func.array_agg(Penalty.game_id).label('penalty_game_ids')
-    ).join(Game, Game.id == Penalty.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Penalty.penalized_player_id == GameRoster.human_id)).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Penalty.penalized_player_id).all()
+    ).join(Game, Game.id == Penalty.game_id).join(GameRoster, and_(Game.id == GameRoster.game_id, Penalty.penalized_player_id == GameRoster.human_id)).join(Division, Game.division_id == Division.id).filter(filter_condition, ~GameRoster.role.ilike('g'), *human_filter).group_by(Game.org_id, Penalty.penalized_player_id).all()
 
     # Combine the results
     stats_dict = {}
@@ -111,6 +126,8 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         if stat.human_id in human_ids_to_filter:
             continue
         key = (aggregation_id, stat.human_id)
+        if stat.games_played < min_games:
+            continue
         stats_dict[key] = {
             'games_played': stat.games_played,
             'goals': 0,
@@ -127,94 +144,26 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         }
 
     for stat in goals_stats:
-        if stat.human_id in human_ids_to_filter:
-            continue
         key = (aggregation_id, stat.human_id)
-        if key not in stats_dict:
-            stats_dict[key] = {
-                'games_played': 0,
-                'goals': stat.goals,
-                'assists': 0,
-                'penalties': 0,
-                'points': stat.goals,  # Initialize points with goals
-                'goals_per_game': 0.0,
-                'points_per_game': 0.0,
-                'assists_per_game': 0.0,
-                'penalties_per_game': 0.0,
-                'game_ids': [],
-                'first_game_id': None,
-                'last_game_id': None
-            }
-        else:
+        if key in stats_dict:
             stats_dict[key]['goals'] += stat.goals
             stats_dict[key]['points'] += stat.goals  # Update points
 
     for stat in assists_stats:
-        if stat.human_id in human_ids_to_filter:
-            continue
         key = (aggregation_id, stat.human_id)
-        if key not in stats_dict:
-            stats_dict[key] = {
-                'games_played': 0,
-                'goals': 0,
-                'assists': stat.assists,
-                'penalties': 0,
-                'points': stat.assists,  # Initialize points with assists
-                'goals_per_game': 0.0,
-                'points_per_game': 0.0,
-                'assists_per_game': 0.0,
-                'penalties_per_game': 0.0,
-                'game_ids': [],
-                'first_game_id': None,
-                'last_game_id': None
-            }
-        else:
+        if key in stats_dict:
             stats_dict[key]['assists'] += stat.assists
             stats_dict[key]['points'] += stat.assists  # Update points
 
     for stat in assists_stats_2:
-        if stat.human_id in human_ids_to_filter:
-            continue
         key = (aggregation_id, stat.human_id)
-        if key not in stats_dict:
-            stats_dict[key] = {
-                'games_played': 0,
-                'goals': 0,
-                'assists': stat.assists,
-                'penalties': 0,
-                'points': stat.assists,  # Initialize points with assists
-                'goals_per_game': 0.0,
-                'points_per_game': 0.0,
-                'assists_per_game': 0.0,
-                'penalties_per_game': 0.0,
-                'game_ids': [],
-                'first_game_id': None,
-                'last_game_id': None
-            }
-        else:
+        if key in stats_dict:
             stats_dict[key]['assists'] += stat.assists
             stats_dict[key]['points'] += stat.assists  # Update points
 
     for stat in penalties_stats:
-        if stat.human_id in human_ids_to_filter:
-            continue
         key = (aggregation_id, stat.human_id)
-        if key not in stats_dict:
-            stats_dict[key] = {
-                'games_played': 0,
-                'goals': 0,
-                'assists': 0,
-                'penalties': stat.penalties,
-                'points': 0,  # Initialize points
-                'goals_per_game': 0.0,
-                'points_per_game': 0.0,
-                'assists_per_game': 0.0,
-                'penalties_per_game': 0.0,
-                'game_ids': [],
-                'first_game_id': None,
-                'last_game_id': None
-            }
-        else:
+        if key in stats_dict:
             stats_dict[key]['penalties'] += stat.penalties
 
     # Calculate per game stats
@@ -237,20 +186,10 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
             stat['first_game_id'] = first_game.id if first_game else None
             stat['last_game_id'] = last_game.id if last_game else None
 
-    # Debug output for totals if filter_human_id is provided
-    if filter_human_id:
-        for key, stat in stats_dict.items():
-            if key[1] == filter_human_id:
-                print(f"Human ID: {filter_human_id}")
-                print(f"Total Games Played: {stat['games_played']}")
-                print(f"Total Goals: {stat['goals']}")
-                print(f"Total Assists: {stat['assists']}")
-                print(f"Total Penalties: {stat['penalties']}")
-
     # Calculate total_in_rank
     total_in_rank = len(stats_dict)
 
-    # Assign ranks
+    # Assign ranks within each level
     def assign_ranks(stats_dict, field):
         sorted_stats = sorted(stats_dict.items(), key=lambda x: x[1][field], reverse=True)
         for rank, (key, stat) in enumerate(sorted_stats, start=1):
@@ -266,13 +205,22 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
     assign_ranks(stats_dict, 'assists_per_game')
     assign_ranks(stats_dict, 'penalties_per_game')
 
+    # Debug output for specific human
+    if debug_human_id:
+        if any(key[1] == debug_human_id for key in stats_dict):
+            human = session.query(Human).filter(Human.id == debug_human_id).first()
+            human_name = f"{human.first_name} {human.last_name}" if human else "Unknown"
+            print(f"For Human {debug_human_id} ({human_name}) for {aggregation_type} {aggregation_id} ({aggregation_name}) , total_in_rank {total_in_rank} and window {aggregation_window}:")
+            for key, stat in stats_dict.items():
+                if key[1] == debug_human_id:
+                    for k, v in stat.items():
+                        print(f"{k}: {v}")
+
     # Insert aggregated stats into the appropriate table with progress output
     total_items = len(stats_dict)
     batch_size = 1000
     for i, (key, stat) in enumerate(stats_dict.items(), 1):
         aggregation_id, human_id = key
-        if stat['games_played'] < min_games:
-            continue
         goals_per_game = stat['goals'] / stat['games_played'] if stat['games_played'] > 0 else 0.0
         points_per_game = (stat['goals'] + stat['assists']) / stat['games_played'] if stat['games_played'] > 0 else 0.0
         assists_per_game = stat['assists'] / stat['games_played'] if stat['games_played'] > 0 else 0.0
@@ -306,33 +254,37 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         # Commit in batches
         if i % batch_size == 0:
             session.commit()
-            print(f"\r{i}/{total_items} ({(i/total_items)*100:.2f}%)", end="")
+            if debug_human_id is None:
+                print(f"\r{i}/{total_items} ({(i/total_items)*100:.2f}%)", end="")
+            
     session.commit()
-    print(f"\r{total_items}/{total_items} (100.00%)")
-    print("\nDone.")
+    if debug_human_id is None:
+        print(f"\r{total_items}/{total_items} (100.00%)")
 
 if __name__ == "__main__":
     session = create_session("boss")
+    human_id_to_debug = 117076
 
     # Get all org_id present in the Organization table
     org_ids = session.query(Organization.id).all()
     org_ids = [org_id[0] for org_id in org_ids]
 
-    # for org_id in org_ids:
-    #     division_ids = get_all_division_ids_for_org(session, org_id)
-    #     print(f"Aggregating skater stats for {len(division_ids)} divisions in org_id {org_id}...")
-    #     total_divisions = len(division_ids)
-    #     processed_divisions = 0
-        # for division_id in division_ids:
-        #     aggregate_skater_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, filter_human_id=None)
-        #     aggregate_skater_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, filter_human_id=None, aggregation_window='Weekly')
-        #     aggregate_skater_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, filter_human_id=None, aggregation_window='Daily')
-        #     processed_divisions += 1
-        #     print(f"\rProcessed {processed_divisions}/{total_divisions} divisions ({(processed_divisions/total_divisions)*100:.2f}%)", end="")
+    for org_id in org_ids:
+        division_ids = get_all_division_ids_for_org(session, org_id)
+        print(f"Aggregating skater stats for {len(division_ids)} divisions in org_id {org_id}...")
+        total_divisions = len(division_ids)
+        processed_divisions = 0
+        for division_id in division_ids:
+            aggregate_skater_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)
+            aggregate_skater_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Weekly')
+            aggregate_skater_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Daily')
+            processed_divisions += 1
+            if human_id_to_debug is None:
+                print(f"\rProcessed {processed_divisions}/{total_divisions} divisions ({(processed_divisions/total_divisions)*100:.2f}%)", end="")
 
-        # aggregate_skater_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, filter_human_id=None)
-        # aggregate_skater_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, filter_human_id=None, aggregation_window='Weekly')
-        # aggregate_skater_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, filter_human_id=None, aggregation_window='Daily')
+        aggregate_skater_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)
+        aggregate_skater_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Weekly')
+        aggregate_skater_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Daily')
         
         # Aggregate by level
     level_ids = session.query(Division.level_id).distinct().all()
@@ -342,7 +294,7 @@ if __name__ == "__main__":
     for level_id in level_ids:
         if level_id is None:
             continue
-        print(f"\rProcessed {processed_levels}/{total_levels} levels ({(processed_levels/total_levels)*100:.2f}%)", end="")
+        if human_id_to_debug is None:
+            print(f"\rProcessed {processed_levels}/{total_levels} levels ({(processed_levels/total_levels)*100:.2f}%)", end="")
         processed_levels += 1
-        aggregate_skater_stats(session, aggregation_type='level', aggregation_id=level_id, names_to_filter_out=not_human_names, filter_human_id=None)
-    print("\nDone.")
+        aggregate_skater_stats(session, aggregation_type='level', aggregation_id=level_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)

hockey_blast_common_lib/assign_skater_skill.py

@@ -0,0 +1,46 @@
+import sys, os
+
+# Add the package directory to the Python path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from hockey_blast_common_lib.models import Human, LevelStatsSkater
+from hockey_blast_common_lib.db_connection import create_session
+from sqlalchemy.sql import func
+
+def calculate_skater_skill_value(human_id, level_stats):
+    max_skill_value = 0
+
+    for stat in level_stats:
+        level_skill_value = stat.level.skill_value
+        if level_skill_value < 0:
+            continue
+        ppg_ratio = stat.points_per_game_rank / stat.total_in_rank
+        games_played_ratio = stat.games_played_rank / stat.total_in_rank
+
+        # Take the maximum of the two ratios
+        skill_value = level_skill_value * max(ppg_ratio, games_played_ratio)
+        max_skill_value = max(max_skill_value, skill_value)
+
+    return max_skill_value
+
+def assign_skater_skill_values():
+    session = create_session("boss")
+
+    humans = session.query(Human).all()
+    total_humans = len(humans)
+    processed_humans = 0
+
+    for human in humans:
+        level_stats = session.query(LevelStatsSkater).filter(LevelStatsSkater.human_id == human.id).all()
+        if level_stats:
+            skater_skill_value = calculate_skater_skill_value(human.id, level_stats)
+            human.skater_skill_value = skater_skill_value
+            session.commit()
+
+        processed_humans += 1
+        print(f"\rProcessed {processed_humans}/{total_humans} humans ({(processed_humans/total_humans)*100:.2f}%)", end="")
+
+    print("\nSkater skill values have been assigned to all humans.")
+
+if __name__ == "__main__":
+    assign_skater_skill_values()

hockey_blast_common_lib/options.py

@@ -4,7 +4,7 @@ MAX_HUMAN_SEARCH_RESULTS = 25
 MAX_TEAM_SEARCH_RESULTS = 25
 MIN_GAMES_FOR_ORG_STATS = 1
 MIN_GAMES_FOR_DIVISION_STATS = 1
-MIN_GAMES_FOR_LEVEL_STATS = 10
+MIN_GAMES_FOR_LEVEL_STATS = 20
 
 orgs = {'caha', 'sharksice', 'tvice'}
 

hockey_blast_common_lib/skills_in_divisions.py

@@ -47,55 +47,6 @@ def analyze_levels(org):
 
     session.close()
 
-def reset_skill_values_in_divisions():
-    session = create_session("boss")
-
-    # Fetch all records from the Division table
-    divisions = session.query(Division).all()
-
-    for division in divisions:
-        # Look up the Skill table using the level from Division
-        div_level = division.level
-        # Query to find the matching Skill
-        level = session.query(Level).filter(Level.org_id == division.org_id, Level.level_name == div_level).one_or_none()
-
-        if not level:
-            # If no match found, check each alternative name individually
-            skills = session.query(Level).filter(Level.org_id == division.org_id).all()
-            for s in skills:
-                alternative_names = s.level_alternative_name.split(',')
-                if div_level in alternative_names:
-                    level = s
-                    break
-
-        if level:
-            # Assign the skill_value and set skill_propagation_sequence to 0
-            division.level_id = level.id
-            if level.is_seed:
-                level.skill_propagation_sequence = 0
-            else:
-                level.skill_propagation_sequence = -1
-                level.skill_value = -1
-        else:
-            # Add new Skill with values previously used for division
-            new_level = Level(
-                org_id=division.org_id,
-                skill_value=-1,
-                level_name=division.level,
-                level_alternative_name='',
-                is_seed=False,
-                skill_propagation_sequence=-1
-            )
-            session.add(new_level)
-            session.commit()
-            division.skill_id = new_level.id
-            print(f"Created new Level for Division {division.level}")
-
-        # Commit the changes to the Division
-        session.commit()
-
-    print("Level values and propagation sequences have been populated into the Division table.")
-
 def fill_seed_skills():
     session = create_session("boss")
 

hockey_blast_common_lib/skills_propagation.py

@@ -6,21 +6,21 @@ import numpy as np
 # Add the project root directory to the Python path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from hockey_blast_common_lib.models import Level, Season
-from hockey_blast_common_lib.stats_models import LevelsGraphEdge, LevelStatsSkater, SkillPropagationCorrelation
+from hockey_blast_common_lib.models import Level, Division
+from hockey_blast_common_lib.stats_models import LevelsGraphEdge, LevelStatsSkater, SkillValuePPGRatio
 from hockey_blast_common_lib.db_connection import create_session
 from sqlalchemy import func
 
 import numpy as np
 
 class Config:
-    MIN_GAMES_PLAYED = 10
-    MIN_PPG = 0.3
-    MIN_HUMANS_FOR_EDGE = 5
-    MAX_START_DATE_DIFF_MONTHS = 15
-    MAX_PROPAGATION_SEQUENCE = 0
-    MIN_CONNECTIONS_FOR_CORRELATION = 40
-    MIN_CONNECTIONS_FOR_PROPAGATION = 3
+    MIN_GAMES_PLAYED_FOR_EDGE = 10
+    MIN_PPG_FOR_EDGE = 0.5
+    MIN_HUMANS_FOR_EDGE = 2
+    MAX_PROPAGATION_SEQUENCE = 4
+    MIN_CONNECTIONS_FOR_CORRELATION = 20
+    MIN_CONNECTIONS_FOR_PROPAGATION = 5
+    MAX_SKILL_DIFF_IN_EDGE = 30
 
     @staticmethod
     def discard_outliers(data, m=2):
@@ -40,8 +40,57 @@ class Config:
         modified_z_score = 0.6745 * diff / med_abs_deviation
         return data[modified_z_score < m]
 
+def reset_skill_values_in_divisions():
+    session = create_session("boss")
+
+    # Fetch all records from the Division table
+    divisions = session.query(Division).all()
+
+    for division in divisions:
+        # Look up the Skill table using the level from Division
+        div_level = division.level
+        # Query to find the matching Skill
+        level = session.query(Level).filter(Level.org_id == division.org_id, Level.level_name == div_level).one_or_none()
+
+        if not level:
+            # If no match found, check each alternative name individually
+            skills = session.query(Level).filter(Level.org_id == division.org_id).all()
+            for s in skills:
+                alternative_names = s.level_alternative_name.split(',')
+                if div_level in alternative_names:
+                    level = s
+                    break
+
+        if level:
+            # Assign the skill_value and set skill_propagation_sequence to 0
+            division.level_id = level.id
+            if level.is_seed:
+                level.skill_propagation_sequence = 0
+            else:
+                level.skill_propagation_sequence = -1
+                level.skill_value = -1
+        else:
+            # Add new Skill with values previously used for division
+            new_level = Level(
+                org_id=division.org_id,
+                skill_value=-1,
+                level_name=division.level,
+                level_alternative_name='',
+                is_seed=False,
+                skill_propagation_sequence=-1
+            )
+            session.add(new_level)
+            session.commit()
+            division.skill_id = new_level.id
+            print(f"Created new Level for Division {division.level}")
+
+        # Commit the changes to the Division
+        session.commit()
+
+    print("Level values and propagation sequences have been populated into the Division table.")
 
 def build_levels_graph_edges():
+    # Creates unique edges from levelA to levelB (there is no reverse edge levelB to levelA)
     session = create_session("boss")
 
     # Delete all existing edges
@@ -55,8 +104,8 @@ def build_levels_graph_edges():
     level_human_stats = defaultdict(lambda: defaultdict(dict))
 
     for stat in level_stats:
-        if stat.games_played >= Config.MIN_GAMES_PLAYED and stat.points_per_game >= Config.MIN_PPG:
-            level_human_stats[stat.aggregation_id][stat.human_id] = {
+        if stat.games_played >= Config.MIN_GAMES_PLAYED_FOR_EDGE and stat.points_per_game >= Config.MIN_PPG_FOR_EDGE:
+            level_human_stats[stat.level_id][stat.human_id] = {
                 'games_played': stat.games_played,
                 'points_per_game': stat.points_per_game
             }
@@ -69,34 +118,32 @@ def build_levels_graph_edges():
     processed_levels = 0
     for from_level_id, from_humans in level_human_stats.items():
         from_level = session.query(Level).filter_by(id=from_level_id).first()
-        from_season = session.query(Season).filter_by(id=from_level.season_id).first()
         for to_level_id, to_humans in level_human_stats.items():
             to_level = session.query(Level).filter_by(id=to_level_id).first()
-            to_season = session.query(Season).filter_by(id=to_level.season_id).first()
-
-            if from_level.skill_value >= to_level.skill_value:
-                continue
-
-            # TMP DEBUG HACK
-            if from_level.skill_value != 10 and to_level.skill_value != 30:
-                continue
-
-            # Check if the start dates are within the allowed difference
-            if abs((from_season.start_date - to_season.start_date).days) > Config.MAX_START_DATE_DIFF_MONTHS * 30:
+            if from_level.id >= to_level.id:
                 continue
 
             common_humans = set(from_humans.keys()) & set(to_humans.keys())
             n_connections = len(common_humans)
+            n_games = 0
 
             if n_connections < Config.MIN_HUMANS_FOR_EDGE:
                 continue
 
             ppg_ratios = []
+            # if from_level.id == 223 and to_level.id == 219: #216
+            #     print(f"Debug: From Level ID: {from_level.id}, To Level ID: {to_level.id}")
             for human_id in common_humans:
                 from_ppg = from_humans[human_id]['points_per_game']
                 to_ppg = to_humans[human_id]['points_per_game']
-                if from_level.skill_value == 10 and to_level.skill_value == 30:
-                    print(f"Human {human_id} From PPG: {from_ppg}, To PPG: {to_ppg}")
+                from_games = from_humans[human_id]['games_played']
+                to_games = to_humans[human_id]['games_played']
+                min_games = min(from_games, to_games)
+                n_games += min_games
+
+                # if from_level.id == 223 and to_level.id == 219: #216
+                #     print(f"Human {human_id} From PPG: {from_ppg}, To PPG: {to_ppg}, Min Games: {min_games} n_games: {n_games}")
+
                 if from_ppg > 0 and to_ppg > 0:
                     ppg_ratios.append(to_ppg / from_ppg)
 
@@ -110,15 +157,16 @@ def build_levels_graph_edges():
                 continue
 
             avg_ppg_ratio = float(sum(ppg_ratios) / len(ppg_ratios))
-            if avg_ppg_ratio < 1.0:
-                avg_ppg_ratio = 1 / avg_ppg_ratio
-                from_level_id, to_level_id = to_level_id, from_level_id
+
+            # if sorted([from_level.id, to_level.id]) == [219, 223]:
+            #     print(f"From {from_level_id} to {to_level_id} n_connections {n_connections} n_games: {n_games}")
 
             edge = LevelsGraphEdge(
                 from_level_id=from_level_id,
                 to_level_id=to_level_id,
                 n_connections=n_connections,
-                ppg_ratio=avg_ppg_ratio
+                ppg_ratio=avg_ppg_ratio,
+                n_games=n_games  # Store the number of games
             )
             edges[(from_level_id, to_level_id)] = edge
 
@@ -133,11 +181,14 @@ def build_levels_graph_edges():
     print("\nLevels graph edges have been populated into the database.")
 
 def propagate_skill_levels(propagation_sequence):
+    min_skill_value = float('inf')
+    max_skill_value = float('-inf')
+
     session = create_session("boss")
 
     if propagation_sequence == 0:
         # Delete all existing correlation data
-        session.query(SkillPropagationCorrelation).delete()
+        session.query(SkillValuePPGRatio).delete()
         session.commit()
 
         # Build and save the correlation data
@@ -160,10 +211,10 @@ def propagate_skill_levels(propagation_sequence):
 
                 if edge.from_level_id == level.id:
                     target_level_id = edge.to_level_id
-                    ppg_ratio = edge.ppg_ratio
+                    ppg_ratio_edge = edge.ppg_ratio
                 else:
-                    target_level_id = edge.from_level_id
-                    ppg_ratio = 1 / edge.ppg_ratio
+                    # We go over same edge twice in this logic, let's skip the reverse edge
+                    continue
 
                 if target_level_id not in level_ids:
                     continue
@@ -173,32 +224,48 @@ def propagate_skill_levels(propagation_sequence):
                     skill_value_from = level.skill_value
                     skill_value_to = target_level.skill_value
 
+                    # Same skill value - no correlation
+                    if skill_value_from == skill_value_to:
+                        continue
+
+
                     # Since we go over all levels in the sequence 0, we will see each edge twice
                     # This condition eliminates duplicates
-                    if skill_value_from >= skill_value_to:
+                    if abs(skill_value_from - skill_value_to) > Config.MAX_SKILL_DIFF_IN_EDGE:
                         continue
 
                     # Debug prints
-                    print(f"From Skill  {level.skill_value} to {target_level.skill_value} ratio: {ppg_ratio}")
+                    # print(f"From Skill  {level.skill_value} to {target_level.skill_value} ratio: {ppg_ratio}")
+
+                    # Ensure INCREASING SKILL VALUES for the correlation data!
+                    if skill_value_from > skill_value_to:
+                        skill_value_from, skill_value_to = skill_value_to, skill_value_from
+                        ppg_ratio_edge = 1 / ppg_ratio_edge
 
                     correlation_data[(skill_value_from, skill_value_to)].append(
-                        ppg_ratio
+                        (ppg_ratio_edge, edge.n_games)
                     )
 
         # Save correlation data to the database
         for (skill_value_from, skill_value_to), ppg_ratios in correlation_data.items():
-            ppg_ratios = Config.discard_outliers(np.array(ppg_ratios))
-            if len(ppg_ratios) > 0:
-                avg_ppg_ratio = float(sum(ppg_ratios) / len(ppg_ratios))
-                correlation = SkillPropagationCorrelation(
-                    skill_value_from=skill_value_from,
-                    skill_value_to=skill_value_to,
-                    ppg_ratio=avg_ppg_ratio
+            ppg_ratios = [(ppg_ratio, n_games) for ppg_ratio, n_games in ppg_ratios]
+            ppg_ratios_array = np.array(ppg_ratios, dtype=[('ppg_ratio', float), ('n_games', int)])
+            ppg_ratios_filtered = Config.discard_outliers(ppg_ratios_array['ppg_ratio'])
+            if len(ppg_ratios_filtered) > 0:
+                avg_ppg_ratio = float(sum(ppg_ratio * n_games for ppg_ratio, n_games in ppg_ratios if ppg_ratio in ppg_ratios_filtered) / sum(n_games for ppg_ratio, n_games in ppg_ratios if ppg_ratio in ppg_ratios_filtered))
+                total_n_games = sum(n_games for ppg_ratio, n_games in ppg_ratios if ppg_ratio in ppg_ratios_filtered)
+                correlation = SkillValuePPGRatio(
+                    from_skill_value=skill_value_from,
+                    to_skill_value=skill_value_to,
+                    ppg_ratio=avg_ppg_ratio,
+                    n_games=total_n_games  # Store the sum of games
                 )
                 session.add(correlation)
                 session.commit()
+                # Update min and max skill values
+                min_skill_value = min(min_skill_value, skill_value_from, skill_value_to)
+                max_skill_value = max(max_skill_value, skill_value_from, skill_value_to)
 
-    return
     # Propagate skill levels
     levels = session.query(Level).filter(Level.skill_propagation_sequence == propagation_sequence).all()
     suggested_skill_values = defaultdict(list)
@@ -215,27 +282,76 @@ def propagate_skill_levels(propagation_sequence):
 
             if edge.from_level_id == level.id:
                 target_level_id = edge.to_level_id
-                ppg_ratio = edge.ppg_ratio
+                ppg_ratio_edge = edge.ppg_ratio
             else:
                 target_level_id = edge.from_level_id
-                ppg_ratio = 1 / edge.ppg_ratio
+                ppg_ratio_edge = 1 / edge.ppg_ratio
 
             target_level = session.query(Level).filter_by(id=target_level_id).first()
             if target_level and target_level.skill_propagation_sequence == -1:
-                correlation = session.query(SkillPropagationCorrelation).filter_by(
-                    skill_value_from=min(level.skill_value, target_level.skill_value),
-                    skill_value_to=max(level.skill_value, target_level.skill_value),
-                    ppg_ratio=ppg_ratio if level.skill_value < target_level.skill_value else 1 / ppg_ratio
-                ).first()
-
-                if correlation:
-                    suggested_skill_values[target_level_id].append(correlation.skill_value_to)
+                correlations = session.query(SkillValuePPGRatio).filter(
+                    (SkillValuePPGRatio.from_skill_value <= level.skill_value) &
+                    (SkillValuePPGRatio.to_skill_value >= level.skill_value)
+                ).all()
+
+                if correlations:
+                    weighted_skill_values = []
+                    for correlation in correlations:
+                        # Skill value always increases in the correlation data
+                        # Let's avoid extrapolating from the end of the edge and away from the edge!
+
+                        # Check left side of the edge
+                        if (level.skill_value == correlation.from_skill_value and level.skill_value > min_skill_value):
+                            if ppg_ratio_edge < 1:
+                                continue
+                        # Check right side of the edge
+                        if (level.skill_value == correlation.to_skill_value and level.skill_value < max_skill_value):
+                            if ppg_ratio_edge > 1:
+                                continue
+
+                        
+                        # First confirm which way are we going here
+                        if (ppg_ratio_edge < 1 and correlation.ppg_ratio > 1) or (ppg_ratio_edge > 1 and correlation.ppg_ratio < 1):
+                            # Reverse the correlation
+                            from_skill_value=correlation.to_skill_value
+                            to_skill_value=correlation.from_skill_value
+                            ppg_ratio_range = 1 / correlation.ppg_ratio
+                        else:
+                            from_skill_value=correlation.from_skill_value
+                            to_skill_value=correlation.to_skill_value
+                            ppg_ratio_range = correlation.ppg_ratio
+
+                        # Now both ratios are either < 1 or > 1
+                        if ppg_ratio_edge < 1:
+                            ppg_ratio_for_extrapolation = 1 / ppg_ratio_edge
+                            ppg_ratio_range = 1 / ppg_ratio_range
+                        else:
+                            ppg_ratio_for_extrapolation = ppg_ratio_edge
+                        
+                        # Interpolate or extrapolate skill value
+                        skill_value_range = to_skill_value - from_skill_value
+                        skill_value_diff = (ppg_ratio_for_extrapolation / ppg_ratio_range) * skill_value_range
+                        new_skill_value = level.skill_value + skill_value_diff
+                        weighted_skill_values.append((new_skill_value, correlation.n_games))
+                        # if target_level.id == 229:
+                        #     print(f"Debug: From Level ID: {level.id}, To Level ID: {target_level.id}")
+                        #     print(f"Debug: From Skill Value: {level.skill_value} PPG Ratio: {ppg_ratio_for_extrapolation}, PPG Ratio Range: {ppg_ratio_range}")
+                        #     print(f"Debug: Skill Value Range: {skill_value_range}, Skill Value Diff: {skill_value_diff}")
+                        #     print(f"Debug: New Skill Value: {new_skill_value}")
+
+                    # Calculate weighted average of new skill values
+                    total_n_games = sum(n_games for _, n_games in weighted_skill_values)
+                    weighted_avg_skill_value = sum(skill_value * n_games for skill_value, n_games in weighted_skill_values) / total_n_games
+                    suggested_skill_values[target_level_id].append(weighted_avg_skill_value)
 
     # Update skill values for target levels
     for target_level_id, skill_values in suggested_skill_values.items():
         skill_values = Config.discard_outliers(np.array(skill_values))
         if len(skill_values) > 0:
             avg_skill_value = float(sum(skill_values) / len(skill_values))
+            avg_skill_value = max(avg_skill_value, 9.6)
+            if avg_skill_value < min_skill_value:
+                avg_skill_value = min_skill_value - 0.01
             session.query(Level).filter_by(id=target_level_id).update({
                 'skill_value': avg_skill_value,
                 'skill_propagation_sequence': propagation_sequence + 1
@@ -245,6 +361,7 @@ def propagate_skill_levels(propagation_sequence):
     print(f"Skill levels have been propagated for sequence {propagation_sequence}.")
 
 if __name__ == "__main__":
+    reset_skill_values_in_divisions()
     build_levels_graph_edges()
 
     for sequence in range(Config.MAX_PROPAGATION_SEQUENCE + 1):

hockey_blast_common_lib/stats_models.py

@@ -583,6 +583,7 @@ class LevelsGraphEdge(db.Model):
     to_level_id = db.Column(db.Integer, db.ForeignKey('levels.id'), nullable=False)
     n_connections = db.Column(db.Integer, nullable=False)
     ppg_ratio = db.Column(db.Float, nullable=False)
+    n_games = db.Column(db.Integer, nullable=False)  # New field to store the number of games
 
     __table_args__ = (
         db.UniqueConstraint('from_level_id', 'to_level_id', name='_from_to_level_uc'),
@@ -598,3 +599,16 @@ class SkillPropagationCorrelation(db.Model):
     __table_args__ = (
         db.UniqueConstraint('skill_value_from', 'skill_value_to', 'ppg_ratio', name='_skill_value_ppg_ratio_uc'),
     )
+
+# How PPG changes with INCREASING SKILL VALUES
+class SkillValuePPGRatio(db.Model):
+    __tablename__ = 'skill_value_ppg_ratios'
+    id = db.Column(db.Integer, primary_key=True)
+    from_skill_value = db.Column(db.Float, nullable=False)
+    to_skill_value = db.Column(db.Float, nullable=False)
+    ppg_ratio = db.Column(db.Float, nullable=False)
+    n_games = db.Column(db.Integer, nullable=False)  # New field to store the sum of games
+
+    __table_args__ = (
+        db.UniqueConstraint('from_skill_value', 'to_skill_value', name='_from_to_skill_value_uc'),
+    )

{hockey_blast_common_lib-0.1.29.dist-info → hockey_blast_common_lib-0.1.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hockey-blast-common-lib
-Version: 0.1.29
+Version: 0.1.31
 Summary: Common library for shared functionality and DB models
 Author: Pavel Kletskov
 Author-email: kletskov@gmail.com

{hockey_blast_common_lib-0.1.29.dist-info → hockey_blast_common_lib-0.1.31.dist-info}/RECORD

@@ -2,19 +2,20 @@ hockey_blast_common_lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 hockey_blast_common_lib/aggregate_goalie_stats.py,sha256=d2qav46Rg2DNIYRj_Ubj1kpQmoPUJHKiwEWOVU25nD4,8742
 hockey_blast_common_lib/aggregate_human_stats.py,sha256=88OMhTgQjzc9xIakf6kW9_lZwbSXkpsZy8C0pX-Wlq8,14229
 hockey_blast_common_lib/aggregate_referee_stats.py,sha256=A0PTyEbPUjqfXxlJCDOVioFaQk9AyjjhiWEuRuu35v0,11036
-hockey_blast_common_lib/aggregate_skater_stats.py,sha256=H_RPz26erdvKXz6w8NjSfQAZgcpXitFKsuhmrYYqWyg,16839
+hockey_blast_common_lib/aggregate_skater_stats.py,sha256=jkBD5u-gJc1DTDIEuxM_qymKsrWtLagFKeEn__2rFgU,16009
+hockey_blast_common_lib/assign_skater_skill.py,sha256=YQYYWKArkvqPsu7shlRzggE-p_7rcEMO3VtnzKQHPG0,1657
 hockey_blast_common_lib/db_connection.py,sha256=HvPxDvOj7j5H85RfslGvHVNevfg7mKCd0syJ6NX21mU,1890
 hockey_blast_common_lib/dump_sample_db.sh,sha256=MHPA-Ciod7wsvAlMbRtXFiyajgnEqU1xR59sJQ9UWR0,738
-hockey_blast_common_lib/hockey_blast_sample_backup.sql.gz,sha256=DcRvR7oyP2gbVU7fYUP8-Z9R79kl0jWt4zgFyNECstk,1033685
+hockey_blast_common_lib/hockey_blast_sample_backup.sql.gz,sha256=Flm7M1rDuz-psuBg_gtlo1JhjjJRjnBMGy-gTwgMhjM,1033685
 hockey_blast_common_lib/models.py,sha256=2YkFHgzvxLSU2-fURsyIqdaXjcQwZ_wjauYR9wWQtDw,15924
-hockey_blast_common_lib/options.py,sha256=S545jnyw14GUiqDyVbwHlNlb3WStpJpsqmRqJ9-mBA0,792
+hockey_blast_common_lib/options.py,sha256=6na8fo-5A2RBPpd_h-7dsqetOLSLoNEJg1QMYgl4jNs,792
 hockey_blast_common_lib/restore_sample_db.sh,sha256=u2zKazC6vNMULkpYzI64nlneCWaGUtDHPBAU-gWgRbw,1861
-hockey_blast_common_lib/skills_in_divisions.py,sha256=BDoA9sGHH86chErTFLmK3oYMiZJwrfUtvKV7kf6pthg,9427
-hockey_blast_common_lib/skills_propagation.py,sha256=tFUXETJJNvV_kq_JTs85LhbS8mRLHKa6ZpW9foYRfI8,9779
-hockey_blast_common_lib/stats_models.py,sha256=W2NbVqQgUucbRsM0uA3tCy7ee7C4ElJ-UOlVDuhBFZM,22388
+hockey_blast_common_lib/skills_in_divisions.py,sha256=tQYSikIV9AN_Cr-770KRy4reaiiR-2vvtwaNPJ8Lvws,7524
+hockey_blast_common_lib/skills_propagation.py,sha256=x6yy7fJ6IX3YiHqiP_v7-p_S2Expb8JJ-mWuajEFBdY,16388
+hockey_blast_common_lib/stats_models.py,sha256=35-6iz1r8MJcmzlyIlJy0uHgWh8oltyf-3H61ocal3o,23048
 hockey_blast_common_lib/utils.py,sha256=odDJWCK0BgbResXeoUzxbVChjaxcXr168ZxbrAw3L_8,3752
 hockey_blast_common_lib/wsgi.py,sha256=7LGUzioigviJp-EUhSEaQcd4jBae0mxbkyBscQfZhlc,730
-hockey_blast_common_lib-0.1.29.dist-info/METADATA,sha256=Dhg1VvioCggFoPw2GqvIP0DSOIU0VswxuKbxCs5IYQo,318
-hockey_blast_common_lib-0.1.29.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-hockey_blast_common_lib-0.1.29.dist-info/top_level.txt,sha256=wIR4LIkE40npoA2QlOdfCYlgFeGbsHR8Z6r0h46Vtgc,24
-hockey_blast_common_lib-0.1.29.dist-info/RECORD,,
+hockey_blast_common_lib-0.1.31.dist-info/METADATA,sha256=F19EMmlAfN34YH4YhXsATH-TdZ3ROGICcIU-T10A5yw,318
+hockey_blast_common_lib-0.1.31.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+hockey_blast_common_lib-0.1.31.dist-info/top_level.txt,sha256=wIR4LIkE40npoA2QlOdfCYlgFeGbsHR8Z6r0h46Vtgc,24
+hockey_blast_common_lib-0.1.31.dist-info/RECORD,,