hockey-blast-common-lib 0.1.32__py3-none-any.whl → 0.1.34__py3-none-any.whl

hockey_blast_common_lib/aggregate_all_stats.py

@@ -0,0 +1,26 @@
+import sys, os
+
+# Add the package directory to the Python path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from hockey_blast_common_lib.aggregate_human_stats import run_aggregate_human_stats
+from hockey_blast_common_lib.aggregate_skater_stats import run_aggregate_skater_stats
+from hockey_blast_common_lib.aggregate_goalie_stats import run_aggregate_goalie_stats
+from hockey_blast_common_lib.aggregate_referee_stats import run_aggregate_referee_stats
+
+if __name__ == "__main__":
+    print("Running aggregate_human_stats...")
+    run_aggregate_human_stats()
+    print("Finished running aggregate_human_stats\n")
+
+    print("Running aggregate_skater_stats...")
+    run_aggregate_skater_stats()
+    print("Finished running aggregate_skater_stats\n")
+
+    print("Running aggregate_goalie_stats...")
+    run_aggregate_goalie_stats()
+    print("Finished running aggregate_goalie_stats\n")
+
+    print("Running aggregate_referee_stats...")
+    run_aggregate_referee_stats()
+    print("Finished running aggregate_referee_stats\n")

hockey_blast_common_lib/aggregate_goalie_stats.py

@@ -5,16 +5,31 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from datetime import datetime, timedelta
 import sqlalchemy
-from hockey_blast_common_lib.models import  Game, GameRoster
-from hockey_blast_common_lib.stats_models import OrgStatsGoalie, DivisionStatsGoalie, OrgStatsWeeklyGoalie, OrgStatsDailyGoalie, DivisionStatsWeeklyGoalie, DivisionStatsDailyGoalie
+
+from hockey_blast_common_lib.models import Game, Goal, Penalty, GameRoster, Organization, Division, Human, Level
+from hockey_blast_common_lib.stats_models import OrgStatsGoalie, DivisionStatsGoalie, OrgStatsWeeklyGoalie, OrgStatsDailyGoalie, DivisionStatsWeeklyGoalie, DivisionStatsDailyGoalie, LevelStatsGoalie
 from hockey_blast_common_lib.db_connection import create_session
 from sqlalchemy.sql import func, case
-from hockey_blast_common_lib.options import not_human_names, parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS
-from hockey_blast_common_lib.utils import get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues, get_all_division_ids_for_org
+from hockey_blast_common_lib.options import not_human_names, parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, MIN_GAMES_FOR_LEVEL_STATS
+from hockey_blast_common_lib.utils import get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues, get_all_division_ids_for_org, get_start_datetime
+from hockey_blast_common_lib.utils import assign_ranks
+from sqlalchemy import func, case, and_
+from collections import defaultdict
 
-def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_filter_out, aggregation_window=None):
+def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_filter_out, debug_human_id=None, aggregation_window=None):
     human_ids_to_filter = get_human_ids_by_names(session, names_to_filter_out)
 
+    # Get the name of the aggregation, for debug purposes
+    if aggregation_type == 'org':
+        aggregation_name = session.query(Organization).filter(Organization.id == aggregation_id).first().organization_name
+        print(f"Aggregating goalie stats for {aggregation_name} with window {aggregation_window}...")
+    elif aggregation_type == 'division':
+        aggregation_name = session.query(Division).filter(Division.id == aggregation_id).first().level
+    elif aggregation_type == 'level':
+        aggregation_name = session.query(Level).filter(Level.id == aggregation_id).first().level_name
+    else:
+        aggregation_name = "Unknown"
+
     if aggregation_type == 'org':
         if aggregation_window == 'Daily':
             StatsModel = OrgStatsDailyGoalie
@@ -33,28 +48,34 @@ def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_f
             StatsModel = DivisionStatsGoalie
         min_games = MIN_GAMES_FOR_DIVISION_STATS
         filter_condition = Game.division_id == aggregation_id
+    elif aggregation_type == 'level':
+        StatsModel = LevelStatsGoalie
+        min_games = MIN_GAMES_FOR_LEVEL_STATS
+        filter_condition = Division.level_id == aggregation_id
+        # Add filter to only include games for the last 5 years
+        five_years_ago = datetime.now() - timedelta(days=5*365)
+        level_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP) >= five_years_ago
+        filter_condition = filter_condition & level_window_filter
     else:
         raise ValueError("Invalid aggregation type")
 
     # Apply aggregation window filter
     if aggregation_window:
-        last_game_datetime = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
-        if last_game_datetime:
-            last_game_datetime = datetime.strptime(last_game_datetime, '%Y-%m-%d %H:%M:%S')
-            if aggregation_window == 'Daily':
-                start_datetime = last_game_datetime - timedelta(days=1)
-            elif aggregation_window == 'Weekly':
-                start_datetime = last_game_datetime - timedelta(weeks=1)
-            else:
-                start_datetime = None
-            if start_datetime:
-                game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime)
-                filter_condition = filter_condition & game_window_filter
+        last_game_datetime_str = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
+        start_datetime = get_start_datetime(last_game_datetime_str, aggregation_window)
+        if start_datetime:
+            game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime_str)
+            filter_condition = filter_condition & game_window_filter
 
     # Delete existing items from the stats table
     session.query(StatsModel).filter(StatsModel.aggregation_id == aggregation_id).delete()
     session.commit()
 
+    # Filter for specific human_id if provided
+    human_filter = []
+    # if debug_human_id:
+    #     human_filter = [GameRoster.human_id == debug_human_id]
+
     # Aggregate games played, goals allowed, and shots faced for each goalie
     goalie_stats = session.query(
         GameRoster.human_id,
@@ -62,7 +83,7 @@ def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_f
         func.sum(case((GameRoster.team_id == Game.home_team_id, Game.visitor_final_score), else_=Game.home_final_score)).label('goals_allowed'),
         func.sum(case((GameRoster.team_id == Game.home_team_id, Game.visitor_period_1_shots + Game.visitor_period_2_shots + Game.visitor_period_3_shots + Game.visitor_ot_shots + Game.visitor_so_shots), else_=Game.home_period_1_shots + Game.home_period_2_shots + Game.home_period_3_shots + Game.home_ot_shots + Game.home_so_shots)).label('shots_faced'),
         func.array_agg(Game.id).label('game_ids')
-    ).join(Game, GameRoster.game_id == Game.id).filter(filter_condition, GameRoster.role == 'G').group_by(GameRoster.human_id).all()
+    ).join(Game, GameRoster.game_id == Game.id).join(Division, Game.division_id == Division.id).filter(filter_condition, GameRoster.role.ilike('g')).group_by(GameRoster.human_id).all()
 
     # Combine the results
     stats_dict = {}
@@ -70,6 +91,8 @@ def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_f
         if stat.human_id in human_ids_to_filter:
             continue
         key = (aggregation_id, stat.human_id)
+        if stat.games_played < min_games:
+            continue
         stats_dict[key] = {
             'games_played': stat.games_played,
             'goals_allowed': stat.goals_allowed if stat.goals_allowed is not None else 0,
@@ -102,37 +125,43 @@ def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_f
     # Calculate total_in_rank
     total_in_rank = len(stats_dict)
 
-    # Assign ranks
-    def assign_ranks(stats_dict, field):
-        sorted_stats = sorted(stats_dict.items(), key=lambda x: x[1][field], reverse=True)
-        for rank, (key, stat) in enumerate(sorted_stats, start=1):
-            stats_dict[key][f'{field}_rank'] = rank
-
+    # Assign ranks within each level
     assign_ranks(stats_dict, 'games_played')
-    assign_ranks(stats_dict, 'goals_allowed')
-    assign_ranks(stats_dict, 'goals_allowed_per_game')
+    assign_ranks(stats_dict, 'goals_allowed', reverse_rank=True)
     assign_ranks(stats_dict, 'shots_faced')
+    assign_ranks(stats_dict, 'goals_allowed_per_game', reverse_rank=True)
     assign_ranks(stats_dict, 'save_percentage')
 
+    # Debug output for specific human
+    if debug_human_id:
+        if any(key[1] == debug_human_id for key in stats_dict):
+            human = session.query(Human).filter(Human.id == debug_human_id).first()
+            human_name = f"{human.first_name} {human.last_name}" if human else "Unknown"
+            print(f"For Human {debug_human_id} ({human_name}) for {aggregation_type} {aggregation_id} ({aggregation_name}) , total_in_rank {total_in_rank} and window {aggregation_window}:")
+            for key, stat in stats_dict.items():
+                if key[1] == debug_human_id:
+                    for k, v in stat.items():
+                        print(f"{k}: {v}")
+
     # Insert aggregated stats into the appropriate table with progress output
     total_items = len(stats_dict)
     batch_size = 1000
     for i, (key, stat) in enumerate(stats_dict.items(), 1):
         aggregation_id, human_id = key
-        if stat['games_played'] < min_games:
-            continue
+        goals_allowed_per_game = stat['goals_allowed'] / stat['games_played'] if stat['games_played'] > 0 else 0.0
+        save_percentage = (stat['shots_faced'] - stat['goals_allowed']) / stat['shots_faced'] if stat['shots_faced'] > 0 else 0.0
         goalie_stat = StatsModel(
             aggregation_id=aggregation_id,
             human_id=human_id,
             games_played=stat['games_played'],
             goals_allowed=stat['goals_allowed'],
-            goals_allowed_per_game=stat['goals_allowed_per_game'],
             shots_faced=stat['shots_faced'],
-            save_percentage=stat['save_percentage'],
+            goals_allowed_per_game=goals_allowed_per_game,
+            save_percentage=save_percentage,
             games_played_rank=stat['games_played_rank'],
             goals_allowed_rank=stat['goals_allowed_rank'],
-            goals_allowed_per_game_rank=stat['goals_allowed_per_game_rank'],
             shots_faced_rank=stat['shots_faced_rank'],
+            goals_allowed_per_game_rank=stat['goals_allowed_per_game_rank'],
             save_percentage_rank=stat['save_percentage_rank'],
             total_in_rank=total_in_rank,
             first_game_id=stat['first_game_id'],
@@ -142,23 +171,45 @@ def aggregate_goalie_stats(session, aggregation_type, aggregation_id, names_to_f
         # Commit in batches
         if i % batch_size == 0:
             session.commit()
-            print(f"\r{i}/{total_items} ({(i/total_items)*100:.2f}%)", end="")
     session.commit()
-    print(f"\r{total_items}/{total_items} (100.00%)")
-    print("\nDone.")
 
-# Example usage
-if __name__ == "__main__":
-    args = parse_args()
-    org_alias = args.org
+def run_aggregate_goalie_stats():
     session = create_session("boss")
-    org_id = get_org_id_from_alias(session, org_alias)
-    division_ids = get_division_ids_for_last_season_in_all_leagues(session, org_id)
-    print(f"Aggregating goalie stats for {len(division_ids)} divisions in {org_alias}...")
-    for division_id in division_ids:
-        aggregate_goalie_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names)
-        aggregate_goalie_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, aggregation_window='Daily')
-        aggregate_goalie_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, aggregation_window='Weekly')
-    aggregate_goalie_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names)
-    aggregate_goalie_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, aggregation_window='Daily')
-    aggregate_goalie_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, aggregation_window='Weekly')
+    human_id_to_debug = None
+
+    # Get all org_id present in the Organization table
+    org_ids = session.query(Organization.id).all()
+    org_ids = [org_id[0] for org_id in org_ids]
+
+    for org_id in org_ids:
+        division_ids = get_all_division_ids_for_org(session, org_id)
+        print(f"Aggregating goalie stats for {len(division_ids)} divisions in org_id {org_id}...")
+        total_divisions = len(division_ids)
+        processed_divisions = 0
+        for division_id in division_ids:
+            aggregate_goalie_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)
+            aggregate_goalie_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Weekly')
+            aggregate_goalie_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Daily')
+            processed_divisions += 1
+            if human_id_to_debug is None:
+                print(f"\rProcessed {processed_divisions}/{total_divisions} divisions ({(processed_divisions/total_divisions)*100:.2f}%)", end="")
+
+        aggregate_goalie_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)
+        aggregate_goalie_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Weekly')
+        aggregate_goalie_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug, aggregation_window='Daily')
+        
+        # Aggregate by level
+    level_ids = session.query(Division.level_id).distinct().all()
+    level_ids = [level_id[0] for level_id in level_ids]
+    total_levels = len(level_ids)
+    processed_levels = 0
+    for level_id in level_ids:
+        if level_id is None:
+            continue
+        if human_id_to_debug is None:
+            print(f"\rProcessed {processed_levels}/{total_levels} levels ({(processed_levels/total_levels)*100:.2f}%)", end="")
+        processed_levels += 1
+        aggregate_goalie_stats(session, aggregation_type='level', aggregation_id=level_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)
+
+if __name__ == "__main__":
+    run_aggregate_goalie_stats()

hockey_blast_common_lib/aggregate_human_stats.py

@@ -6,17 +6,21 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from datetime import datetime, timedelta
 import sqlalchemy
-from hockey_blast_common_lib.models import Game, GameRoster
-from hockey_blast_common_lib.stats_models import OrgStatsHuman, DivisionStatsHuman, OrgStatsDailyHuman, OrgStatsWeeklyHuman, DivisionStatsDailyHuman, DivisionStatsWeeklyHuman
+from hockey_blast_common_lib.models import Game, GameRoster, Organization, Division
+from hockey_blast_common_lib.stats_models import OrgStatsHuman, DivisionStatsHuman, OrgStatsDailyHuman, OrgStatsWeeklyHuman, DivisionStatsDailyHuman, DivisionStatsWeeklyHuman, LevelStatsHuman
 from hockey_blast_common_lib.db_connection import create_session
 from sqlalchemy.sql import func, case
-from hockey_blast_common_lib.options import parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, not_human_names
-from hockey_blast_common_lib.utils import get_fake_human_for_stats, get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues
+from hockey_blast_common_lib.options import parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, MIN_GAMES_FOR_LEVEL_STATS, not_human_names
+from hockey_blast_common_lib.utils import get_fake_human_for_stats, get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues, get_all_division_ids_for_org
+from hockey_blast_common_lib.utils import assign_ranks
+from hockey_blast_common_lib.utils import get_start_datetime
 
 def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_filter_out, human_id_filter=None, aggregation_window=None):
     human_ids_to_filter = get_human_ids_by_names(session, names_to_filter_out)
 
     if aggregation_type == 'org':
+        aggregation_name = session.query(Organization).filter(Organization.id == aggregation_id).first().organization_name
+        print(f"Aggregating goalie stats for {aggregation_name} with window {aggregation_window}...")
         if aggregation_window == 'Daily':
             StatsModel = OrgStatsDailyHuman
         elif aggregation_window == 'Weekly':
@@ -34,9 +38,25 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
             StatsModel = DivisionStatsHuman
         min_games = MIN_GAMES_FOR_DIVISION_STATS
         filter_condition = Game.division_id == aggregation_id
+    elif aggregation_type == 'level':
+        StatsModel = LevelStatsHuman
+        min_games = MIN_GAMES_FOR_LEVEL_STATS
+        filter_condition = Division.level_id == aggregation_id
+        # Add filter to only include games for the last 5 years
+        five_years_ago = datetime.now() - timedelta(days=5*365)
+        level_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP) >= five_years_ago
+        filter_condition = filter_condition & level_window_filter
     else:
         raise ValueError("Invalid aggregation type")
 
+    # Apply aggregation window filter
+    if aggregation_window:
+        last_game_datetime_str = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
+        start_datetime = get_start_datetime(last_game_datetime_str, aggregation_window)
+        if start_datetime:
+            game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime_str)
+            filter_condition = filter_condition & game_window_filter
+
     # Delete existing items from the stats table
     session.query(StatsModel).filter(StatsModel.aggregation_id == aggregation_id).delete()
     session.commit()
@@ -49,65 +69,94 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
     # Filter games by status
     game_status_filter = Game.status.like('Final%')
 
-    # Apply aggregation window filter
-    if aggregation_window:
-        last_game_datetime = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, game_status_filter).scalar()
-        if last_game_datetime:
-            last_game_datetime = datetime.strptime(last_game_datetime, '%Y-%m-%d %H:%M:%S')
-            if aggregation_window == 'Daily':
-                start_datetime = last_game_datetime - timedelta(days=1)
-            elif aggregation_window == 'Weekly':
-                start_datetime = last_game_datetime - timedelta(weeks=1)
-            else:
-                start_datetime = None
-            if start_datetime:
-                game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime)
-                filter_condition = filter_condition & game_window_filter
-
-    # Aggregate games played for each human in each role
-    human_stats = session.query(
+    # Aggregate skater games played
+    skater_stats = session.query(
+        GameRoster.human_id,
+        func.count(func.distinct(Game.id)).label('games_skater'),
+        func.array_agg(func.distinct(Game.id)).label('skater_game_ids')
+    ).join(Game, GameRoster.game_id == Game.id).join(Division, Game.division_id == Division.id).filter(filter_condition, game_status_filter, ~GameRoster.role.ilike('G'), *human_filter).group_by(GameRoster.human_id).all()
+
+    # Aggregate goalie games played
+    goalie_stats = session.query(
         GameRoster.human_id,
-        func.count(func.distinct(case((GameRoster.role != 'G', Game.id), else_=None))).label('games_skater'),
-        func.count(func.distinct(case((GameRoster.role == 'G', Game.id), else_=None))).label('games_goalie'),
-        func.array_agg(func.distinct(Game.id)).label('game_ids')
-    ).join(Game, GameRoster.game_id == Game.id).filter(filter_condition, game_status_filter, *human_filter).group_by(GameRoster.human_id).all()
+        func.count(func.distinct(Game.id)).label('games_goalie'),
+        func.array_agg(func.distinct(Game.id)).label('goalie_game_ids')
+    ).join(Game, GameRoster.game_id == Game.id).join(Division, Game.division_id == Division.id).filter(filter_condition, game_status_filter, GameRoster.role.ilike('G'), *human_filter).group_by(GameRoster.human_id).all()
 
     # Aggregate referee and scorekeeper games from Game table
     referee_stats = session.query(
         Game.referee_1_id.label('human_id'),
         func.count(func.distinct(Game.id)).label('games_referee'),
         func.array_agg(func.distinct(Game.id)).label('referee_game_ids')
-    ).filter(filter_condition, game_status_filter, *human_filter).group_by(Game.referee_1_id).all()
+    ).join(Division, Game.division_id == Division.id).filter(filter_condition, game_status_filter, *human_filter).group_by(Game.referee_1_id).all()
 
     referee_stats_2 = session.query(
         Game.referee_2_id.label('human_id'),
         func.count(func.distinct(Game.id)).label('games_referee'),
         func.array_agg(func.distinct(Game.id)).label('referee_game_ids')
-    ).filter(filter_condition, game_status_filter, *human_filter).group_by(Game.referee_2_id).all()
+    ).join(Division, Game.division_id == Division.id).filter(filter_condition, game_status_filter, *human_filter).group_by(Game.referee_2_id).all()
 
     scorekeeper_stats = session.query(
         Game.scorekeeper_id.label('human_id'),
         func.count(func.distinct(Game.id)).label('games_scorekeeper'),
         func.array_agg(func.distinct(Game.id)).label('scorekeeper_game_ids')
-    ).filter(filter_condition, game_status_filter, *human_filter).group_by(Game.scorekeeper_id).all()
+    ).join(Division, Game.division_id == Division.id).filter(filter_condition, game_status_filter, *human_filter).group_by(Game.scorekeeper_id).all()
 
     # Combine the results
     stats_dict = {}
-    for stat in human_stats:
+    for stat in skater_stats:
         if stat.human_id in human_ids_to_filter:
             continue
         key = (aggregation_id, stat.human_id)
         stats_dict[key] = {
-            'games_total': stat.games_skater + stat.games_goalie,
+            'games_total': stat.games_skater,
             'games_skater': stat.games_skater,
-            'games_goalie': stat.games_goalie,
+            'games_goalie': 0,
             'games_referee': 0,
             'games_scorekeeper': 0,
-            'game_ids': stat.game_ids,
+            'skater_game_ids': stat.skater_game_ids,
+            'goalie_game_ids': [],
             'referee_game_ids': [],
-            'scorekeeper_game_ids': []
+            'scorekeeper_game_ids': [],
+            'first_game_id_skater': None,
+            'last_game_id_skater': None,
+            'first_game_id_goalie': None,
+            'last_game_id_goalie': None,
+            'first_game_id_referee': None,
+            'last_game_id_referee': None,
+            'first_game_id_scorekeeper': None,
+            'last_game_id_scorekeeper': None
         }
 
+    for stat in goalie_stats:
+        if stat.human_id in human_ids_to_filter:
+            continue
+        key = (aggregation_id, stat.human_id)
+        if key not in stats_dict:
+            stats_dict[key] = {
+                'games_total': stat.games_goalie,
+                'games_skater': 0,
+                'games_goalie': stat.games_goalie,
+                'games_referee': 0,
+                'games_scorekeeper': 0,
+                'skater_game_ids': [],
+                'goalie_game_ids': stat.goalie_game_ids,
+                'referee_game_ids': [],
+                'scorekeeper_game_ids': [],
+                'first_game_id_skater': None,
+                'last_game_id_skater': None,
+                'first_game_id_goalie': None,
+                'last_game_id_goalie': None,
+                'first_game_id_referee': None,
+                'last_game_id_referee': None,
+                'first_game_id_scorekeeper': None,
+                'last_game_id_scorekeeper': None
+            }
+        else:
+            stats_dict[key]['games_goalie'] += stat.games_goalie
+            stats_dict[key]['games_total'] += stat.games_goalie
+            stats_dict[key]['goalie_game_ids'] += stat.goalie_game_ids
+
     for stat in referee_stats:
         if stat.human_id in human_ids_to_filter:
             continue
@@ -119,9 +168,18 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
                 'games_goalie': 0,
                 'games_referee': stat.games_referee,
                 'games_scorekeeper': 0,
-                'game_ids': [],
+                'skater_game_ids': [],
+                'goalie_game_ids': [],
                 'referee_game_ids': stat.referee_game_ids,
-                'scorekeeper_game_ids': []
+                'scorekeeper_game_ids': [],
+                'first_game_id_skater': None,
+                'last_game_id_skater': None,
+                'first_game_id_goalie': None,
+                'last_game_id_goalie': None,
+                'first_game_id_referee': None,
+                'last_game_id_referee': None,
+                'first_game_id_scorekeeper': None,
+                'last_game_id_scorekeeper': None
             }
         else:
             stats_dict[key]['games_referee'] += stat.games_referee
@@ -139,9 +197,18 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
                 'games_goalie': 0,
                 'games_referee': stat.games_referee,
                 'games_scorekeeper': 0,
-                'game_ids': [],
+                'skater_game_ids': [],
+                'goalie_game_ids': [],
                 'referee_game_ids': stat.referee_game_ids,
-                'scorekeeper_game_ids': []
+                'scorekeeper_game_ids': [],
+                'first_game_id_skater': None,
+                'last_game_id_skater': None,
+                'first_game_id_goalie': None,
+                'last_game_id_goalie': None,
+                'first_game_id_referee': None,
+                'last_game_id_referee': None,
+                'first_game_id_scorekeeper': None,
+                'last_game_id_scorekeeper': None
             }
         else:
             stats_dict[key]['games_referee'] += stat.games_referee
@@ -159,9 +226,18 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
                 'games_goalie': 0,
                 'games_referee': 0,
                 'games_scorekeeper': stat.games_scorekeeper,
-                'game_ids': [],
+                'skater_game_ids': [],
+                'goalie_game_ids': [],
                 'referee_game_ids': [],
-                'scorekeeper_game_ids': stat.scorekeeper_game_ids
+                'scorekeeper_game_ids': stat.scorekeeper_game_ids,
+                'first_game_id_skater': None,
+                'last_game_id_skater': None,
+                'first_game_id_goalie': None,
+                'last_game_id_goalie': None,
+                'first_game_id_referee': None,
+                'last_game_id_referee': None,
+                'first_game_id_scorekeeper': None,
+                'last_game_id_scorekeeper': None
             }
         else:
             stats_dict[key]['games_scorekeeper'] += stat.games_scorekeeper
@@ -174,36 +250,61 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
     # Calculate total_in_rank
     total_in_rank = len(stats_dict)
 
-    # Assign ranks
-    def assign_ranks(stats_dict, field):
-        sorted_stats = sorted(stats_dict.items(), key=lambda x: x[1][field], reverse=True)
-        for rank, (key, stat) in enumerate(sorted_stats, start=1):
-            stats_dict[key][f'{field}_rank'] = rank
+    # Calculate number of items in rank per role
+    skaters_in_rank = len([stat for stat in stats_dict.values() if stat['games_skater'] > 0])
+    goalies_in_rank = len([stat for stat in stats_dict.values() if stat['games_goalie'] > 0])
+    referees_in_rank = len([stat for stat in stats_dict.values() if stat['games_referee'] > 0])
+    scorekeepers_in_rank = len([stat for stat in stats_dict.values() if stat['games_scorekeeper'] > 0])
+
+    # Filter out humans with less than min_games
+    stats_dict = {key: value for key, value in stats_dict.items() if value['games_total'] >= min_games}
 
+    # Assign ranks
     assign_ranks(stats_dict, 'games_total')
     assign_ranks(stats_dict, 'games_skater')
     assign_ranks(stats_dict, 'games_goalie')
     assign_ranks(stats_dict, 'games_referee')
     assign_ranks(stats_dict, 'games_scorekeeper')
 
-    # Populate first_game_id and last_game_id
+    # Populate first_game_id and last_game_id for each role
     for key, stat in stats_dict.items():
-        all_game_ids = stat['game_ids'] + stat['referee_game_ids'] + stat['scorekeeper_game_ids']
+        all_game_ids = stat['skater_game_ids'] + stat['goalie_game_ids'] + stat['referee_game_ids'] + stat['scorekeeper_game_ids']
         if all_game_ids:
             first_game = session.query(Game).filter(Game.id.in_(all_game_ids)).order_by(Game.date, Game.time).first()
             last_game = session.query(Game).filter(Game.id.in_(all_game_ids)).order_by(Game.date.desc(), Game.time.desc()).first()
             stat['first_game_id'] = first_game.id if first_game else None
             stat['last_game_id'] = last_game.id if last_game else None
 
+        if stat['skater_game_ids']:
+            first_game_skater = session.query(Game).filter(Game.id.in_(stat['skater_game_ids'])).order_by(Game.date, Game.time).first()
+            last_game_skater = session.query(Game).filter(Game.id.in_(stat['skater_game_ids'])).order_by(Game.date.desc(), Game.time.desc()).first()
+            stat['first_game_id_skater'] = first_game_skater.id if first_game_skater else None
+            stat['last_game_id_skater'] = last_game_skater.id if last_game_skater else None
+
+        if stat['goalie_game_ids']:
+            first_game_goalie = session.query(Game).filter(Game.id.in_(stat['goalie_game_ids'])).order_by(Game.date, Game.time).first()
+            last_game_goalie = session.query(Game).filter(Game.id.in_(stat['goalie_game_ids'])).order_by(Game.date.desc(), Game.time.desc()).first()
+            stat['first_game_id_goalie'] = first_game_goalie.id if first_game_goalie else None
+            stat['last_game_id_goalie'] = last_game_goalie.id if last_game_goalie else None
+
+        if stat['referee_game_ids']:
+            first_game_referee = session.query(Game).filter(Game.id.in_(stat['referee_game_ids'])).order_by(Game.date, Game.time).first()
+            last_game_referee = session.query(Game).filter(Game.id.in_(stat['referee_game_ids'])).order_by(Game.date.desc(), Game.time.desc()).first()
+            stat['first_game_id_referee'] = first_game_referee.id if first_game_referee else None
+            stat['last_game_id_referee'] = last_game_referee.id if last_game_referee else None
+
+        if stat['scorekeeper_game_ids']:
+            first_game_scorekeeper = session.query(Game).filter(Game.id.in_(stat['scorekeeper_game_ids'])).order_by(Game.date, Game.time).first()
+            last_game_scorekeeper = session.query(Game).filter(Game.id.in_(stat['scorekeeper_game_ids'])).order_by(Game.date.desc(), Game.time.desc()).first()
+            stat['first_game_id_scorekeeper'] = first_game_scorekeeper.id if first_game_scorekeeper else None
+            stat['last_game_id_scorekeeper'] = last_game_scorekeeper.id if last_game_scorekeeper else None
+
     # Insert aggregated stats into the appropriate table with progress output
-    total_items = len(stats_dict)
     batch_size = 1000
     for i, (key, stat) in enumerate(stats_dict.items(), 1):
         aggregation_id, human_id = key
         if human_id_filter and human_id != human_id_filter:
             continue
-        if stat['games_total'] < min_games:
-            continue
 
         human_stat = StatsModel(
             aggregation_id=aggregation_id,
@@ -219,14 +320,25 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
             games_scorekeeper=stat['games_scorekeeper'],
             games_scorekeeper_rank=stat['games_scorekeeper_rank'],
             total_in_rank=total_in_rank,
+            skaters_in_rank=skaters_in_rank,
+            goalies_in_rank=goalies_in_rank,
+            referees_in_rank=referees_in_rank,
+            scorekeepers_in_rank=scorekeepers_in_rank,
             first_game_id=stat['first_game_id'],
-            last_game_id=stat['last_game_id']
+            last_game_id=stat['last_game_id'],
+            first_game_id_skater=stat['first_game_id_skater'],
+            last_game_id_skater=stat['last_game_id_skater'],
+            first_game_id_goalie=stat['first_game_id_goalie'],
+            last_game_id_goalie=stat['last_game_id_goalie'],
+            first_game_id_referee=stat['first_game_id_referee'],
+            last_game_id_referee=stat['last_game_id_referee'],
+            first_game_id_scorekeeper=stat['first_game_id_scorekeeper'],
+            last_game_id_scorekeeper=stat['last_game_id_scorekeeper']
         )
         session.add(human_stat)
         # Commit in batches
         if i % batch_size == 0:
             session.commit()
-            print(f"\r{i}/{total_items} ({(i/total_items)*100:.2f}%)", end="")
     session.commit()
 
     # Fetch fake human ID for overall stats
@@ -240,12 +352,24 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
         'games_referee': sum(stat['games_referee'] for stat in stats_dict.values()),
         'games_scorekeeper': sum(stat['games_scorekeeper'] for stat in stats_dict.values()),
         'total_in_rank': total_in_rank,
+        'skaters_in_rank': skaters_in_rank,
+        'goalies_in_rank': goalies_in_rank,
+        'referees_in_rank': referees_in_rank,
+        'scorekeepers_in_rank': scorekeepers_in_rank,
         'first_game_id': None,
-        'last_game_id': None
+        'last_game_id': None,
+        'first_game_id_skater': None,
+        'last_game_id_skater': None,
+        'first_game_id_goalie': None,
+        'last_game_id_goalie': None,
+        'first_game_id_referee': None,
+        'last_game_id_referee': None,
+        'first_game_id_scorekeeper': None,
+        'last_game_id_scorekeeper': None
     }
 
     # Populate first_game_id and last_game_id for overall stats
-    all_game_ids = [game_id for stat in stats_dict.values() for game_id in stat['game_ids'] + stat['referee_game_ids'] + stat['scorekeeper_game_ids']]
+    all_game_ids = [game_id for stat in stats_dict.values() for game_id in stat['skater_game_ids'] + stat['goalie_game_ids'] + stat['referee_game_ids'] + stat['scorekeeper_game_ids']]
     if all_game_ids:
         first_game = session.query(Game).filter(Game.id.in_(all_game_ids)).order_by(Game.date, Game.time).first()
         last_game = session.query(Game).filter(Game.id.in_(all_game_ids)).order_by(Game.date.desc(), Game.time.desc()).first()
@@ -267,29 +391,61 @@ def aggregate_human_stats(session, aggregation_type, aggregation_id, names_to_fi
         games_scorekeeper=overall_stats['games_scorekeeper'],
         games_scorekeeper_rank=0,  # Overall stats do not need a rank
         total_in_rank=overall_stats['total_in_rank'],
+        skaters_in_rank=overall_stats['skaters_in_rank'],
+        goalies_in_rank=overall_stats['goalies_in_rank'],
+        referees_in_rank=overall_stats['referees_in_rank'],
+        scorekeepers_in_rank=overall_stats['scorekeepers_in_rank'],
         first_game_id=overall_stats['first_game_id'],
-        last_game_id=overall_stats['last_game_id']
+        last_game_id=overall_stats['last_game_id'],
+        first_game_id_skater=overall_stats['first_game_id_skater'],
+        last_game_id_skater=overall_stats['last_game_id_skater'],
+        first_game_id_goalie=overall_stats['first_game_id_goalie'],
+        last_game_id_goalie=overall_stats['last_game_id_goalie'],
+        first_game_id_referee=overall_stats['first_game_id_referee'],
+        last_game_id_referee=overall_stats['last_game_id_referee'],
+        first_game_id_scorekeeper=overall_stats['first_game_id_scorekeeper'],
+        last_game_id_scorekeeper=overall_stats['last_game_id_scorekeeper']
     )
     session.add(overall_human_stat)
     session.commit()
 
-    print(f"\r{total_items}/{total_items} (100.00%)")
-    print("\nDone.")
+def run_aggregate_human_stats():
+    session = create_session("boss")
+    human_id_to_debug = None
+
+    # Aggregate by Org and Division inside Org
+    org_ids = session.query(Organization.id).all()
+    org_ids = [org_id[0] for org_id in org_ids]
+
+    for org_id in org_ids:
+        division_ids = get_all_division_ids_for_org(session, org_id)
+        print(f"Aggregating human stats for {len(division_ids)} divisions in org_id {org_id}...")
+        total_divisions = len(division_ids)
+        processed_divisions = 0
+        for division_id in division_ids:
+            aggregate_human_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug)
+            aggregate_human_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug, aggregation_window='Weekly')
+            aggregate_human_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug, aggregation_window='Daily')
+            processed_divisions += 1
+            if human_id_to_debug is None:
+                print(f"\rProcessed {processed_divisions}/{total_divisions} divisions ({(processed_divisions/total_divisions)*100:.2f}%)", end="")
+        print("")
+        aggregate_human_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug)
+        aggregate_human_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug, aggregation_window='Weekly')
+        aggregate_human_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug, aggregation_window='Daily')
+        
+        # Aggregate by level
+    level_ids = session.query(Division.level_id).distinct().all()
+    level_ids = [level_id[0] for level_id in level_ids]
+    total_levels = len(level_ids)
+    processed_levels = 0
+    for level_id in level_ids:
+        if level_id is None:
+            continue
+        if human_id_to_debug is None:
+            print(f"\rProcessed {processed_levels}/{total_levels} levels ({(processed_levels/total_levels)*100:.2f}%)", end="")
+        processed_levels += 1
+        aggregate_human_stats(session, aggregation_type='level', aggregation_id=level_id, names_to_filter_out=not_human_names, human_id_filter=human_id_to_debug)
 
-# Example usage
 if __name__ == "__main__":
-    args = parse_args()
-    org_alias=args.org
-    session = create_session("boss")
-    org_id = get_org_id_from_alias(session, org_alias)
-
-    division_ids = get_division_ids_for_last_season_in_all_leagues(session, org_id)
-    print(f"Aggregating human stats for {len(division_ids)} divisions in {org_alias}...")
-    for division_id in division_ids:
-        aggregate_human_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, human_id_filter=None)
-        aggregate_human_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, human_id_filter=None, aggregation_window='Daily')
-        aggregate_human_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, human_id_filter=None, aggregation_window='Weekly')
-
-    aggregate_human_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, human_id_filter=None)
-    aggregate_human_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, human_id_filter=None, aggregation_window='Daily')
-    aggregate_human_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, human_id_filter=None, aggregation_window='Weekly')
+    run_aggregate_human_stats()

hockey_blast_common_lib/aggregate_referee_stats.py

@@ -6,17 +6,21 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from datetime import datetime, timedelta
 import sqlalchemy
-from hockey_blast_common_lib.models import Game, Penalty
-from hockey_blast_common_lib.stats_models import OrgStatsReferee, DivisionStatsReferee,OrgStatsWeeklyReferee, OrgStatsDailyReferee, DivisionStatsWeeklyReferee, DivisionStatsDailyReferee
+from hockey_blast_common_lib.models import Game, Penalty, Organization, Division
+from hockey_blast_common_lib.stats_models import OrgStatsReferee, DivisionStatsReferee, OrgStatsWeeklyReferee, OrgStatsDailyReferee, DivisionStatsWeeklyReferee, DivisionStatsDailyReferee, LevelStatsReferee
 from hockey_blast_common_lib.db_connection import create_session
 from sqlalchemy.sql import func, case
-from hockey_blast_common_lib.options import parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, not_human_names
-from hockey_blast_common_lib.utils import get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues
+from hockey_blast_common_lib.options import parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, MIN_GAMES_FOR_LEVEL_STATS, not_human_names
+from hockey_blast_common_lib.utils import get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues, get_all_division_ids_for_org
+from hockey_blast_common_lib.utils import assign_ranks
+from hockey_blast_common_lib.utils import get_start_datetime
 
 def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_filter_out, aggregation_window=None):
     human_ids_to_filter = get_human_ids_by_names(session, names_to_filter_out)
 
     if aggregation_type == 'org':
+        aggregation_name = session.query(Organization).filter(Organization.id == aggregation_id).first().organization_name
+        print(f"Aggregating referee stats for {aggregation_name} with window {aggregation_window}...")
         if aggregation_window == 'Daily':
             StatsModel = OrgStatsDailyReferee
         elif aggregation_window == 'Weekly':
@@ -34,23 +38,24 @@ def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_
             StatsModel = DivisionStatsReferee
         min_games = MIN_GAMES_FOR_DIVISION_STATS
         filter_condition = Game.division_id == aggregation_id
+    elif aggregation_type == 'level':
+        StatsModel = LevelStatsReferee
+        min_games = MIN_GAMES_FOR_LEVEL_STATS
+        filter_condition = Division.level_id == aggregation_id
+        # Add filter to only include games for the last 5 years
+        five_years_ago = datetime.now() - timedelta(days=5*365)
+        level_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP) >= five_years_ago
+        filter_condition = filter_condition & level_window_filter
     else:
         raise ValueError("Invalid aggregation type")
 
     # Apply aggregation window filter
     if aggregation_window:
-        last_game_datetime = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
-        if last_game_datetime:
-            last_game_datetime = datetime.strptime(last_game_datetime, '%Y-%m-%d %H:%M:%S')
-            if aggregation_window == 'Daily':
-                start_datetime = last_game_datetime - timedelta(days=1)
-            elif aggregation_window == 'Weekly':
-                start_datetime = last_game_datetime - timedelta(weeks=1)
-            else:
-                start_datetime = None
-            if start_datetime:
-                game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime)
-                filter_condition = filter_condition & game_window_filter
+        last_game_datetime_str = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
+        start_datetime = get_start_datetime(last_game_datetime_str, aggregation_window)
+        if start_datetime:
+            game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime_str)
+            filter_condition = filter_condition & game_window_filter
 
     # Delete existing items from the stats table
     session.query(StatsModel).filter(StatsModel.aggregation_id == aggregation_id).delete()
@@ -117,36 +122,14 @@ def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_
     for stat in penalties_given_stats:
         if stat.referee_1_id and stat.referee_1_id not in human_ids_to_filter:
             key = (aggregation_id, stat.referee_1_id)
-            if key not in stats_dict:
-                stats_dict[key] = {
-                    'games_reffed': 0,
-                    'penalties_given': stat.penalties_given / 2,
-                    'gm_given': stat.gm_given / 2,
-                    'penalties_per_game': 0.0,
-                    'gm_per_game': 0.0,
-                    'game_ids': [stat.game_id],
-                    'first_game_id': None,
-                    'last_game_id': None
-                }
-            else:
+            if key in stats_dict:
                 stats_dict[key]['penalties_given'] += stat.penalties_given / 2
                 stats_dict[key]['gm_given'] += stat.gm_given / 2
                 stats_dict[key]['game_ids'].append(stat.game_id)
 
         if stat.referee_2_id and stat.referee_2_id not in human_ids_to_filter:
             key = (aggregation_id, stat.referee_2_id)
-            if key not in stats_dict:
-                stats_dict[key] = {
-                    'games_reffed': 0,
-                    'penalties_given': stat.penalties_given / 2,
-                    'gm_given': stat.gm_given / 2,
-                    'penalties_per_game': 0.0,
-                    'gm_per_game': 0.0,
-                    'game_ids': [stat.game_id],
-                    'first_game_id': None,
-                    'last_game_id': None
-                }
-            else:
+            if key in stats_dict:
                 stats_dict[key]['penalties_given'] += stat.penalties_given / 2
                 stats_dict[key]['gm_given'] += stat.gm_given / 2
                 stats_dict[key]['game_ids'].append(stat.game_id)
@@ -160,6 +143,9 @@ def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_
     # Ensure all keys have valid human_id values
     stats_dict = {key: value for key, value in stats_dict.items() if key[1] is not None}
 
+    # Filter out referees with less than min_games
+    stats_dict = {key: value for key, value in stats_dict.items() if value['games_reffed'] >= min_games}
+
     # Populate first_game_id and last_game_id
     for key, stat in stats_dict.items():
         all_game_ids = stat['game_ids']
@@ -173,11 +159,6 @@ def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_
     total_in_rank = len(stats_dict)
 
     # Assign ranks
-    def assign_ranks(stats_dict, field):
-        sorted_stats = sorted(stats_dict.items(), key=lambda x: x[1][field], reverse=True)
-        for rank, (key, stat) in enumerate(sorted_stats, start=1):
-            stats_dict[key][f'{field}_rank'] = rank
-
     assign_ranks(stats_dict, 'games_reffed')
     assign_ranks(stats_dict, 'penalties_given')
     assign_ranks(stats_dict, 'penalties_per_game')
@@ -189,8 +170,6 @@ def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_
     batch_size = 1000
     for i, (key, stat) in enumerate(stats_dict.items(), 1):
         aggregation_id, human_id = key
-        if stat['games_reffed'] < min_games:
-            continue
         referee_stat = StatsModel(
             aggregation_id=aggregation_id,
             human_id=human_id,
@@ -212,23 +191,45 @@ def aggregate_referee_stats(session, aggregation_type, aggregation_id, names_to_
         # Commit in batches
         if i % batch_size == 0:
             session.commit()
-            print(f"\r{i}/{total_items} ({(i/total_items)*100:.2f}%)", end="")
     session.commit()
-    print(f"\r{total_items}/{total_items} (100.00%)")
-    print("\nDone.")
 
-# Example usage
-if __name__ == "__main__":
-    args = parse_args()
-    org_alias = args.org
+def run_aggregate_referee_stats():
     session = create_session("boss")
-    org_id = get_org_id_from_alias(session, org_alias)
-    division_ids = get_division_ids_for_last_season_in_all_leagues(session, org_id)
-    print(f"Aggregating referee stats for {len(division_ids)} divisions in {org_alias}...")
-    for division_id in division_ids:
-        aggregate_referee_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names)
-        aggregate_referee_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, aggregation_window='Weekly')
-        aggregate_referee_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, aggregation_window='Daily')
-    aggregate_referee_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names)
-    aggregate_referee_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, aggregation_window='Weekly')
-    aggregate_referee_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, aggregation_window='Daily')
+    human_id_to_debug = None
+
+    # Get all org_id present in the Organization table
+    org_ids = session.query(Organization.id).all()
+    org_ids = [org_id[0] for org_id in org_ids]
+
+    for org_id in org_ids:
+        division_ids = get_all_division_ids_for_org(session, org_id)
+        print(f"Aggregating referee stats for {len(division_ids)} divisions in org_id {org_id}...")
+        total_divisions = len(division_ids)
+        processed_divisions = 0
+        for division_id in division_ids:
+            aggregate_referee_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names)
+            aggregate_referee_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, aggregation_window='Weekly')
+            aggregate_referee_stats(session, aggregation_type='division', aggregation_id=division_id, names_to_filter_out=not_human_names, aggregation_window='Daily')
+            processed_divisions += 1
+            if human_id_to_debug is None:
+                print(f"\rProcessed {processed_divisions}/{total_divisions} divisions ({(processed_divisions/total_divisions)*100:.2f}%)", end="")
+
+        aggregate_referee_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names)
+        aggregate_referee_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, aggregation_window='Weekly')
+        aggregate_referee_stats(session, aggregation_type='org', aggregation_id=org_id, names_to_filter_out=not_human_names, aggregation_window='Daily')
+        
+        # Aggregate by level
+    level_ids = session.query(Division.level_id).distinct().all()
+    level_ids = [level_id[0] for level_id in level_ids]
+    total_levels = len(level_ids)
+    processed_levels = 0
+    for level_id in level_ids:
+        if level_id is None:
+            continue
+        if human_id_to_debug is None:
+            print(f"\rProcessed {processed_levels}/{total_levels} levels ({(processed_levels/total_levels)*100:.2f}%)", end="")
+        processed_levels += 1
+        aggregate_referee_stats(session, aggregation_type='level', aggregation_id=level_id, names_to_filter_out=not_human_names)
+
+if __name__ == "__main__":
+    run_aggregate_referee_stats()

hockey_blast_common_lib/aggregate_skater_stats.py

@@ -12,6 +12,7 @@ from hockey_blast_common_lib.db_connection import create_session
 from sqlalchemy.sql import func, case
 from hockey_blast_common_lib.options import not_human_names, parse_args, MIN_GAMES_FOR_ORG_STATS, MIN_GAMES_FOR_DIVISION_STATS, MIN_GAMES_FOR_LEVEL_STATS
 from hockey_blast_common_lib.utils import get_org_id_from_alias, get_human_ids_by_names, get_division_ids_for_last_season_in_all_leagues, get_all_division_ids_for_org
+from hockey_blast_common_lib.utils import get_start_datetime
 from sqlalchemy import func, case, and_
 from collections import defaultdict
 
@@ -21,6 +22,9 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
     # Get the name of the aggregation, for debug purposes
     if aggregation_type == 'org':
         aggregation_name = session.query(Organization).filter(Organization.id == aggregation_id).first().organization_name
+        print(f"Aggregating skater stats for {aggregation_name} with window {aggregation_window}...")
+
+        aggregation_name = session.query(Organization).filter(Organization.id == aggregation_id).first().organization_name
     elif aggregation_type == 'division':
         aggregation_name = session.query(Division).filter(Division.id == aggregation_id).first().level
     elif aggregation_type == 'level':
@@ -59,18 +63,11 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
 
     # Apply aggregation window filter
     if aggregation_window:
-        last_game_datetime = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
-        if last_game_datetime:
-            last_game_datetime = datetime.strptime(last_game_datetime, '%Y-%m-%d %H:%M:%S')
-            if aggregation_window == 'Daily':
-                start_datetime = last_game_datetime - timedelta(days=1)
-            elif aggregation_window == 'Weekly':
-                start_datetime = last_game_datetime - timedelta(weeks=1)
-            else:
-                start_datetime = None
-            if start_datetime:
-                game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime)
-                filter_condition = filter_condition & game_window_filter
+        last_game_datetime_str = session.query(func.max(func.concat(Game.date, ' ', Game.time))).filter(filter_condition, Game.status.like('Final%')).scalar()
+        start_datetime = get_start_datetime(last_game_datetime_str, aggregation_window)
+        if start_datetime:
+            game_window_filter = func.cast(func.concat(Game.date, ' ', Game.time), sqlalchemy.types.TIMESTAMP).between(start_datetime, last_game_datetime_str)
+            filter_condition = filter_condition & game_window_filter
 
     # Delete existing items from the stats table
     session.query(StatsModel).filter(StatsModel.aggregation_id == aggregation_id).delete()
@@ -254,16 +251,11 @@ def aggregate_skater_stats(session, aggregation_type, aggregation_id, names_to_f
         # Commit in batches
         if i % batch_size == 0:
             session.commit()
-            if debug_human_id is None:
-                print(f"\r{i}/{total_items} ({(i/total_items)*100:.2f}%)", end="")
-            
     session.commit()
-    if debug_human_id is None:
-        print(f"\r{total_items}/{total_items} (100.00%)")
 
-if __name__ == "__main__":
+def run_aggregate_skater_stats():
     session = create_session("boss")
-    human_id_to_debug = 117076
+    human_id_to_debug = None
 
     # Get all org_id present in the Organization table
     org_ids = session.query(Organization.id).all()
@@ -298,3 +290,6 @@ if __name__ == "__main__":
             print(f"\rProcessed {processed_levels}/{total_levels} levels ({(processed_levels/total_levels)*100:.2f}%)", end="")
         processed_levels += 1
         aggregate_skater_stats(session, aggregation_type='level', aggregation_id=level_id, names_to_filter_out=not_human_names, debug_human_id=human_id_to_debug)
+
+if __name__ == "__main__":
+    run_aggregate_skater_stats()

hockey_blast_common_lib/models.py

@@ -224,6 +224,7 @@ class Organization(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     alias = db.Column(db.String(100), unique=True)
     organization_name = db.Column(db.String(100), unique=True)
+    website = db.Column(db.String(100), nullable=True)  # New field for website
 
 class Penalty(db.Model):
     __tablename__ = 'penalties'

hockey_blast_common_lib/skills_in_divisions.py

@@ -137,7 +137,7 @@ def populate_league_ids():
     session.commit()
     print("League IDs have been populated into the Season table.")
 
-if __name__ == "__main__":
+#if __name__ == "__main__":
     # delete_all_skills()
     #fill_seed_skills()
     #populate_season_ids()  # Call the function to populate season_ids

hockey_blast_common_lib/stats_models.py

@@ -17,8 +17,20 @@ class BaseStatsHuman(db.Model):
     games_goalie = db.Column(db.Integer, default=0)
     games_goalie_rank = db.Column(db.Integer, default=0)
     total_in_rank = db.Column(db.Integer, default=0)
-    first_game_id = db.Column(db.Integer, db.ForeignKey('games.id'))
-    last_game_id = db.Column(db.Integer, db.ForeignKey('games.id'))
+    skaters_in_rank = db.Column(db.Integer, default=0)
+    goalies_in_rank = db.Column(db.Integer, default=0)
+    referees_in_rank = db.Column(db.Integer, default=0)
+    scorekeepers_in_rank = db.Column(db.Integer, default=0)
+    first_game_id = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    last_game_id = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    first_game_id_skater = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    last_game_id_skater = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    first_game_id_goalie = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    last_game_id_goalie = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    first_game_id_referee = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    last_game_id_referee = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    first_game_id_scorekeeper = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
+    last_game_id_scorekeeper = db.Column(db.Integer, db.ForeignKey('games.id'), nullable=True)
 
     @declared_attr
     def __table_args__(cls):
@@ -196,6 +208,19 @@ class DivisionStatsHuman(BaseStatsHuman):
     def get_aggregation_column(cls):
         return 'division_id'
 
+class LevelStatsHuman(BaseStatsHuman):
+    __tablename__ = 'level_stats_human'
+    level_id = db.Column(db.Integer, db.ForeignKey('levels.id'), nullable=False)
+    aggregation_id = synonym('level_id')
+
+    @declared_attr
+    def aggregation_type(cls):
+        return 'level'
+
+    @classmethod
+    def get_aggregation_column(cls):
+        return 'level_id'
+
 class OrgStatsSkater(BaseStatsSkater):
     __tablename__ = 'org_stats_skater'
     org_id = db.Column(db.Integer, db.ForeignKey('organizations.id'), nullable=False)
@@ -262,6 +287,19 @@ class DivisionStatsGoalie(BaseStatsGoalie):
     def get_aggregation_column(cls):
         return 'division_id'
 
+class LevelStatsGoalie(BaseStatsGoalie):
+    __tablename__ = 'level_stats_goalie'
+    level_id = db.Column(db.Integer, db.ForeignKey('levels.id'), nullable=False)
+    aggregation_id = synonym('level_id')
+
+    @declared_attr
+    def aggregation_type(cls):
+        return 'level'
+
+    @classmethod
+    def get_aggregation_column(cls):
+        return 'level_id'
+
 
 class OrgStatsReferee(BaseStatsReferee):
     __tablename__ = 'org_stats_referee'
@@ -289,6 +327,19 @@ class DivisionStatsReferee(BaseStatsReferee):
     def get_aggregation_column(cls):
         return 'division_id'
 
+class LevelStatsReferee(BaseStatsReferee):
+    __tablename__ = 'level_stats_referee'
+    level_id = db.Column(db.Integer, db.ForeignKey('levels.id'), nullable=False)
+    aggregation_id = synonym('level_id')
+
+    @declared_attr
+    def aggregation_type(cls):
+        return 'level'
+
+    @classmethod
+    def get_aggregation_column(cls):
+        return 'level_id'
+
 
 class OrgStatsScorekeeper(BaseStatsScorekeeper):
     __tablename__ = 'org_stats_scorekeeper'

hockey_blast_common_lib/utils.py

@@ -1,5 +1,6 @@
 import sys
 import os
+from datetime import datetime, timedelta
 
 # Add the package directory to the Python path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -11,9 +12,9 @@ from sqlalchemy.sql import func
 def get_org_id_from_alias(session, org_alias):
     # Predefined organizations
     predefined_organizations = [
-        {"id": 1, "organization_name": "Sharks Ice", "alias": "sharksice"},
-        {"id": 2, "organization_name": "TriValley Ice", "alias": "tvice"},
-        {"id": 3, "organization_name": "CAHA", "alias": "caha"}
+        {"id": 1, "organization_name": "Sharks Ice", "alias": "sharksice", "website": "https://www.sharksice.com"},
+        {"id": 2, "organization_name": "TriValley Ice", "alias": "tvice", "website": "https://www.trivalleyice.com"},
+        {"id": 3, "organization_name": "CAHA", "alias": "caha", "website": "https://www.caha.com"}
     ]
 
     # Check if the organization exists
@@ -25,7 +26,7 @@ def get_org_id_from_alias(session, org_alias):
         for org in predefined_organizations:
             existing_org = session.query(Organization).filter_by(id=org["id"]).first()
             if not existing_org:
-                new_org = Organization(id=org["id"], organization_name=org["organization_name"], alias=org["alias"])
+                new_org = Organization(id=org["id"], organization_name=org["organization_name"], alias=org["alias"], website=org["website"])
                 session.add(new_org)
         session.commit()
 
@@ -82,6 +83,21 @@ def get_fake_human_for_stats(session):
 
     return human.id
 
+def get_start_datetime(last_game_datetime_str, aggregation_window):
+    if last_game_datetime_str:
+        last_game_datetime = datetime.strptime(last_game_datetime_str, '%Y-%m-%d %H:%M:%S')
+        if aggregation_window == 'Daily':
+            # From 10AM till midnight, 14 hours to avoid last day games
+            return last_game_datetime - timedelta(hours=14)
+        elif aggregation_window == 'Weekly':
+            return last_game_datetime - timedelta(weeks=1)
+    return None
+
+def assign_ranks(stats_dict, field, reverse_rank=False):
+    sorted_stats = sorted(stats_dict.items(), key=lambda x: x[1][field], reverse=not reverse_rank)
+    for rank, (key, stat) in enumerate(sorted_stats, start=1):
+        stats_dict[key][f'{field}_rank'] = rank
+
 #TEST DB CONNECTION, PERMISSIONS...
 # from hockey_blast_common_lib.db_connection import create_session
 # session = create_session("frontend")

{hockey_blast_common_lib-0.1.32.dist-info → hockey_blast_common_lib-0.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hockey-blast-common-lib
-Version: 0.1.32
+Version: 0.1.34
 Summary: Common library for shared functionality and DB models
 Author: Pavel Kletskov
 Author-email: kletskov@gmail.com

hockey_blast_common_lib-0.1.34.dist-info/RECORD

@@ -0,0 +1,23 @@
+hockey_blast_common_lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hockey_blast_common_lib/aggregate_all_stats.py,sha256=MUjT23mdOMfCTx-kRSY-LGrLHZ9HNlR6OMqv5KLdzR4,1056
+hockey_blast_common_lib/aggregate_goalie_stats.py,sha256=xDjCYaGLXh5UgkhoUvWKcsnv__QmbCzqLvYNHREFyFY,11937
+hockey_blast_common_lib/aggregate_human_stats.py,sha256=di3h_nQjgMaNnGDPW6LQavnrYOxnZDMohNZEh3ixYNM,23700
+hockey_blast_common_lib/aggregate_referee_stats.py,sha256=ATKBId0Hhzy0FVGmAFDTMgRftYNBDOOA9Z2yPhFA06U,11546
+hockey_blast_common_lib/aggregate_skater_stats.py,sha256=LY6temy5zzqJ85ENymt9nEfpKHqL8Isk_kozfWqHNFI,15803
+hockey_blast_common_lib/assign_skater_skill.py,sha256=p-0fbodGpM8BCjKHDpxNb7BH2FcIlBsJwON844KNrUY,1817
+hockey_blast_common_lib/db_connection.py,sha256=HvPxDvOj7j5H85RfslGvHVNevfg7mKCd0syJ6NX21mU,1890
+hockey_blast_common_lib/dump_sample_db.sh,sha256=MHPA-Ciod7wsvAlMbRtXFiyajgnEqU1xR59sJQ9UWR0,738
+hockey_blast_common_lib/hockey_blast_sample_backup.sql.gz,sha256=9xsTyRZZa0eVIvEwa1IdsogVX83lidXQhTm2nOGUU3M,1033689
+hockey_blast_common_lib/models.py,sha256=Gh0WgYdZE6ClF_48t7zk3G0CItf02yHVzi43afshAgQ,16064
+hockey_blast_common_lib/options.py,sha256=6na8fo-5A2RBPpd_h-7dsqetOLSLoNEJg1QMYgl4jNs,792
+hockey_blast_common_lib/restore_sample_db.sh,sha256=u2zKazC6vNMULkpYzI64nlneCWaGUtDHPBAU-gWgRbw,1861
+hockey_blast_common_lib/skills_in_divisions.py,sha256=RR-x-D7V_lQX--2a2GHEYHtATtIOj2ACpvcEUDzVgkY,7487
+hockey_blast_common_lib/skills_propagation.py,sha256=x6yy7fJ6IX3YiHqiP_v7-p_S2Expb8JJ-mWuajEFBdY,16388
+hockey_blast_common_lib/stats_models.py,sha256=qvkt-XRFb4ZW7yBj7vltedzUS_YwWagm_efMRcsioSA,25120
+hockey_blast_common_lib/stats_utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hockey_blast_common_lib/utils.py,sha256=EDMBiWpF2bNCC1xc1WUNahQlUsG8bv1Wh0Eal3Iw2ls,4689
+hockey_blast_common_lib/wsgi.py,sha256=7LGUzioigviJp-EUhSEaQcd4jBae0mxbkyBscQfZhlc,730
+hockey_blast_common_lib-0.1.34.dist-info/METADATA,sha256=ROphG8zkwmNGOdHFyrafZ2cJAUBEuvr_-LZ6rBxMPEY,318
+hockey_blast_common_lib-0.1.34.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+hockey_blast_common_lib-0.1.34.dist-info/top_level.txt,sha256=wIR4LIkE40npoA2QlOdfCYlgFeGbsHR8Z6r0h46Vtgc,24
+hockey_blast_common_lib-0.1.34.dist-info/RECORD,,

hockey_blast_common_lib-0.1.32.dist-info/RECORD

@@ -1,21 +0,0 @@
-hockey_blast_common_lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hockey_blast_common_lib/aggregate_goalie_stats.py,sha256=d2qav46Rg2DNIYRj_Ubj1kpQmoPUJHKiwEWOVU25nD4,8742
-hockey_blast_common_lib/aggregate_human_stats.py,sha256=88OMhTgQjzc9xIakf6kW9_lZwbSXkpsZy8C0pX-Wlq8,14229
-hockey_blast_common_lib/aggregate_referee_stats.py,sha256=A0PTyEbPUjqfXxlJCDOVioFaQk9AyjjhiWEuRuu35v0,11036
-hockey_blast_common_lib/aggregate_skater_stats.py,sha256=jkBD5u-gJc1DTDIEuxM_qymKsrWtLagFKeEn__2rFgU,16009
-hockey_blast_common_lib/assign_skater_skill.py,sha256=p-0fbodGpM8BCjKHDpxNb7BH2FcIlBsJwON844KNrUY,1817
-hockey_blast_common_lib/db_connection.py,sha256=HvPxDvOj7j5H85RfslGvHVNevfg7mKCd0syJ6NX21mU,1890
-hockey_blast_common_lib/dump_sample_db.sh,sha256=MHPA-Ciod7wsvAlMbRtXFiyajgnEqU1xR59sJQ9UWR0,738
-hockey_blast_common_lib/hockey_blast_sample_backup.sql.gz,sha256=fO5SsdPB6XYptPLx1rD2VVSTySLnJmyubeSS0A0HGyw,1033692
-hockey_blast_common_lib/models.py,sha256=ebRnnvDOVNDfqAp8CA8u7uk3LCOfI3iUwOpHgzoBy0U,15984
-hockey_blast_common_lib/options.py,sha256=6na8fo-5A2RBPpd_h-7dsqetOLSLoNEJg1QMYgl4jNs,792
-hockey_blast_common_lib/restore_sample_db.sh,sha256=u2zKazC6vNMULkpYzI64nlneCWaGUtDHPBAU-gWgRbw,1861
-hockey_blast_common_lib/skills_in_divisions.py,sha256=LV6f2nzmRi1IHl1_EDVf61I7nHUNc0xYuNL4dBsZI_U,7486
-hockey_blast_common_lib/skills_propagation.py,sha256=x6yy7fJ6IX3YiHqiP_v7-p_S2Expb8JJ-mWuajEFBdY,16388
-hockey_blast_common_lib/stats_models.py,sha256=35-6iz1r8MJcmzlyIlJy0uHgWh8oltyf-3H61ocal3o,23048
-hockey_blast_common_lib/utils.py,sha256=odDJWCK0BgbResXeoUzxbVChjaxcXr168ZxbrAw3L_8,3752
-hockey_blast_common_lib/wsgi.py,sha256=7LGUzioigviJp-EUhSEaQcd4jBae0mxbkyBscQfZhlc,730
-hockey_blast_common_lib-0.1.32.dist-info/METADATA,sha256=DZgI2eRmWQGFrcaxenwqrWLZdWmgDK5LbU57iLTlI0w,318
-hockey_blast_common_lib-0.1.32.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-hockey_blast_common_lib-0.1.32.dist-info/top_level.txt,sha256=wIR4LIkE40npoA2QlOdfCYlgFeGbsHR8Z6r0h46Vtgc,24
-hockey_blast_common_lib-0.1.32.dist-info/RECORD,,