flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/SchedulingVisits.py

@@ -0,0 +1,592 @@
+from collections.abc import Callable
+import asyncio
+from datetime import datetime, date, timedelta
+from pathlib import Path
+from typing import Any
+import numpy as np
+import pandas as pd
+import calendar
+import requests
+from sklearn.neighbors import BallTree
+from geopy.distance import geodesic  # For calculating distances
+from ..exceptions import DataNotFound, ComponentError, ConfigError
+from .flow import FlowComponent
+
+
+# OSRM base URL for routing requests
+OSRM_BASE_URL = "http://router.project-osrm.org"
+
+class SchedulingVisits(FlowComponent):
+    """Generating the Schedule of Employee Visits with Market Constraints and Visit Cadence.
+
+    Overview:
+        The SchedulingVisits class is a Flowtask component for generating a schedule of employee visits based on
+        a set of rules and constraints. This component can be used to optimize the order of visits,
+        minimize travel time, and balance workloads across employees.
+        The schedule is generated by solving a combinatorial optimization
+        problem with support for custom objective functions and constraints.
+
+        Example of row consumed:
+        ```
+        associate_oid -> object -> G3Q86F5E1JXN1XVM
+        corporate_email -> object -> buko@trocglobal.com
+        employee_position -> object -> (3442724.8764311927, -10973885.176252203)
+        store_id -> object -> BBY0178
+        store_position -> object -> (3564143.804984759, -10887222.41833608)
+        market -> object -> Market1
+        visit_rule -> int64 -> 2
+        visit_frequency -> object -> Monthly
+        ```
+
+        Example of Row Generated:
+        ```
+
+        ```
+    
+
+        Example:
+
+        ```yaml
+        SchedulingVisits:
+          use_ghost_employees: true
+          ghost_employees: 1
+          in_store_percentage: 0.6
+          in_store_visit: 0.75
+          max_stores: 4
+          max_distance: 120
+          year: 2024
+          month: 12
+          exception_dates:
+          - '2024-12-25'
+          exceptions_filename: /home/ubuntu/symbits/Scheduling-Visits-Exceptions.xlsx
+        ```
+
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        # TODO: add support for Masks
+        # total hours worked per day
+        self.day_duration: float = kwargs.pop('day_duration', 8.0)
+        # 60% of the day in store
+        self.in_store_percentage: float = kwargs.pop('in_store_percentage', 0.6)
+        # near to 45 minutes in store
+        self.in_store_visit: float = kwargs.pop('in_store_visit', 0.75)
+        self.max_stores: int = kwargs.pop('max_stores', 4)
+        # no more than 600 miles covered at day
+        self.max_distance: int = kwargs.pop('max_distance', 600)
+        # Average Speed:
+        self.average_speed: float = kwargs.pop('average_speed', 40)
+        # Objective function: minimize total travel time
+        self.use_ghost_employees: bool = kwargs.pop('use_ghost_employees', False)
+        # Using 3 ghost employees per market if no employees are available.
+        self.ghost_employees: int = kwargs.pop('ghost_employees', 3)
+        # calculate year and month of current day:
+        today = date.today()
+        self._today = today
+        self.year: int = kwargs.pop('year', today.year)
+        self.month: int = kwargs.pop('month', today.month)
+        super(SchedulingVisits, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        # Ghost Account:
+        self._ghost_account = kwargs.pop('ghost_account', 'ghost_{}@company.com')
+        # exception days:
+        self._exception_dates = kwargs.pop('exception_dates', [])
+
+    def get_workdays(self, year: int, month: int, exception_dates: list = None):
+        """Get all workdays (Monday to Friday) in a given month, excluding exception dates."""
+        first_day = date(year, month, 1)
+        last_day = date(year, month, calendar.monthrange(year, month)[1])
+        workdays = pd.bdate_range(first_day, last_day)
+        if exception_dates:
+            # Convert exception_dates to datetime
+            exception_dates = pd.to_datetime(exception_dates)
+            workdays = workdays.difference(exception_dates)
+        return workdays
+
+    def _get_fdom(self, year, month):
+        """Function to get the first Monday of a given month
+        (which is Labor Day in the US)."""
+        cal = calendar.Calendar()
+        first_monday = None
+        for day in cal.itermonthdays2(year, month):
+            if day[0] != 0 and day[1] == 0:  # day[1] == 0 means Monday
+                first_monday = day[0]
+                break
+        return datetime(year, month, first_monday)
+
+    def get_distance(self, coord1, coord2):
+        """Function to calculate distance
+        between two points (latitude, longitude)."""
+        return geodesic(coord1, coord2).miles
+
+    def to_miles(self, distance) -> float:
+        return distance * 0.621371  # Convert to miles
+
+    def to_hours(self, minutes) -> float:
+        return minutes / 60  # Convert to hours
+
+    def get_labor_days(self, year: int = 2024, month: int = 9):
+        """Function to get all workdays (Monday to Friday) in a given month."""
+        # Get first Labor Day (first Monday) of the month
+        labor_day = self._get_fdom(year, month)
+        # Generate list of weekdays (excluding weekends) starting from Labor Day
+        workdays = []
+        current_day = labor_day
+        while current_day.month == month:
+            if current_day.weekday() < 5:  # Only Monday to Friday (weekday < 5)
+                workdays.append(current_day)
+            current_day += timedelta(days=1)
+        return workdays
+
+    def get_travel(self, waypoints, transportation: str = 'driving'):
+        # Build the request URL for OSRM driving route
+        # including all waypoints
+        osrm_url = f"{OSRM_BASE_URL}/route/v1/{transportation}/{waypoints}?overview=false"
+        # Send the request to OSRM API
+        response = requests.get(osrm_url)
+        # Check if the request was successful
+        if response.status_code == 200:
+            route_data = response.json()
+            # Extract total travel duration and distance (in seconds and meters)
+            # Total duration
+            duration_seconds = route_data['routes'][0]['duration']
+            # Total distance
+            distance_meters = route_data['routes'][0]['distance']
+            # Convert to more readable formats
+            duration_minutes = duration_seconds / 60
+            distance_km = distance_meters / 1000
+            return duration_minutes, distance_km
+        else:
+            return 0, 0
+
+    def get_travel_duration(self, origin, destination):
+        """Helper function to get distance and duration between two points.
+        """
+        waypoints = f"{origin[1]},{origin[0]};{destination[1]},{destination[0]}"
+        duration_minutes, distance_km = self.get_travel(waypoints, transportation='driving')
+        distance_miles = self.to_miles(distance_km)
+        return distance_miles, duration_minutes
+
+    def get_scheduled_dates(
+        self,
+        cadence: str,
+        visit_rule: int,
+        visit_frequency: Any,
+        workdays: pd.DatetimeIndex,
+        store_index: int
+    ):
+        """Given the visit_rule and visit_frequency, return a list of scheduled dates for the visits."""
+        scheduled_dates = []
+        # Set visit_frequency and visit_rule based on cadence if provided
+        if cadence:
+            cadence = cadence.lower()
+            if 'xweek' in cadence:
+                num = int(cadence[0])
+                visit_rule = num
+                visit_frequency = 'weekly'
+            elif 'xmonth' in cadence:
+                num = int(cadence[0])
+                visit_rule = num
+                visit_frequency = 'monthly'
+            elif 'xqtr' in cadence:
+                num = int(cadence[0])
+                visit_rule = num
+                visit_frequency = 'quarterly'
+
+        if visit_frequency.lower() == 'quarterly':
+            # For simplicity, schedule as monthly with 1 visit per month
+            visit_frequency = 'monthly'
+            visit_rule = 1
+
+        if visit_frequency.lower() == 'weekly':
+            # Get the weeks in the month
+            workdays_df = pd.DataFrame({'date': workdays})
+            workdays_df['week'] = workdays_df['date'].dt.isocalendar().week
+            weeks = workdays_df['week'].unique()
+            for week in weeks:
+                week_days = workdays_df[workdays_df['week'] == week]['date'].reset_index(drop=True)
+                num_days = len(week_days)
+                # If visit_rule > num_days in week, limit to num_days
+                num_visits = min(visit_rule, num_days)
+                for i in range(num_visits):
+                    day_index = (store_index + i) % num_days
+                    scheduled_date = week_days[day_index]
+                    scheduled_dates.append(scheduled_date)
+        elif visit_frequency.lower() == 'monthly':
+            # Visit 'visit_rule' times per month
+            total_days = len(workdays)
+            if visit_rule == 0:
+                visit_rule = 1
+            interval = total_days // visit_rule
+            for i in range(visit_rule):
+                day_index = i * interval
+                if day_index >= total_days:
+                    day_index = total_days - 1
+                scheduled_dates.append(workdays[day_index])
+        else:
+            # Default to monthly
+            total_days = len(workdays)
+            interval = total_days // visit_rule
+            for i in range(visit_rule):
+                day_index = i * interval
+                if day_index >= total_days:
+                    day_index = total_days - 1
+                scheduled_dates.append(workdays[day_index])
+        return scheduled_dates
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data: pd.DataFrame = self.input
+            if not isinstance(self.data, pd.DataFrame):
+                raise ConfigError(
+                    "Incompatible Pandas Dataframe", status=404
+                )
+        else:
+            raise DataNotFound(
+                "Data Not Found", status=404
+            )
+        await super().start(**kwargs)
+        # if dataframe doesn't have a store_position attribute
+        if 'store_position' not in self.data.columns:
+            # Create the store_position column
+            self.data['store_position'] = self.data.apply(
+                lambda row: (row['latitude'], row['longitude']),
+                axis=1
+            )
+        # Exceptions Filename:
+        self._exceptions_file = None
+        if hasattr(self, 'exceptions_filename'):
+            self._exceptions_file = Path(self.exceptions_filename).resolve()
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        self._logger.debug('=== RUNNING FUNCTION SCHEDULING VISITS ===')
+
+        # Get workdays
+        workdays = self.get_workdays(self.year, self.month, self._exception_dates)
+
+        # Initialize a dictionary to keep track of assignments and exceptions
+        schedule_rows = []
+        exception_rows = []
+
+        if self.use_ghost_employees or 'associate_oid' not in self.data.columns:
+            # Create multiple ghost employees per market
+            markets = self.data['market'].unique()
+            ghost_employees = {}
+            self.data['associate_oid'] = None  # Initialize associate_oid column
+            for market in markets:
+                market_data = self.data[self.data['market'] == market]
+                positions = np.array([pos for pos in market_data['store_position']])
+                mean_position = positions.mean(axis=0)
+                # Create self.ghost_employees ghost employees per market
+                ghost_employee_ids = [f'{market}_ghost_{i+1}' for i in range(self.ghost_employees)]
+                # Generate unique emails for ghost employees
+                ghost_employee_emails = [self._ghost_account.format(i + 1) for i in range(self.ghost_employees)]
+                # Generate positions with small variations
+                ghost_employee_positions = []
+                for i in range(self.ghost_employees):
+                    # Generate small random offsets in degrees (~50 meters variation)
+                    # 1 degree latitude ~ 111 km, so 50 meters ~ 0.00045 degrees
+                    lat_offset = np.random.uniform(-0.00045, 0.00045)
+                    lon_offset = np.random.uniform(-0.00045, 0.00045)
+                    ghost_position = (mean_position[0] + lat_offset, mean_position[1] + lon_offset)
+                    ghost_employee_positions.append(ghost_position)
+
+                # Assign stores to ghost employees in a round-robin fashion
+                market_store_indices = market_data.index
+                num_stores = len(market_store_indices)
+                for idx, store_idx in enumerate(market_store_indices):
+                    assigned_employee_index = idx % self.ghost_employees
+                    assigned_employee_id = ghost_employee_ids[assigned_employee_index]
+                    self.data.at[store_idx, 'associate_oid'] = assigned_employee_id
+
+                # Store the email and position for each ghost employee
+                for i, assigned_employee_id in enumerate(ghost_employee_ids):
+                    ghost_employees[assigned_employee_id] = {
+                        'position': ghost_employee_positions[i],
+                        'email': ghost_employee_emails[i],
+                        'market': market
+                    }
+
+            # After assigning stores to ghost employees
+            store_assignments = self.data.groupby('store_id')['associate_oid'].nunique()
+            overlapping_stores = store_assignments[store_assignments > 1]
+            if not overlapping_stores.empty:
+                print("Stores assigned to multiple employees:")
+                print(overlapping_stores)
+            else:
+                print("All stores uniquely assigned.")
+
+            # Now group by associate_oid
+            employee_groups = self.data.groupby('associate_oid')
+
+        # Check if employee information is available
+        elif 'associate_oid' in self.data.columns:
+            employee_groups = self.data.groupby('associate_oid')
+            # Group the data by employee
+            ghost_employees = {}  # Not needed but kept for consistency
+        else:
+            raise ComponentError("No employee information available.")
+
+        # Prepare a list to collect scheduled visits
+        scheduled_visits = []
+
+        # Iterate over employees
+        for employee_id, employee_data in employee_groups:
+            # Get employee information
+            employee_info = employee_data.iloc[0]
+            if 'corporate_email' in employee_info:
+                employee_email = employee_info['corporate_email']
+            else:
+                # Get from ghost_employees
+                employee_email = ghost_employees[employee_id]['email']
+            # Get employee position
+            if 'employee_position' in employee_info:
+                employee_position = employee_info['employee_position']
+            else:
+                # Get from ghost_employees
+                employee_position = ghost_employees[employee_id]['position']
+
+            # Get unique stores for the employee
+            stores = employee_data.drop_duplicates('store_id').reset_index(drop=True)
+
+            self._logger.notice(
+                f"Generating schedule for: {employee_email} ({employee_position}) for {len(stores)} stores."
+            )
+
+            # For each store, generate scheduled visits
+            for idx, store_row in stores.iterrows():
+                store_id = store_row['store_id']
+                store_position = store_row['store_position']
+                store_name = store_row.get('store_name', 'Unknown')
+                try:
+                    visit_rule = store_row['visit_rule']
+                    visit_frequency = store_row['visit_frequency']
+                except KeyError:
+                    visit_rule = 1
+                    visit_frequency = 'weekly'
+                visit_market = store_row['market']
+                try:
+                    cadence = store_row['cadence']
+                except KeyError:
+                    cadence = None
+
+                # Generate scheduled dates for this store
+                # Use store_id hash as the unique identifier
+                store_unique_id = hash(store_row['store_id']) % (10 ** 8)
+                scheduled_dates = self.get_scheduled_dates(
+                    cadence,
+                    visit_rule,
+                    visit_frequency,
+                    workdays,
+                    store_index=store_unique_id
+                )
+
+                for scheduled_date in scheduled_dates:
+                    scheduled_visit = {
+                        'associate_oid': employee_id,
+                        'corporate_email': employee_email,
+                        'employee_position': employee_position,
+                        'store_id': store_id,
+                        'store_name': store_name,
+                        'market': visit_market,
+                        'store_position': store_position,
+                        'scheduled_date': scheduled_date,
+                        'visit_rule': visit_rule,
+                        'visit_frequency': visit_frequency
+                    }
+                    scheduled_visits.append(scheduled_visit)
+
+        # Convert scheduled visits to DataFrame
+        scheduled_visits_df = pd.DataFrame(scheduled_visits)
+
+        # Ensure that 'scheduled_date' is of datetime type
+        scheduled_visits_df['scheduled_date'] = pd.to_datetime(scheduled_visits_df['scheduled_date'])
+        scheduled_visits_df['week'] = scheduled_visits_df['scheduled_date'].dt.isocalendar().week
+        duplicates = scheduled_visits_df.duplicated(subset=['store_id', 'week'], keep=False)
+        duplicate_visits = scheduled_visits_df[duplicates]
+        if not duplicate_visits.empty:
+            print("Duplicate visits found for the same store in the same week:")
+            print(duplicate_visits)
+        else:
+            print("No duplicate visits found.")
+
+        # Group the scheduled visits by associate_oid
+        employee_scheduled_visits = scheduled_visits_df.groupby('associate_oid')
+
+        for employee_id, employee_visits in employee_scheduled_visits:
+            # Get employee information
+            employee_info = employee_visits.iloc[0]
+            employee_email = employee_info['corporate_email']
+            employee_position = employee_info['employee_position']
+            market = employee_info['market']
+
+            # Group visits by week
+            employee_visits['week'] = employee_visits['scheduled_date'].dt.isocalendar().week
+            visits_by_week = employee_visits.groupby('week')
+
+            # Group workdays by week
+            workdays_series = pd.Series(workdays)
+            workdays_series.index = workdays_series.dt.isocalendar().week
+            workdays_by_week = workdays_series.groupby(level=0)
+
+            for week_number, week_visits in visits_by_week:
+                # Get the workdays for this week
+                if week_number in workdays_by_week.groups:
+                    week_workdays = workdays_by_week.get_group(week_number).values
+                else:
+                    continue  # No workdays in this week
+
+                # Initialize day schedules for each day in the week
+                day_schedules = {}
+                for day in week_workdays:
+                    day_schedules[day] = {
+                        'associate_oid': employee_id,
+                        'corporate_email': employee_email,
+                        'start_position': employee_position,
+                        'market': market,
+                        'day': day,
+                        'month': self.month,
+                        'year': self.year,
+                        'total_time_minutes': 0,
+                        'total_time_hours': 0,
+                        'total_in_store_time': 0,
+                        'total_travel_time': 0,
+                        'total_distance': 0,
+                        'stores_visited_count': 0,
+                        'visited_stores': {},
+                        'store_ids': []
+                    }
+
+                unvisited_stores = week_visits.copy().reset_index(drop=True)
+                unvisited_store_reasons = {}
+
+                for idx, store_row in unvisited_stores.iterrows():
+                    store_id = store_row['store_id']
+                    scheduled_date = store_row['scheduled_date']
+                    scheduled_dates_to_try = [day for day in [scheduled_date] + list(week_workdays) if day in day_schedules]
+                    store_scheduled = False
+                    reasons_for_store = []
+
+                    for day in scheduled_dates_to_try:
+
+                        if day not in day_schedules:
+                            reasons_for_store.append(
+                                f'Day {day} is not in day_schedules'
+                            )
+                            continue  # Skip days not in day_schedules
+
+                        day_schedule = day_schedules[day]
+                        # Check if day has capacity
+                        if day_schedule['stores_visited_count'] >= self.max_stores:
+                            reasons_for_store.append(
+                                f'Day {day}: Max stores reached'
+                            )
+                            continue  # Day is full
+
+                        # Set current position to the last store visited on the day, or employee_position
+                        if day_schedule['stores_visited_count'] > 0:
+                            last_store = list(day_schedule['visited_stores'].values())[-1]
+                            current_position = (last_store['latitude'], last_store['longitude'])
+                        else:
+                            current_position = employee_position
+
+                        # Calculate distance and time to this store
+                        distance_miles = self.get_distance(current_position, store_row['store_position'])
+                        travel_time = (distance_miles / self.average_speed) * 60  # in minutes
+                        time_in_store = self.in_store_visit * 60  # in minutes
+                        potential_total_time = day_schedule['total_time_minutes'] + travel_time + time_in_store
+
+                        # Check constraints
+                        if potential_total_time > self.day_duration * 60:
+                            reasons_for_store.append(
+                                f'Day {day}: Exceeds day duration'
+                            )
+                            continue  # Cannot schedule on this day due to time constraint
+                        if day_schedule['total_distance'] + distance_miles > self.max_distance:
+                            reasons_for_store.append(
+                                f'Day {day} with distance {distance_miles} in greater than {self.max_distance}'
+                            )
+                            continue  # Cannot schedule on this day due to distance constraint
+
+                        # Schedule the store
+                        day_schedule['total_time_minutes'] = potential_total_time
+                        day_schedule['total_distance'] += distance_miles
+                        day_schedule['total_in_store_time'] += time_in_store
+                        day_schedule['total_travel_time'] += travel_time
+                        day_schedule['stores_visited_count'] += 1
+                        day_schedule['visited_stores'][store_id] = {
+                            'store_id': store_id,
+                            'store_name': store_row.get('store_name', 'Unknown'),
+                            'latitude': store_row['store_position'][0],
+                            'longitude': store_row['store_position'][1],
+                            'visit_rule': store_row.get('visit_rule', None),
+                            'visit_frequency': store_row.get('visit_frequency', None),
+                            'market': store_row['market']
+                        }
+                        day_schedule['store_ids'].append(store_id)
+                        store_scheduled = True
+                        break  # Break the loop over days, since store is scheduled
+
+                    if not store_scheduled:
+                        # Record the reasons
+                        unvisited_store_reasons[store_id] = reasons_for_store
+                        # Add to exception_rows
+                        reason = '; '.join(reasons_for_store) if reasons_for_store else 'Could not schedule on any day'
+                        exception_row = {
+                            'associate_oid': employee_id,
+                            'corporate_email': employee_email,
+                            'market': store_row['market'],
+                            'year': self.year,
+                            'month': self.month,
+                            'store_id': store_id,
+                            'store_name': store_row.get('store_name', 'Unknown'),
+                            'store_position': store_row['store_position'],
+                            'scheduled_date': store_row['scheduled_date'],
+                            'reason': reason
+                        }
+                        exception_rows.append(exception_row)
+
+                # After attempting to schedule all stores
+                # Add day_schedules to schedule_rows
+                for day_schedule in day_schedules.values():
+                    if day_schedule['stores_visited_count'] > 0:
+                        day_schedule['total_time_hours'] = day_schedule['total_time_minutes'] / 60
+                        schedule_rows.append(day_schedule)
+        # ============
+        # Save the schedule and exceptions
+        schedule_df = pd.DataFrame(schedule_rows)
+        exception_stores_df = pd.DataFrame(exception_rows)
+
+        # Set the final results
+        self.schedule_df = schedule_df
+        self.exception_stores_df = exception_stores_df
+
+        print(' === Visit Schedule === ')
+        print(schedule_df.head())
+        print('=== Exception Stores ====')
+        print(exception_stores_df.head())
+        # Saving the Exception Stores Dataframe to filesystem:
+        if self._exceptions_file:
+            if self._exceptions_file.suffix == '.xlsx':
+                exception_stores_df.to_excel(self._exceptions_file, index=False)
+            else:
+                exception_stores_df.to_csv(self._exceptions_file, index=False)
+
+        self._result = schedule_df
+
+        self._print_data_(self._result, 'Schedule')
+
+        return self._result