energy-reports-pkg 0.6.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- energy_reports_pkg-0.6.14.dist-info/METADATA +10 -0
- energy_reports_pkg-0.6.14.dist-info/RECORD +221 -0
- energy_reports_pkg-0.6.14.dist-info/WHEEL +5 -0
- energy_reports_pkg-0.6.14.dist-info/top_level.txt +1 -0
- mindsett_energy_reports_pkg/config/__init__.py +6 -0
- mindsett_energy_reports_pkg/config/email/__init__.py +37 -0
- mindsett_energy_reports_pkg/config/monday/__init__.py +8 -0
- mindsett_energy_reports_pkg/config/monday/auth/__init__.py +7 -0
- mindsett_energy_reports_pkg/config/monday/columns_concerned/__init__.py +22 -0
- mindsett_energy_reports_pkg/config/postgresdb/__init__.py +33 -0
- mindsett_energy_reports_pkg/config/postgresdb/table/__init__.py +2 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_building/__init__.py +1 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_building_total/__init__.py +1 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_cache/__init__.py +30 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_iot/__init__.py +1 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_mailing/__init__.py +15 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_meta/__init__.py +2 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_occupancy/__init__.py +3 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_on_off/__init__.py +2 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_org/__init__.py +1 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_price/__init__.py +2 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_schedule/__init__.py +1 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_sqm/__init__.py +2 -0
- mindsett_energy_reports_pkg/config/postgresdb/table_timezone/__init__.py +2 -0
- mindsett_energy_reports_pkg/config/sharepoint/__init__.py +12 -0
- mindsett_energy_reports_pkg/config/sharepoint/excel/__init__.py +5 -0
- mindsett_energy_reports_pkg/config/sharepoint/excel/columns_concerned/__init__.py +7 -0
- mindsett_energy_reports_pkg/config/smtp/__init__.py +10 -0
- mindsett_energy_reports_pkg/jobs/databricks_job.py +31 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/DataProcessor/__init__.py +14 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/DataProcessor/launchInteractiveProcess.py +6 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorBoardColumnNotMatch/__init__.py +6 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorBoardNotFound/__init__.py +5 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorException/__init__.py +4 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorNoColumInBoard/__init__.py +5 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorNoGroupsInBoard/__init__.py +5 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorNoItemInGroup/__init__.py +6 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/MondayProcessorPageNotSupported/__init__.py +7 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/MondayProcessorExceptions_/__init__.py +8 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/__init__.py +40 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchBoardById.py +13 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchBoardLogsById.py +33 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchColumnValueByGroup.py +103 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchGroups.py +19 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchPrismConf.py +10 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchPrismConfByBoard.py +83 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_fetchPrismConfByGroup.py +89 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_searchColumnIdsByName.py +23 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_searchConfColumnIds.py +30 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/_searchConfColumnIdsMatch.py +20 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/__init__.py +11 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/boards/__init__.py +12 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/boards/groups/__init__.py +11 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/boards/groups/items_page/__init__.py +10 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/boards/groups/items_page/items/__init__.py +12 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/boards/groups/items_page/items/column_values/__init__.py +10 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/dev/graphql_query/boards/groups/items_page/items/column_values/column/__init__.py +2 -0
- mindsett_energy_reports_pkg/libs/MondayProcessor/launchInteractiveProcess.py +7 -0
- mindsett_energy_reports_pkg/libs/__init__.py +4 -0
- mindsett_energy_reports_pkg/libs/cache_return/__init__.py +47 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/__init__.py +29 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/load_cf_from_db/__init__.py +17 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/__init__.py +52 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/cache_cf_to_db/__init__.py +11 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/cache_cf_to_db/append_df_to_table/__init__.py +29 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/cache_cf_to_db/empty_table/__init__.py +11 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/extract_contacts/__init__.py +57 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/extract_contacts/get_mailing_list/__init__.py +25 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/extract_contacts/load_mailing_list/__init__.py +50 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/extract_contacts/load_mailing_list/columns_concerned/__init__.py +22 -0
- mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/get_last_update_time/__init__.py +26 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/__init__.py +27 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/load_cf_from_db/__init__.py +17 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/__init__.py +28 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/cache_cf_to_db/__init__.py +11 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/cache_cf_to_db/append_df_to_table/__init__.py +29 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/cache_cf_to_db/empty_table/__init__.py +11 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/__init__.py +13 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/format_contacts/__init__.py +57 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/format_contacts/columns_concerned/__init__.py +8 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/format_contacts/extract_id/# __init__.py +11 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/format_contacts/extract_id/__init__.py +19 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/format_contacts/get_mailing_list/__init__.py +10 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/format_contacts/uuid_for_nona/__init__.py +12 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/load_mailing_list/__init__.py +30 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/load_mailing_list/get_excel_file_from_link/__init__.py +30 -0
- mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/load_mailing_list/load_excel_as_df/__init__.py +45 -0
- mindsett_energy_reports_pkg/libs/generate_dag_id/__init__.py +20 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/__init__.py +42 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/__version__.py +8 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_airflow/__init__.py +47 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/__init__.py +25 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/get_credential_url/__init__.py +17 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/get_credential_url/databricks/__init__.py +10 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/get_credential_url/local/__init__.py +13 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_synapse/__init__.py +15 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_synapse/connect_lake_and_fetch_json/__init__.py +19 -0
- mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_synapse/datalake/__init__.py +7 -0
- mindsett_energy_reports_pkg/mdl/__init__.py +2 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/__init__.py +119 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_end_time/__init__.py +8 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_period_range/__init__.py +8 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_schedule_records/__init__.py +34 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_start_time/__init__.py +6 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_start_time_co2_barchart/__init__.py +7 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/__init__.py +93 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/email_report/__init__.py +47 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/__init__.py +75 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/apply_price_from_db/__init__.py +13 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/apply_price_from_db/load_energy_price/__init__.py +14 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/attach_id_to_buildings/__init__.py +24 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/attach_id_to_buildings/load_building/__init__.py +19 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/attach_id_to_buildings/load_org/__init__.py +13 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/check_and_keep_valid/__init__.py +36 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/load_sqm/__init__.py +11 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/load_config/load_timezone/__init__.py +11 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/__init__.py +43 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/__init__.py +47 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/__init__.py +40 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/__init__.py +71 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/plot_temperature_graph/__init__.py +35 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/plot_the_legend/__init__.py +26 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/setup_x_tick_lim/__init__.py +26 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/setup_x_tick_lim/generate_day_code/__init__.py +18 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/setup_x_tick_lim/generate_day_code/generate_day_code_list/__init__.py +38 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/setup_y_label_and_get_pos_bot_range/__init__.py +40 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/setup_y_label_and_get_pos_bot_range/get_pos_bot_and_ylim/__init__.py +30 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/plot_shifted_daily_barchart_with_temp/setup_y_tick_lim_label_zero_line/__init__.py +23 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/preprocessing_for_barchart/__init__.py +21 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/preprocessing_for_barchart/groupby_date_and_ooh_current_period/__init__.py +39 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/preprocessing_for_barchart/patch_missing_columns_for_ooh_and_sign/__init__.py +18 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_daily_barchart_with_temp/preprocessing_for_barchart/patch_missing_dates/__init__.py +28 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_energy_meter_with_benchmarking/__init__.py +42 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_energy_meter_with_benchmarking/energy_meter_with_benchmarking/__init__.py +66 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_energy_meter_with_benchmarking/energy_meter_with_benchmarking/benchmarking_design/__init__.py +194 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_energy_meter_with_benchmarking/energy_meter_with_benchmarking/benchmarking_design/ruler_anno_shink_cf/__init__.py +24 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_energy_meter_with_benchmarking/energy_meter_with_benchmarking/energy_meter_design/__init__.py +233 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_energy_meter_with_benchmarking/preprocessing_for_energy_meter_with_benchmarking/__init__.py +23 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_insight_statements/__init__.py +42 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_insight_statements/insight_statements/__init__.py +41 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_insight_statements/insight_statements/statement_for_avg_action_time/__init__.py +61 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_insight_statements/insight_statements/statement_for_biggest_ooh/__init__.py +31 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_insight_statements/insight_statements/statement_for_total_ooh/__init__.py +18 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_insight_statements/preprocessing_for_statement/__init__.py +85 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/__init__.py +29 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/plot_shifted_period_barchart/__init__.py +49 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/plot_shifted_period_barchart/set_x_label_prepare_xtick/__init__.py +24 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/plot_shifted_period_barchart/set_x_tick_and_lim/__init__.py +26 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/plot_shifted_period_barchart/setup_y_tick_lim_label_zero_line/__init__.py +31 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/preprocessing_for_period_barchart/__init__.py +43 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_period_barchart/preprocessing_for_period_barchart/patch_missing_ooh_periods_sign/__init__.py +23 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/__init__.py +56 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/__init__.py +109 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/colors/__init__.py +28 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/hatches/__init__.py +2 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_centre_pct/__init__.py +28 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_phase_balance/__init__.py +42 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_phase_balance/get_phase_markers/__init__.py +28 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_phase_balance/get_tick_values/__init__.py +44 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_phase_balance/plot_phase_balance_part/__init__.py +108 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_phase_balance/plot_phase_balance_part/ruler_anno_shink_cf/__init__.py +29 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_phase_balance/plot_phase_balance_part/ruler_symbol_cf/__init__.py +35 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_ring/__init__.py +87 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_ring/adjust_gap_for_visibility/__init__.py +19 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_ring/adjust_to_inner_ring/__init__.py +32 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_ring/adjust_values_for_better_visibility/__init__.py +27 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_ring/annotate_pct_on_pies/__init__.py +21 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/plot_ring/generate_pct_fmt/__init__.py +8 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/__init__.py +36 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/generate_legend_labels/__init__.py +50 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/generate_legend_labels/get_formatted_arrow/__init__.py +18 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/generate_legend_labels/get_formatted_name/__init__.py +14 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/generate_legend_labels/get_formatted_pct/__init__.py +44 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/generate_legend_labels/get_formatted_sum/__init__.py +31 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/plot_adaptive_ring_with_phase/show_legend/get_legend_handles/__init__.py +22 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_phases/__init__.py +37 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/__init__.py +26 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/group_and_pivot/__init__.py +24 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/transform_asset_stats_for_adaptive_piechart/__init__.py +30 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/transform_asset_stats_for_adaptive_piechart/get_change_pct_and_sort_and_trim_cols/__init__.py +34 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/transform_asset_stats_for_adaptive_piechart/get_entries_to_display/__init__.py +34 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/transform_asset_stats_for_adaptive_piechart/get_rank_overall_protect/__init__.py +14 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/transform_asset_stats_for_adaptive_piechart/get_subgroup_sum_and_rank/__init__.py +22 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_components/generate_piechart_with_phase/preprocessing_for_piechart/transform_asset_stats_for_adaptive_piechart/get_sum_for_curr_and_prev/__init__.py +20 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_report/PDF/__init__.py +44 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_report/PDF/write_multicell_with_styles.py +30 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/generate_report/__init__.py +162 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/__init__.py +78 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/__init__.py +32 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/import_data_with_meta_postgres/__init__.py +47 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/import_data_with_meta_postgres/load_iot_data/__init__.py +28 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/import_data_with_meta_postgres/load_iot_data/load_iot_data_partition/__init__.py +24 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/import_data_with_meta_postgres/load_metadata/__init__.py +34 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/import_data_with_meta_postgres/load_metadata/get_meta_statement/__init__.py +32 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/load_source_data/query_building_total/__init__.py +40 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/refine_group_name/__init__.py +20 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/refine_group_name/exchange_group_name_column/__init__.py +10 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/prepare_report/preparing_source_data/validate_data_not_empty/__init__.py +9 -0
- mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/select_sites_and_emails/__init__.py +19 -0
- mindsett_energy_reports_pkg/mdl/utils_/__init__.py +17 -0
- mindsett_energy_reports_pkg/mdl/utils_/cache_df_to_db/__init__.py +11 -0
- mindsett_energy_reports_pkg/mdl/utils_/cache_df_to_db/append_df_to_table/__init__.py +29 -0
- mindsett_energy_reports_pkg/mdl/utils_/cache_df_to_db/empty_table/__init__.py +11 -0
- mindsett_energy_reports_pkg/mdl/utils_/cache_obj/__init__.py +50 -0
- mindsett_energy_reports_pkg/mdl/utils_/color_dict/__init__.py +13 -0
- mindsett_energy_reports_pkg/mdl/utils_/delete_files_in_directory/__init__.py +15 -0
- mindsett_energy_reports_pkg/mdl/utils_/enriching_time_features/__init__.py +66 -0
- mindsett_energy_reports_pkg/mdl/utils_/enriching_time_features/__version__.py +4 -0
- mindsett_energy_reports_pkg/mdl/utils_/generate_time_partition_list/__init__.py +14 -0
- mindsett_energy_reports_pkg/mdl/utils_/get_group_with_others/__init__.py +7 -0
- mindsett_energy_reports_pkg/mdl/utils_/get_pos_bot_range_unit_format/__init__.py +37 -0
- mindsett_energy_reports_pkg/mdl/utils_/get_pos_bot_range_unit_format/get_pos_bot_and_ylim/__init__.py +31 -0
- mindsett_energy_reports_pkg/mdl/utils_/modify_group_names/__init__.py +17 -0
- mindsett_energy_reports_pkg/mdl/utils_/patch_missing_periods/__init__.py +34 -0
- mindsett_energy_reports_pkg/mdl/utils_/plot_stacked_barchart/__init__.py +50 -0
- mindsett_energy_reports_pkg/mdl/utils_/send_email/__init__.py +73 -0
- mindsett_energy_reports_pkg/mdl/utils_/setup_table/__init__.py +40 -0
- mindsett_energy_reports_pkg/mdl/utils_/setup_table/check_table_existence/__init__.py +25 -0
- mindsett_energy_reports_pkg/mdl/utils_/setup_table/create_new_schema/__init__.py +8 -0
- mindsett_energy_reports_pkg/mdl/utils_/setup_table/create_table/__init__.py +73 -0
- mindsett_energy_reports_pkg/mdl/utils_/setup_table/empty_table/__init__.py +11 -0
@@ -0,0 +1,29 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
from .sync_with_monday import sync_with_monday
|
4
|
+
from .load_cf_from_db import load_cf_from_db
|
5
|
+
|
6
|
+
def fetch_monday_info(engine,
|
7
|
+
monday_token,
|
8
|
+
table_name_cache,
|
9
|
+
table_columns_cache,
|
10
|
+
monday_board_id,
|
11
|
+
columns_concerned=None,
|
12
|
+
email_prefix=None,
|
13
|
+
sync_monday=None):
|
14
|
+
|
15
|
+
if sync_monday is None:
|
16
|
+
sync_monday = True
|
17
|
+
|
18
|
+
# get contacts from db
|
19
|
+
df_monday = load_cf_from_db(engine, table_name_cache, table_columns_cache)
|
20
|
+
|
21
|
+
if sync_monday:
|
22
|
+
df_monday = sync_with_monday(engine,
|
23
|
+
df_monday,
|
24
|
+
monday_token,
|
25
|
+
table_name_cache,
|
26
|
+
monday_board_id,
|
27
|
+
columns_concerned=columns_concerned,
|
28
|
+
email_prefix=email_prefix)
|
29
|
+
return df_monday
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
import pandas as pd
|
3
|
+
|
4
|
+
|
5
|
+
def load_cf_from_db(engine, table_name_cache, columns):
|
6
|
+
|
7
|
+
columns_str = ', '.join(columns)
|
8
|
+
|
9
|
+
cache_query = f"""select {columns_str} from {table_name_cache};"""
|
10
|
+
|
11
|
+
with engine.connect() as conn:
|
12
|
+
df_cache = pd.read_sql_query(cache_query, con=conn)
|
13
|
+
|
14
|
+
if df_cache.shape[0] > 0:
|
15
|
+
df_cache.mailing_list = df_cache.mailing_list.apply(lambda x: x[1:-1].split(','))
|
16
|
+
|
17
|
+
return df_cache
|
@@ -0,0 +1,52 @@
|
|
1
|
+
|
2
|
+
from .get_last_update_time import get_last_update_time
|
3
|
+
from .cache_cf_to_db import cache_cf_to_db
|
4
|
+
from .extract_contacts import extract_contacts
|
5
|
+
|
6
|
+
def sync_with_monday(engine,
|
7
|
+
df_config,
|
8
|
+
monday_token,
|
9
|
+
table_name_cache,
|
10
|
+
monday_board_id,
|
11
|
+
columns_concerned=None,
|
12
|
+
email_prefix=None):
|
13
|
+
|
14
|
+
print('syncing config with monday board ...')
|
15
|
+
|
16
|
+
# check the activity log from monday
|
17
|
+
last_update_time = get_last_update_time(monday_token, monday_board_id)
|
18
|
+
# print('last_update_time: ', last_update_time)
|
19
|
+
|
20
|
+
# if activity log time > db last updates, then extract contacts from monday and update db
|
21
|
+
|
22
|
+
flag_reload_from_monday = True
|
23
|
+
|
24
|
+
if (df_config.shape[0] > 0) and (not df_config.last_update.isna().all()): # last_update was made null when the last_update_time was not available
|
25
|
+
|
26
|
+
prev_update_time = df_config.last_update.max()
|
27
|
+
print('prev_update_time: ', prev_update_time)
|
28
|
+
|
29
|
+
if not (prev_update_time < last_update_time):
|
30
|
+
|
31
|
+
print('[INFO]: using config from the db cache as it is up-to-date!')
|
32
|
+
flag_reload_from_monday = False
|
33
|
+
|
34
|
+
if flag_reload_from_monday:
|
35
|
+
|
36
|
+
print('extracting contacts ... (about 3 min 30 secs)')
|
37
|
+
df_config = extract_contacts(monday_token, monday_board_id,
|
38
|
+
columns_concerned=columns_concerned,
|
39
|
+
email_prefix=email_prefix,
|
40
|
+
# caching=True
|
41
|
+
)
|
42
|
+
|
43
|
+
# cache the result into db for the future usage and efficiency
|
44
|
+
|
45
|
+
df_config['last_update'] = last_update_time
|
46
|
+
# df_config = attach_id_to_buildings(engine, df_config, table_name_org, table_name_building) # df_config has to be the same name, so not renaming it here
|
47
|
+
|
48
|
+
# print('df_config: ', df_config)
|
49
|
+
# quit()
|
50
|
+
cache_cf_to_db(engine, table_name_cache, df_config)
|
51
|
+
|
52
|
+
return df_config
|
mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/cache_cf_to_db/__init__.py
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
|
2
|
+
from .append_df_to_table import append_df_to_table
|
3
|
+
from .empty_table import empty_table
|
4
|
+
|
5
|
+
def cache_cf_to_db(engine, table_name_cache, df_config):
|
6
|
+
|
7
|
+
empty_table(engine, table_name_cache, with_log=False)
|
8
|
+
append_df_to_table(engine, table_name_cache, df_config)
|
9
|
+
|
10
|
+
return
|
11
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
def append_df_to_table(engine, table_name, df):
|
2
|
+
|
3
|
+
table_schema_name_list = table_name.split(".")
|
4
|
+
|
5
|
+
if len(table_schema_name_list) > 1:
|
6
|
+
table_schema = table_schema_name_list[0]
|
7
|
+
table_non_schema_name = table_schema_name_list[1]
|
8
|
+
else:
|
9
|
+
table_schema = "public"
|
10
|
+
table_non_schema_name = table_name
|
11
|
+
|
12
|
+
total_records = len(df)
|
13
|
+
|
14
|
+
|
15
|
+
with engine.connect() as conn:
|
16
|
+
|
17
|
+
df.to_sql(
|
18
|
+
name=table_non_schema_name,
|
19
|
+
con=conn,
|
20
|
+
if_exists="append",
|
21
|
+
schema=table_schema,
|
22
|
+
chunksize=5000,
|
23
|
+
index=False
|
24
|
+
)
|
25
|
+
|
26
|
+
print(f'The config cache in database table [{table_name}] is updated with [{total_records}] records.')
|
27
|
+
# print(f'[{total_records}] records have been written to Table [{table_name}] successfully.')
|
28
|
+
|
29
|
+
return
|
@@ -0,0 +1,11 @@
|
|
1
|
+
def empty_table(engine, table_name, with_log=True):
|
2
|
+
|
3
|
+
with engine.connect() as conn:
|
4
|
+
|
5
|
+
# delete data (empty) in the table
|
6
|
+
query_empty_table = f"""DELETE FROM {table_name};"""
|
7
|
+
conn.execute(query_empty_table)
|
8
|
+
|
9
|
+
if with_log:
|
10
|
+
print(f'The table [{table_name}] is empty now.')
|
11
|
+
return
|
mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/extract_contacts/__init__.py
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
import uuid
|
2
|
+
import numpy as np
|
3
|
+
|
4
|
+
from cache_return import cache_return
|
5
|
+
|
6
|
+
from .load_mailing_list import load_mailing_list
|
7
|
+
# from ..cache_result import cache_result
|
8
|
+
from .get_mailing_list import get_mailing_list
|
9
|
+
|
10
|
+
@cache_return
|
11
|
+
def extract_contacts(monday_token,
|
12
|
+
board_id,
|
13
|
+
columns_concerned=None,
|
14
|
+
email_prefix=None):
|
15
|
+
|
16
|
+
# email_columns = [f'email {idx+1}' for idx in range(max_no_email)]
|
17
|
+
# columns = [building_column, *email_columns]
|
18
|
+
# print('columns: ', columns)
|
19
|
+
|
20
|
+
df_contact = load_mailing_list(monday_token, board_id,
|
21
|
+
columns_concerned=columns_concerned,
|
22
|
+
email_prefix=email_prefix,
|
23
|
+
caching=False)
|
24
|
+
|
25
|
+
# # make the names lower case
|
26
|
+
df_contact.columns = [column.lower() for column in df_contact.columns]
|
27
|
+
df_contact = df_contact.rename(columns={'name': 'item_name'})
|
28
|
+
df_contact['building_id'] = df_contact['building_id'].apply(uuid.UUID)
|
29
|
+
|
30
|
+
# df_contact['review'] = df_contact[role_column].str.contains('Review', na=False)
|
31
|
+
# df_contact['receive'] = df_contact[role_column].str.contains('Receive', na=False)
|
32
|
+
|
33
|
+
# df_receive = df_contact.loc[df_contact['receive']]
|
34
|
+
# df_review = df_contact.loc[df_contact['review']]
|
35
|
+
|
36
|
+
# contacts = {}
|
37
|
+
|
38
|
+
# contacts['receivers'] = df_receive.groupby(name_column)[email_column].apply(list).reset_index().to_dict('records')
|
39
|
+
# contacts['reviewers'] = df_review.groupby(name_column)[email_column].apply(list).reset_index().to_dict('records')
|
40
|
+
|
41
|
+
# df_contact.publish = (df_contact.publish == 'v')
|
42
|
+
# df_contact.testing = (df_contact.testing == 'v')
|
43
|
+
# df_contact.occupancy_available = (df_contact.occupancy_available == 'v')
|
44
|
+
df_contact.pct_level_tobe_others = df_contact.pct_level_tobe_others.astype('float')
|
45
|
+
df_contact.pct_hide = df_contact.pct_hide.replace('', '0').astype('float')
|
46
|
+
df_contact.conv_mwh_price = df_contact.conv_mwh_price.replace('', '0').astype('float')
|
47
|
+
# df_contact.insight_statements = df_contact.insight_statements.replace('', None)
|
48
|
+
df_contact.floor_sqm = df_contact.floor_sqm.replace('', np.NaN).astype('float')
|
49
|
+
|
50
|
+
df_contact['mailing_list'] = df_contact.apply(get_mailing_list, axis=1)
|
51
|
+
|
52
|
+
monday_columns = df_contact.columns
|
53
|
+
email_columns = [column for column in monday_columns if 'email' in column]
|
54
|
+
|
55
|
+
df_contact = df_contact.drop(columns=email_columns)
|
56
|
+
|
57
|
+
return df_contact
|
@@ -0,0 +1,25 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
def get_mailing_list(sr_contact_row):
|
4
|
+
|
5
|
+
# df_contact_site = df_contact.loc[(df_contact.name==(site_name).replace("''", "'")) & (df_contact.org==org_name)]
|
6
|
+
|
7
|
+
monday_columns = sr_contact_row.index
|
8
|
+
email_columns = [column for column in monday_columns if 'email' in column]
|
9
|
+
|
10
|
+
# df_emails_site = df_contact_site[email_columns]
|
11
|
+
|
12
|
+
ls_emails_site = sr_contact_row[email_columns].tolist()
|
13
|
+
|
14
|
+
# print('ls_emails_site: ', ls_emails_site)
|
15
|
+
|
16
|
+
ls_emails_site_nona = [i for i in ls_emails_site if i != None]
|
17
|
+
|
18
|
+
# ls_emails_site_nona
|
19
|
+
# print('ls_emails_site_nona: ', ls_emails_site_nona)
|
20
|
+
|
21
|
+
ls_emails_site_nona_with_at = [email_str for email_str in ls_emails_site_nona if '@' in email_str]
|
22
|
+
|
23
|
+
# print('ls_emails_site_nona: ', ls_emails_site_nona)
|
24
|
+
|
25
|
+
return ls_emails_site_nona_with_at
|
@@ -0,0 +1,50 @@
|
|
1
|
+
import pandas as pd
|
2
|
+
|
3
|
+
from MondayProcessor import MondayProcessor
|
4
|
+
from cache_return import cache_return
|
5
|
+
|
6
|
+
@cache_return
|
7
|
+
def load_mailing_list(monday_token, board_id,
|
8
|
+
columns_concerned=None,
|
9
|
+
email_prefix=None):
|
10
|
+
|
11
|
+
if columns_concerned is None:
|
12
|
+
from .columns_concerned import columns_concerned
|
13
|
+
|
14
|
+
|
15
|
+
email_prefix = email_prefix or 'email'
|
16
|
+
|
17
|
+
mp = MondayProcessor(monday_token)
|
18
|
+
|
19
|
+
groups_dict = mp._fetchGroups(board_id)
|
20
|
+
|
21
|
+
columns_data = mp.monday.boards.fetch_columns_by_board_id(board_ids=board_id)
|
22
|
+
|
23
|
+
columns_all = [column['title'] for column in columns_data['data']['boards'][0]['columns']]
|
24
|
+
len_ep = len(email_prefix)
|
25
|
+
columns_emails = [column for column in columns_all if column[:len_ep]==(email_prefix[:len_ep])]
|
26
|
+
|
27
|
+
missing_columns = []
|
28
|
+
for concerned_column in columns_concerned:
|
29
|
+
if concerned_column not in columns_all:
|
30
|
+
missing_columns.append(concerned_column)
|
31
|
+
|
32
|
+
if len(missing_columns) > 0:
|
33
|
+
raise Exception(f'[ERROR]: There are missing columns: {missing_columns}!')
|
34
|
+
|
35
|
+
columns_select = columns_concerned + columns_emails # merge the two lists
|
36
|
+
|
37
|
+
|
38
|
+
# print('columns_all: ', columns_all)
|
39
|
+
# print('columns_select: ', columns_select)
|
40
|
+
|
41
|
+
df_board = pd.DataFrame([])
|
42
|
+
for group_id in groups_dict:
|
43
|
+
|
44
|
+
board_results = mp._fetchColumnValueByGroup(board_id=board_id, groupId=group_id, columns=columns_select)
|
45
|
+
|
46
|
+
df_group = pd.DataFrame(board_results)
|
47
|
+
df_group['group_name'] = groups_dict[group_id]
|
48
|
+
df_board = pd.concat([df_board, df_group])
|
49
|
+
|
50
|
+
return df_board
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
columns_concerned = ['Name',
|
3
|
+
'publish',
|
4
|
+
'testing',
|
5
|
+
'industry',
|
6
|
+
'manager_name',
|
7
|
+
'currency',
|
8
|
+
'occupancy_available',
|
9
|
+
'asset_group',
|
10
|
+
'fillna_value',
|
11
|
+
'insight_statements',
|
12
|
+
'group_name_column_exchange',
|
13
|
+
'group_name_modification',
|
14
|
+
'pct_level_tobe_others',
|
15
|
+
'floor_sqm',
|
16
|
+
'pct_hide',
|
17
|
+
'conv_mwh_price',
|
18
|
+
'period_freq',
|
19
|
+
'building_id',
|
20
|
+
'board_id',
|
21
|
+
'hvac_board_id'
|
22
|
+
]
|
mindsett_energy_reports_pkg/libs/fetch_monday_info/sync_with_monday/get_last_update_time/__init__.py
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
import pandas as pd
|
3
|
+
from MondayProcessor import MondayProcessor
|
4
|
+
|
5
|
+
def get_last_update_time(monday_token, board_id):
|
6
|
+
|
7
|
+
mp = MondayProcessor(monday_token)
|
8
|
+
|
9
|
+
js_logs = mp._fetchBoardLogsById(board_id)
|
10
|
+
|
11
|
+
dct_records = js_logs['data']['boards'][0]['activity_logs']
|
12
|
+
|
13
|
+
# print('dct_records: ', dct_records)
|
14
|
+
|
15
|
+
if dct_records == None: # it happened on 11 Nov 24 that the activity log is not available and returned None
|
16
|
+
|
17
|
+
print('js_logs: ', js_logs)
|
18
|
+
print('[WARN]: using time now as last_update_time because the Monday ACTIVITY LOG is NOT AVAILABLE !')
|
19
|
+
last_update_time = pd.Timestamp.now(tz='UTC').ceil(freq='s')
|
20
|
+
return last_update_time
|
21
|
+
else:
|
22
|
+
df_logs = pd.DataFrame.from_records(dct_records)
|
23
|
+
df_logs['created_at'] = pd.to_datetime(df_logs['created_at'].astype('int').div(10), unit='us', utc=True).dt.ceil(freq='s')
|
24
|
+
last_update_time = df_logs.created_at.max()
|
25
|
+
|
26
|
+
return last_update_time
|
@@ -0,0 +1,27 @@
|
|
1
|
+
|
2
|
+
from .load_cf_from_db import load_cf_from_db
|
3
|
+
from .sync_with_sharepoint import sync_with_sharepoint
|
4
|
+
|
5
|
+
def fetch_sharepoint_info(engine,
|
6
|
+
sharepoint,
|
7
|
+
table_name_cache,
|
8
|
+
table_columns_cache,
|
9
|
+
sync_source=None):
|
10
|
+
|
11
|
+
if sync_source is None:
|
12
|
+
sync_source = True
|
13
|
+
|
14
|
+
# get contacts/configs from the db
|
15
|
+
df_cfg_db = load_cf_from_db(engine, table_name_cache, table_columns_cache)
|
16
|
+
|
17
|
+
# print(f'{df_cfg_db.info()=}')
|
18
|
+
|
19
|
+
if sync_source:
|
20
|
+
df_cfg = sync_with_sharepoint(engine,
|
21
|
+
df_cfg_db,
|
22
|
+
sharepoint,
|
23
|
+
table_name_cache)
|
24
|
+
else:
|
25
|
+
df_cfg = df_cfg_db
|
26
|
+
|
27
|
+
return df_cfg
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
import pandas as pd
|
3
|
+
|
4
|
+
|
5
|
+
def load_cf_from_db(engine, table_name_cache, columns):
|
6
|
+
|
7
|
+
columns_str = ', '.join(columns)
|
8
|
+
|
9
|
+
cache_query = f"""select {columns_str} from {table_name_cache};"""
|
10
|
+
|
11
|
+
with engine.connect() as conn:
|
12
|
+
df_cache = pd.read_sql_query(cache_query, con=conn)
|
13
|
+
|
14
|
+
if df_cache.shape[0] > 0:
|
15
|
+
df_cache.mailing_list = df_cache.mailing_list.apply(lambda x: x[1:-1].split(','))
|
16
|
+
|
17
|
+
return df_cache
|
@@ -0,0 +1,28 @@
|
|
1
|
+
|
2
|
+
from .extract_contacts import extract_contacts
|
3
|
+
from .cache_cf_to_db import cache_cf_to_db
|
4
|
+
|
5
|
+
def sync_with_sharepoint(engine, df_cfg_db, sharepoint, table_name_cache):
|
6
|
+
|
7
|
+
df_cfg_shp = extract_contacts(sharepoint) # currently, we are getting the last_update time by directly querying the file, as it is relative quick
|
8
|
+
# if needed in the future, a dedicated lighter get_last_update_time function can be designed, and the actual extraction processing can be conditional
|
9
|
+
|
10
|
+
flag_reload_from_sharepoint = True
|
11
|
+
|
12
|
+
if (df_cfg_db.shape[0] > 0) and (not df_cfg_db.last_update.isna().all()): # last_update was make null when the last_update_time was not available
|
13
|
+
|
14
|
+
last_update_db = df_cfg_db['last_update'].max()
|
15
|
+
last_update_shp = df_cfg_shp['last_update'].max()
|
16
|
+
# print(f'{last_update_shp=}')
|
17
|
+
|
18
|
+
if last_update_db == last_update_shp: # here some efforts of writing to db can be saved if the cached data is already up-to-date.
|
19
|
+
|
20
|
+
print('[INFO]: using config from the db cache as it is up-to-date!')
|
21
|
+
flag_reload_from_sharepoint = False
|
22
|
+
|
23
|
+
if flag_reload_from_sharepoint:
|
24
|
+
|
25
|
+
# pass
|
26
|
+
cache_cf_to_db(engine, table_name_cache, df_cfg_shp)
|
27
|
+
|
28
|
+
return df_cfg_shp
|
@@ -0,0 +1,11 @@
|
|
1
|
+
|
2
|
+
from .append_df_to_table import append_df_to_table
|
3
|
+
from .empty_table import empty_table
|
4
|
+
|
5
|
+
def cache_cf_to_db(engine, table_name_cache, df_config):
|
6
|
+
|
7
|
+
empty_table(engine, table_name_cache, with_log=False)
|
8
|
+
append_df_to_table(engine, table_name_cache, df_config)
|
9
|
+
|
10
|
+
return
|
11
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
def append_df_to_table(engine, table_name, df):
|
2
|
+
|
3
|
+
table_schema_name_list = table_name.split(".")
|
4
|
+
|
5
|
+
if len(table_schema_name_list) > 1:
|
6
|
+
table_schema = table_schema_name_list[0]
|
7
|
+
table_non_schema_name = table_schema_name_list[1]
|
8
|
+
else:
|
9
|
+
table_schema = "public"
|
10
|
+
table_non_schema_name = table_name
|
11
|
+
|
12
|
+
total_records = len(df)
|
13
|
+
|
14
|
+
|
15
|
+
with engine.connect() as conn:
|
16
|
+
|
17
|
+
df.to_sql(
|
18
|
+
name=table_non_schema_name,
|
19
|
+
con=conn,
|
20
|
+
if_exists="append",
|
21
|
+
schema=table_schema,
|
22
|
+
chunksize=5000,
|
23
|
+
index=False
|
24
|
+
)
|
25
|
+
|
26
|
+
print(f'The config cache in database table [{table_name}] is updated with [{total_records}] records.')
|
27
|
+
# print(f'[{total_records}] records have been written to Table [{table_name}] successfully.')
|
28
|
+
|
29
|
+
return
|
@@ -0,0 +1,11 @@
|
|
1
|
+
def empty_table(engine, table_name, with_log=True):
|
2
|
+
|
3
|
+
with engine.connect() as conn:
|
4
|
+
|
5
|
+
# delete data (empty) in the table
|
6
|
+
query_empty_table = f"""DELETE FROM {table_name};"""
|
7
|
+
conn.execute(query_empty_table)
|
8
|
+
|
9
|
+
if with_log:
|
10
|
+
print(f'The table [{table_name}] is empty now.')
|
11
|
+
return
|
@@ -0,0 +1,13 @@
|
|
1
|
+
|
2
|
+
from .load_mailing_list import load_mailing_list
|
3
|
+
from .format_contacts import format_contacts
|
4
|
+
|
5
|
+
def extract_contacts(sharepoint):
|
6
|
+
|
7
|
+
df_contact_raw = load_mailing_list(sharepoint)
|
8
|
+
|
9
|
+
df_contact = format_contacts(df_contact_raw,
|
10
|
+
columns_concerned=sharepoint.excel.columns_concerned
|
11
|
+
)
|
12
|
+
|
13
|
+
return df_contact
|
@@ -0,0 +1,57 @@
|
|
1
|
+
|
2
|
+
import uuid
|
3
|
+
|
4
|
+
from .uuid_for_nona import uuid_for_nona
|
5
|
+
from .extract_id import extract_id
|
6
|
+
from .get_mailing_list import get_mailing_list
|
7
|
+
|
8
|
+
def format_contacts(df_contact,
|
9
|
+
columns_concerned=None):
|
10
|
+
|
11
|
+
if columns_concerned is None:
|
12
|
+
from .columns_concerned import columns_concerned
|
13
|
+
|
14
|
+
pct_level_tobe_others_default = 0.03
|
15
|
+
pct_hide_default = 4
|
16
|
+
conv_mwh_price_default = 190
|
17
|
+
|
18
|
+
# standardize the column names
|
19
|
+
df_contact.columns = [column.lower() for column in df_contact.columns]
|
20
|
+
df_contact = df_contact.rename(columns={'building_name': 'item_name', 'org': 'group_name'})
|
21
|
+
|
22
|
+
# fill the names for null values
|
23
|
+
df_contact['group_name'] = df_contact['group_name'].ffill()
|
24
|
+
df_contact['insight_statements'] = df_contact['insight_statements'].fillna('')
|
25
|
+
df_contact['group_name_column_exchange'] = df_contact['group_name_column_exchange'].fillna('')
|
26
|
+
df_contact['group_name_modification'] = df_contact['group_name_modification'].fillna('')
|
27
|
+
df_contact['manager_name'] = df_contact['manager_name'].fillna('')
|
28
|
+
|
29
|
+
# columns type formatting
|
30
|
+
df_contact['publish'] = (df_contact['publish'] == 'v')
|
31
|
+
df_contact['testing'] = (df_contact['testing'] == 'v')
|
32
|
+
df_contact['occupancy_available'] = (df_contact['occupancy_available'] == 'v')
|
33
|
+
df_contact['building_id'] = df_contact['building_id'].apply(uuid_for_nona)
|
34
|
+
|
35
|
+
df_contact['pct_level_tobe_others'] = df_contact['pct_level_tobe_others'].fillna(pct_level_tobe_others_default).astype('float')
|
36
|
+
df_contact['floor_sqm'] = df_contact['floor_sqm'].astype('float')
|
37
|
+
|
38
|
+
df_contact['pct_hide'] = df_contact['pct_hide'].fillna(pct_hide_default).astype('float')
|
39
|
+
|
40
|
+
df_contact['conv_mwh_price'] = df_contact['conv_mwh_price'].fillna(conv_mwh_price_default).astype('float')
|
41
|
+
|
42
|
+
# format the monday board ids
|
43
|
+
df_contact['board_id'] = df_contact['board_id'].apply(extract_id)
|
44
|
+
df_contact['hvac_board_id'] = df_contact['hvac_board_id'].apply(extract_id)
|
45
|
+
|
46
|
+
# get_mailing_list
|
47
|
+
df_contact['mailing_list'] = df_contact.apply(get_mailing_list, axis=1)
|
48
|
+
|
49
|
+
# drop the email related columns
|
50
|
+
monday_columns = df_contact.columns
|
51
|
+
email_columns = [column for column in monday_columns if 'email' in column]
|
52
|
+
df_contact = df_contact.drop(columns=email_columns)
|
53
|
+
|
54
|
+
# select the concerned columns
|
55
|
+
df_contact_sel = df_contact[columns_concerned]
|
56
|
+
|
57
|
+
return df_contact_sel
|
@@ -0,0 +1,8 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
columns_concerned = ['last_update', 'building_id', 'item_name', 'group_name', 'board_id',
|
4
|
+
'hvac_board_id', 'publish', 'testing', 'industry', 'manager_name',
|
5
|
+
'currency', 'occupancy_available', 'asset_group', 'fillna_value',
|
6
|
+
'insight_statements', 'group_name_column_exchange',
|
7
|
+
'group_name_modification', 'pct_level_tobe_others', 'floor_sqm', 'pct_hide',
|
8
|
+
'conv_mwh_price', 'period_freq', 'mailing_list']
|
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
def extract_id(x):
|
4
|
+
|
5
|
+
# example value : '5515593330 - https://cloudfmgroup-company.monday.com/boards/5515593330'
|
6
|
+
|
7
|
+
# if isinstance(x, float):
|
8
|
+
# x = int(x)
|
9
|
+
|
10
|
+
str_x = str(x)
|
11
|
+
if ' - ' in str_x:
|
12
|
+
id_str = str_x.split(' - ')[0]
|
13
|
+
else:
|
14
|
+
id_str = str_x
|
15
|
+
|
16
|
+
if id_str == 'nan':
|
17
|
+
return None
|
18
|
+
else:
|
19
|
+
return id_str.split('.')[0] # there might be the case that it's converted from float with . e.g. 8370787257.0
|
@@ -0,0 +1,10 @@
|
|
1
|
+
|
2
|
+
def get_mailing_list(sr_contact_row):
|
3
|
+
|
4
|
+
monday_columns = sr_contact_row.index
|
5
|
+
email_columns = [column for column in monday_columns if 'email' in column]
|
6
|
+
ls_emails_site = sr_contact_row[email_columns].tolist()
|
7
|
+
ls_emails_site_nona = [i for i in ls_emails_site if isinstance(i, str)]
|
8
|
+
ls_emails_site_nona_with_at = [email_str for email_str in ls_emails_site_nona if '@' in email_str]
|
9
|
+
|
10
|
+
return ls_emails_site_nona_with_at
|
@@ -0,0 +1,30 @@
|
|
1
|
+
|
2
|
+
from office365.sharepoint.client_context import ClientContext
|
3
|
+
from office365.runtime.auth.client_credential import ClientCredential
|
4
|
+
|
5
|
+
from .get_excel_file_from_link import get_excel_file_from_link
|
6
|
+
from .load_excel_as_df import load_excel_as_df
|
7
|
+
|
8
|
+
|
9
|
+
def load_mailing_list(sharepoint):
|
10
|
+
|
11
|
+
# get sharepoint context
|
12
|
+
ctx = ClientContext(sharepoint.site_url).with_credentials(ClientCredential(sharepoint.client_id, sharepoint.client_secret))
|
13
|
+
|
14
|
+
file_name = getattr(sharepoint.excel, 'file_name', None)
|
15
|
+
sheet_name = getattr(sharepoint.excel, 'sheet', None)
|
16
|
+
|
17
|
+
if file_name is None:
|
18
|
+
raise Exception('[ERROR]: Please specify the excel file name!')
|
19
|
+
|
20
|
+
sharepoint_link = sharepoint.folder_url + '/' + file_name
|
21
|
+
|
22
|
+
excel_file = get_excel_file_from_link(ctx, sharepoint_link)
|
23
|
+
|
24
|
+
# sheet_name = sharepoint.excel.sheet
|
25
|
+
|
26
|
+
df_contact = load_excel_as_df(ctx, excel_file,
|
27
|
+
sheet_name=sheet_name)
|
28
|
+
|
29
|
+
return df_contact
|
30
|
+
|