calendar_event_engine 0.1.0__tar.gz

calendar_event_engine-0.1.0/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Ezequiel Valencia
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

calendar_event_engine-0.1.0/PKG-INFO

@@ -0,0 +1,57 @@
+Metadata-Version: 2.1
+Name: calendar_event_engine
+Version: 0.1.0
+Summary: 
+Author: Ezequiel Valencia
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: aiohttp (>=3.10.10,<4.0.0)
+Requires-Dist: beautifulsoup4 (>=4.13.4,<5.0.0)
+Requires-Dist: geopy (>=2.4.1,<3.0.0)
+Requires-Dist: google-api-python-client (>=2.127.0,<3.0.0)
+Requires-Dist: google-auth-httplib2 (>=0.2.0,<0.3.0)
+Requires-Dist: google-auth-oauthlib (>=1.2.0,<2.0.0)
+Requires-Dist: gql (==3.5.0)
+Requires-Dist: icalendar (>=6.0.1,<7.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: pydantic (>=2.7.0,<3.0.0)
+Requires-Dist: pytest (>=8.2.2,<9.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
+Requires-Dist: slack_sdk (>=3.32.0,<4.0.0)
+Requires-Dist: tenacity (>=8.2.3,<9.0.0)
+Requires-Dist: validators (>=0.34.0,<0.35.0)
+Description-Content-Type: text/markdown
+
+# EventScraper
+
+A project dedicated to scrapping events and posting them on 
+different platforms for searching. Unbinding posted events from any single platform to help illuminate what is locally happening.
+
+## Docker (Recommended Method)
+#### [Docker Image](https://hub.docker.com/repository/docker/avocadomoon/event-scraper/general)
+#### Env Variables
+Mobilizon Related Variables:
+- MOBILIZON_ENDPOINT: Graphql endpoint for your mobilizon instance
+
+- MOBILIZON_EMAIL: User email
+
+- MOBILIZON_PASSWORD: User password
+
+OS Related Variables:
+- USER_ID: User process ID for running the application
+
+- GROUP_ID: Group process ID for running the application
+
+Misc:
+- RUNNER_SUBMISSION_JSON_PATH: Remote submission file that tells where to publish and what group packages to use for sources.
+
+- SLACK_WEBHOOK: Slack token to give notifications on how the scraping process is going and whether any maintenance is required.
+
+---
+#### Docker Volumes
+/app/config:[HOST DIR] -- The files needed within this config folder are "token.json" if you happen to use the Google Calendar scraper. Please follow instructions on the wiki page to create this "token.json" file.
+

calendar_event_engine-0.1.0/README.md

@@ -0,0 +1,28 @@
+# EventScraper
+
+A project dedicated to scrapping events and posting them on 
+different platforms for searching. Unbinding posted events from any single platform to help illuminate what is locally happening.
+
+## Docker (Recommended Method)
+#### [Docker Image](https://hub.docker.com/repository/docker/avocadomoon/event-scraper/general)
+#### Env Variables
+Mobilizon Related Variables:
+- MOBILIZON_ENDPOINT: Graphql endpoint for your mobilizon instance
+
+- MOBILIZON_EMAIL: User email
+
+- MOBILIZON_PASSWORD: User password
+
+OS Related Variables:
+- USER_ID: User process ID for running the application
+
+- GROUP_ID: Group process ID for running the application
+
+Misc:
+- RUNNER_SUBMISSION_JSON_PATH: Remote submission file that tells where to publish and what group packages to use for sources.
+
+- SLACK_WEBHOOK: Slack token to give notifications on how the scraping process is going and whether any maintenance is required.
+
+---
+#### Docker Volumes
+/app/config:[HOST DIR] -- The files needed within this config folder are "token.json" if you happen to use the Google Calendar scraper. Please follow instructions on the wiki page to create this "token.json" file.

calendar_event_engine-0.1.0/calendar_event_engine/Runner.py

@@ -0,0 +1,169 @@
+import datetime
+import os
+import time
+import traceback
+from urllib.error import HTTPError
+
+from slack_sdk.webhook import WebhookClient
+
+from calendar_event_engine.db_cache import SQLiteDB
+from calendar_event_engine.filter import normalize_generic_event
+from calendar_event_engine.logger import create_logger_from_designated_logger
+from calendar_event_engine.parser.submission import get_runner_submission
+from calendar_event_engine.publishers.abc_publisher import Publisher
+from calendar_event_engine.scrapers.abc_scraper import Scraper
+from calendar_event_engine.types.custom_scraper import CustomScraperJob
+from calendar_event_engine.types.submission import GroupPackage, EventsToUploadFromCalendarID, GroupEventsKernel
+from calendar_event_engine.types.submission_handlers import RunnerSubmission
+from calendar_event_engine.scrapers.google_calendar.api import ExpiredToken
+
+logger = create_logger_from_designated_logger(__name__)
+
+
+def _runner(runner_submission: RunnerSubmission, custom_scrapers: dict[Publisher, list[CustomScraperJob]] = None):
+    continue_scraping = True
+    num_retries = 0
+    theres_an_expired_token = False
+    while continue_scraping and num_retries < 5:
+        try:
+            submitted_publishers: {Publisher: list[GroupPackage]} = runner_submission.publishers
+            for publisher in submitted_publishers.keys():
+                publisher: Publisher
+                publisher.connect()
+                group_package: GroupPackage
+                for group_package in submitted_publishers[publisher]:
+                    logger.info(f"Reading Group Package: {group_package.package_name}")
+
+                    for scraper_type in group_package.scraper_type_and_kernels.keys():
+                        scraper: Scraper = runner_submission.respective_scrapers[scraper_type]
+                        try:
+                            scraper.connect_to_source()
+                            group_event_kernels: list[GroupEventsKernel] = group_package.scraper_type_and_kernels[
+                                scraper_type]
+                            for event_kernel in group_event_kernels:
+                                event_kernel: GroupEventsKernel
+                                try:
+                                    events: list[EventsToUploadFromCalendarID] = scraper.retrieve_from_source(event_kernel)
+                                except HTTPError as err:
+                                    if err.code == 404:
+                                        logger.warning(f"The following group is no longer available: {event_kernel.group_name}")
+                                    else:
+                                        raise
+                                normalize_generic_event(events)
+                                publisher.upload(events)
+                            scraper.close_connection_to_source()
+                        except ExpiredToken:
+                            theres_an_expired_token = True
+                            logger.warning("Expired token.json needs to be replaced. Will continue scraping other types")
+                            continue
+                publisher.close()
+
+
+            if custom_scrapers:
+                for publisher in custom_scrapers.keys():
+                    publisher.connect()
+                    for scrap in custom_scrapers[publisher]:
+                        try:
+                            logger.info(f"Using custom scraper {scrap.scraper_name}")
+                            scrap.custom_scraper.connect_to_source()
+                            events = scrap.custom_scraper.retrieve_from_source(None)
+                            normalize_generic_event(events)
+                            publisher.upload(events)
+                        except Exception as custom_err:
+                            logger.error("Exception for custom scraper: " + scrap.scraper_name, custom_err)
+                    publisher.close()
+            continue_scraping = False
+        except HTTPError as err:
+            if err.code == 500 and err.reason.lower() == 'Too many requests'.lower():
+                num_retries += 1
+                logger.warning("Going to sleep then retrying to scrape. Retry Num: " + num_retries)
+                time.sleep(120)
+    if theres_an_expired_token:
+        raise ExpiredToken
+
+def _days_to_sleep(days):
+    now = datetime.datetime.now()
+    seconds_from_zero = (now.hour * 60 * 60)
+    time_at_2am = 2 * 60 * 60
+    time_to_sleep = time_at_2am
+    if seconds_from_zero > time_at_2am:
+        time_to_sleep = ((23 * 60 * 60) - seconds_from_zero) + time_at_2am
+    else:
+        time_to_sleep = time_at_2am - seconds_from_zero
+    
+    return time_to_sleep + (60 * 60 * 24 * days)
+
+
+def _produce_slack_message(color, title, text, priority):
+    return {
+            "color": color,
+            "author_name": "CTEvent Scraper",
+            "author_icon": "https://ctgrassroots.org/favicon.ico",
+            "title": title,
+            "title_link": "google.com",
+            "text": text,
+            "fields": [
+                {
+                    "title": "Priority",
+                    "value": priority,
+                    "short": "false"
+                }
+            ],
+            "footer": "CTEvent Scraper",
+        }
+
+def start_event_engine(remote_json_url: str, slack_webhook: WebhookClient = None, custom_scrapers: dict[Publisher, list[CustomScraperJob]] = None, test_mode: bool = False):
+    logger.info("Scraper Started")
+    sleeping = 2
+    while True:
+        #####################
+        # Create Submission #
+        #####################
+
+        cache_db: SQLiteDB = SQLiteDB(test_mode)
+        submission: RunnerSubmission = get_runner_submission(remote_json_url, test_mode, cache_db)
+
+        ######################
+        # Execute Submission #
+        ######################
+        time_to_sleep = _days_to_sleep(sleeping)
+        logger.info("Scraping")
+        try:
+            _runner(submission, custom_scrapers)
+            logger.info("Sleeping " + str(sleeping) + " Days Until Next Scrape")
+        except ExpiredToken:
+            time_to_sleep = _days_to_sleep(2)
+            logger.info("Sleeping " + str(sleeping) + " Days Until Next Scrape")
+            if slack_webhook is not None:
+                response = slack_webhook.send(attachments=[
+                    _produce_slack_message("#e6e209", "Expired Token", "Replace token.json", "Medium")
+                ])
+
+        except Exception as e:
+            logger.error("Unknown Error")
+            logger.error(e)
+            logger.error(traceback.format_exc())
+            logger.error("Going to Sleep for 7 days")
+            if slack_webhook is not None:
+                slack_webhook.send(attachments=[
+                    _produce_slack_message("#ab1a13", "Event Scraper Unknown Error", "Check logs for error.", "High")
+                ])
+            time_to_sleep = _days_to_sleep(7)
+
+        cache_db.close()
+        time.sleep(time_to_sleep)
+
+
+if __name__ == "__main__":
+    env_webhook = None if os.environ.get("SLACK_WEBHOOK") is None else WebhookClient(os.environ.get("SLACK_WEBHOOK"))
+    env_test_mode = False if "TEST_MODE" not in os.environ else True
+    submission_json_path = os.getenv("RUNNER_SUBMISSION_JSON_PATH")
+    if submission_json_path is None:
+        err = NotImplementedError("Did not find JSON submission path")
+        logger.error(err)
+        raise err
+
+
+    start_event_engine(submission_json_path, env_webhook, None, env_test_mode)
+
+

calendar_event_engine-0.1.0/calendar_event_engine/db_cache.py

@@ -0,0 +1,147 @@
+import os
+import sqlite3
+from datetime import datetime
+
+from calendar_event_engine.logger import create_logger_from_designated_logger
+
+logger = create_logger_from_designated_logger(__name__)
+
+class UploadedEventRow:
+    uuid: str
+    id: str
+    title: str
+    date: str
+    groupID: str
+    groupName: str
+    
+    def __init__(self, uuid: str, id: str, title: str, date: str, group_id: str, group_name: str):
+        """_summary_
+
+        Args:
+            uuid (str): _description_
+            id (str): _description_
+            title (str): _description_
+            date (str): Has to be of format ISO8601 that is YYYY-MM-DD HH:MM:SS.SSS. Can also have T in the center if desired.
+            group_id (str): _description_
+        """
+        
+        self.uuid = uuid
+        self.id = id
+        self.title = title
+        self.date = date
+        self.groupID = group_id
+        self.groupName = group_name
+
+class UploadSource:
+    uuid: str
+    websiteURL: str
+    source: str
+    sourceType: str
+    
+    def __init__(self, uuid:str, website_url: str, source:str, source_type: str):
+        self.uuid = uuid
+        self.websiteURL = website_url
+        self.source = source
+        self.sourceType = source_type
+
+
+
+class SQLiteDB:
+    sql_db_connection: sqlite3.Connection
+    uploaded_events_table_name = "uploaded_events"
+    event_source_table_name = "event_source"
+    
+    allColumns = f"""{uploaded_events_table_name}.uuid, {uploaded_events_table_name}.id,
+    {uploaded_events_table_name}.title, {uploaded_events_table_name}.date, 
+    {uploaded_events_table_name}.group_id, {uploaded_events_table_name}.group_name,
+    {event_source_table_name}.websiteURL, {event_source_table_name}.source, {event_source_table_name}.sourceType"""
+    
+    def __init__(self, in_memory_sq_lite: bool = False):
+        if in_memory_sq_lite:
+            self.sql_db_connection = sqlite3.connect(":memory:")
+        else:
+            cache_db_path = os.environ.get("CACHE_DB_PATH")
+            if cache_db_path is not None:
+                self.sql_db_connection = sqlite3.connect(cache_db_path + "/event_cache.db")
+            else:
+                self.sql_db_connection = sqlite3.connect("event_cache.db")
+        self.initialize_db()
+    
+    def initialize_db(self) -> sqlite3.Connection:
+        db_cursor = self.sql_db_connection.cursor()
+        db_cursor.execute(f"""CREATE TABLE IF NOT EXISTS {self.uploaded_events_table_name} 
+                          (uuid PRIMARY KEY, id, title text, date text, group_id, group_name)""")
+        db_cursor.execute(f"""CREATE  TABLE IF NOT EXISTS {self.event_source_table_name}
+                          (uuid, websiteURL text, source text, sourceType text, 
+                          FOREIGN KEY (uuid) REFERENCES uploaded_events(uuid) ON DELETE CASCADE)""" )
+
+    def close(self):
+        self.sql_db_connection.close()
+
+    # https://www.sqlite.org/lang_datefunc.html
+    # Uses built in date time function
+    def delete_all_month_old_events(self):
+        db_cursor = self.sql_db_connection.cursor()
+        db_cursor.execute(f"DELETE FROM {self.uploaded_events_table_name} WHERE datetime(date) < datetime('now', '-1 month')")
+        
+        self.sql_db_connection.commit()
+
+    def select_all_rows_with_calendar_id(self, source):
+        db_cursor = self.sql_db_connection.cursor()
+        # Comma at the end of (groupID,) turns it into a tuple
+        res = db_cursor.execute(f"""SELECT {self.allColumns} FROM {self.uploaded_events_table_name} 
+                                INNER JOIN {self.event_source_table_name} ON 
+                                {self.uploaded_events_table_name}.uuid={self.event_source_table_name}.uuid
+                                WHERE source = ?""", (source,))
+        return res
+    
+    def get_last_event_date_for_source_id(self, calendar_id) -> datetime:
+        db_cursor = self.sql_db_connection.cursor()
+        res = db_cursor.execute(f"""SELECT date FROM {self.uploaded_events_table_name}
+                                INNER JOIN {self.event_source_table_name} ON 
+                                {self.uploaded_events_table_name}.uuid={self.event_source_table_name}.uuid
+                                WHERE source = ?
+                                ORDER BY date DESC LIMIT 1""", (calendar_id,))
+        # Conversion to ISO format does not like the Z, that represents UTC aka no time zone
+        # so using +00:00 is an equivalent to it
+        date_string = res.fetchone()[0]
+        logger.debug(f"Last date found for calendar ID {calendar_id}: {date_string}")
+        return datetime.fromisoformat(date_string)
+    
+    def no_entries_with_source_id(self, calendar_id: str) -> bool:
+        res = self.select_all_rows_with_calendar_id(calendar_id)
+        return len(res.fetchall()) == 0
+    
+    def entry_already_in_cache(self, date:str, title:str, source_id:str) -> bool:
+        db_cursor = self.sql_db_connection.cursor()
+        res = db_cursor.execute(f"""SELECT {self.allColumns} FROM {self.uploaded_events_table_name}
+                                INNER JOIN {self.event_source_table_name} ON 
+                                {self.uploaded_events_table_name}.uuid={self.event_source_table_name}.uuid
+                                WHERE date = ? AND title = ? AND source = ?""", (date, title, source_id))
+        query = res.fetchall()
+        if len(query) > 0:
+            return True
+        return False
+
+    def insert_uploaded_event(self, row_to_add: UploadedEventRow, event_source: UploadSource):
+        db_cursor: sqlite3.Cursor = self.sql_db_connection.cursor()
+        insert_row = (row_to_add.uuid, row_to_add.id, row_to_add.title, row_to_add.date, row_to_add.groupID, row_to_add.groupName)
+        event_source_row= (event_source.uuid, event_source.websiteURL, event_source.source, event_source.sourceType)
+        
+        db_cursor.execute(f"INSERT INTO {self.uploaded_events_table_name} VALUES (?, ?, ?, ? , ?, ?)", insert_row)
+        db_cursor.execute(f"INSERT INTO {self.event_source_table_name} VALUES (?, ? , ?, ?)", event_source_row)
+        self.sql_db_connection.commit()
+    
+    def select_all_from_upload_table(self) -> sqlite3.Cursor:
+        db_cursor = self.sql_db_connection.cursor()
+        res = db_cursor.execute(f"SELECT * FROM {self.uploaded_events_table_name}")
+        return res
+
+    def select_all_from_event_source_table(self) -> sqlite3.Cursor:
+        db_cursor = self.sql_db_connection.cursor()
+        res = db_cursor.execute(f"SELECT * FROM {self.event_source_table_name}")
+        return res
+    
+        
+        
+    

calendar_event_engine-0.1.0/calendar_event_engine/filter.py

@@ -0,0 +1,28 @@
+import requests
+import validators
+
+from calendar_event_engine.types.generics import GenericEvent
+from calendar_event_engine.types.submission import EventsToUploadFromCalendarID
+
+
+# https://www.geeksforgeeks.org/http-headers-content-type/
+
+def normalize_generic_event(events_to_upload_list: list[EventsToUploadFromCalendarID]):
+    allowed_image_content_type_header = ["image/gif", "image/jpeg", "image/png"]
+    for events_to_upload in events_to_upload_list:
+        generic_event: GenericEvent
+        for generic_event in events_to_upload.events:
+            # If valid URL but not a picture, set to empty string
+            if validators.url(generic_event.picture):
+                picture_head = requests.head(generic_event.picture)
+                if picture_head.status_code != 200 and picture_head.headers.get("Content-type") not in allowed_image_content_type_header:
+                    generic_event.picture = ""
+
+            if not validators.url(generic_event.online_address):
+                generic_event.online_address = ""
+
+
+
+
+
+

calendar_event_engine-0.1.0/calendar_event_engine/logger.py

@@ -0,0 +1,10 @@
+import logging
+
+
+def create_logger_from_designated_logger(logger_name: str, codebase_logger_level: int=logging.INFO):
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+    logger = logging.getLogger("codebase_logger")
+    logger.setLevel(codebase_logger_level)
+    return logger.getChild(logger_name)
+
+

calendar_event_engine-0.1.0/calendar_event_engine/parser/package.py

@@ -0,0 +1,58 @@
+import json
+import urllib
+
+from calendar_event_engine.types.generics import GenericAddress, GenericEvent
+from calendar_event_engine.types.submission import ScraperTypes, GroupPackage, TimeInfo, GroupEventsKernel
+
+
+def none_if_not_present(x, dictionary):
+    return None if x not in dictionary else dictionary[x]
+
+
+def retrieve_source_type(source_type_string) -> ScraperTypes:
+    match source_type_string:
+        case "STATIC":
+            return ScraperTypes.STATIC
+        case "GOOGLE_CAL":
+            return ScraperTypes.GOOGLE_CAL
+        case "ICAL":
+            return ScraperTypes.ICAL
+        case "CUSTOM":
+            return ScraperTypes.CUSTOM
+    raise TypeError("Expected a string that resolves to a scraper type, instead got: " + source_type_string)
+
+
+def get_group_package(json_path: str) -> GroupPackage:
+    group_schema: dict = None
+    with urllib.request.urlopen(json_path) as f:
+        group_schema = json.load(f)
+
+    group_package: GroupPackage = GroupPackage({}, none_if_not_present("name", group_schema),
+                                               none_if_not_present("description", group_schema))
+
+    for group_name, group_info in group_schema["groupKernels"].items():
+
+        event_address = None if "defaultLocation" not in group_info else GenericAddress(**group_info["defaultLocation"])
+        event_kernel = GenericEvent(group_info["publisherInfo"], none_if_not_present("title", group_info),
+                                    none_if_not_present("beginsOn", group_info),
+                                    none_if_not_present("defaultDescription", group_info),
+                                    none_if_not_present("endsOn", group_info),
+                                    group_info["onlineAddress"], none_if_not_present("phoneAddress", group_info),
+                                    event_address)
+
+        calendar_ids = group_info["calendarIDs"]
+        scraper_type: ScraperTypes = retrieve_source_type(group_info["calendarType"])
+        time_info = None
+        if scraper_type == ScraperTypes.STATIC:
+            time_info = TimeInfo(group_info["defaultTimes"], group_info["endDate"])
+
+
+
+        if scraper_type not in group_package.scraper_type_and_kernels:
+            group_package.scraper_type_and_kernels[scraper_type] = []
+
+        group_package.scraper_type_and_kernels[scraper_type].append(
+            GroupEventsKernel(event_kernel, group_name, calendar_ids=calendar_ids,
+                              scraper_type=scraper_type, json_source_url=json_path, time_info=time_info))
+
+    return group_package

calendar_event_engine-0.1.0/calendar_event_engine/parser/submission.py

@@ -0,0 +1,51 @@
+import json
+import os
+import urllib
+
+import requests
+
+from calendar_event_engine.db_cache import SQLiteDB
+from calendar_event_engine.logger import create_logger_from_designated_logger
+from calendar_event_engine.parser.package import get_group_package
+from calendar_event_engine.publishers.abc_publisher import Publisher
+from calendar_event_engine.publishers.mobilizon.uploader import MobilizonUploader
+from calendar_event_engine.scrapers.abc_scraper import Scraper
+from calendar_event_engine.scrapers.google_calendar.scraper import GoogleCalendarScraper
+from calendar_event_engine.scrapers.ical.scraper import ICALScraper
+from calendar_event_engine.scrapers.statics.scraper import StaticScraper
+from calendar_event_engine.types.submission import PublisherTypes, ScraperTypes, GroupPackage
+from calendar_event_engine.types.submission_handlers import RunnerSubmission
+
+logger = create_logger_from_designated_logger(__name__)
+
+
+def get_runner_submission(remote_submission_url: str, test_mode: bool, cache_db: SQLiteDB) -> RunnerSubmission:
+    json_submission: dict = requests.get(remote_submission_url).json()
+
+    publisher_package: dict[Publisher, list[GroupPackage]] = dict()
+    respective_scrapers: dict[ScraperTypes, Scraper] = dict()
+    for publisher in json_submission.keys():
+        publisher_instance: Publisher
+        match publisher:
+            case PublisherTypes.MOBILIZON.value:
+                publisher_instance = MobilizonUploader(test_mode, cache_db)
+            case _:
+                raise TypeError("Expected publisher that is accepted, instead got: " + publisher)
+        publisher_package[publisher_instance] = []
+        for group_package_source_path in json_submission[publisher]:
+            group_package: GroupPackage = get_group_package(group_package_source_path)
+            publisher_package[publisher_instance].append(group_package)
+            for scraper_type in list(group_package.scraper_type_and_kernels.keys()):
+                if scraper_type not in respective_scrapers:
+                    logger.info(f"Creating scraper of type: {scraper_type}")
+                    match scraper_type:
+                        case ScraperTypes.GOOGLE_CAL:
+                            respective_scrapers[scraper_type] = GoogleCalendarScraper(cache_db)
+                        case ScraperTypes.STATIC:
+                            respective_scrapers[scraper_type] = StaticScraper()
+                        case ScraperTypes.ICAL:
+                            respective_scrapers[scraper_type] = ICALScraper(cache_db)
+
+    return RunnerSubmission(cache_db, publisher_package, test_mode, respective_scrapers)
+
+

calendar_event_engine-0.1.0/calendar_event_engine/publishers/abc_publisher.py

@@ -0,0 +1,60 @@
+
+from abc import ABC, abstractmethod
+
+from calendar_event_engine.types.submission import EventsToUploadFromCalendarID
+
+
+class Publisher(ABC):
+    """
+    The resulting website or remote destination where all scraped events will be stored.
+    """
+    @abstractmethod
+    def upload(self, events_to_upload: list[EventsToUploadFromCalendarID]) -> list[EventsToUploadFromCalendarID]:
+        """
+        Upload events to the publishers, and returns a list of successfully uploaded events.
+        :param events_to_upload:
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def update(self) -> None:
+        """
+        Update an event that has already been uploaded to the publisher.
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def connect(self) -> None:
+        """
+        Connect to the external publisher.
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """
+        Close the connection created with the publisher.
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def monitor(self) -> None:
+        """
+        Used to check already scraped events if they have changed, and update them if needed.
+        """
+        pass
+
+
+
+
+
+
+
+
+
+
+