calendar_event_engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calendar_event_engine/Runner.py +169 -0
- calendar_event_engine/__init__.py +0 -0
- calendar_event_engine/db_cache.py +147 -0
- calendar_event_engine/filter.py +28 -0
- calendar_event_engine/logger.py +10 -0
- calendar_event_engine/parser/__init__.py +0 -0
- calendar_event_engine/parser/package.py +58 -0
- calendar_event_engine/parser/submission.py +51 -0
- calendar_event_engine/publishers/__init__.py +0 -0
- calendar_event_engine/publishers/abc_publisher.py +60 -0
- calendar_event_engine/publishers/mobilizon/__init__.py +0 -0
- calendar_event_engine/publishers/mobilizon/api.py +123 -0
- calendar_event_engine/publishers/mobilizon/gql_requests.py +288 -0
- calendar_event_engine/publishers/mobilizon/types.py +147 -0
- calendar_event_engine/publishers/mobilizon/uploader.py +137 -0
- calendar_event_engine/scrapers/__init__.py +0 -0
- calendar_event_engine/scrapers/abc_scraper.py +30 -0
- calendar_event_engine/scrapers/google_calendar/__init__.py +0 -0
- calendar_event_engine/scrapers/google_calendar/api.py +178 -0
- calendar_event_engine/scrapers/google_calendar/scraper.py +73 -0
- calendar_event_engine/scrapers/ical/__init__.py +0 -0
- calendar_event_engine/scrapers/ical/scraper.py +111 -0
- calendar_event_engine/scrapers/statics/__init__.py +0 -0
- calendar_event_engine/scrapers/statics/scraper.py +75 -0
- calendar_event_engine/types/__init__.py +0 -0
- calendar_event_engine/types/custom_scraper.py +17 -0
- calendar_event_engine/types/generics.py +88 -0
- calendar_event_engine/types/submission.py +90 -0
- calendar_event_engine/types/submission_handlers.py +21 -0
- calendar_event_engine/utils/__init__.py +0 -0
- calendar_event_engine/utils/location.py +34 -0
- calendar_event_engine-0.1.0.dist-info/LICENSE +28 -0
- calendar_event_engine-0.1.0.dist-info/METADATA +57 -0
- calendar_event_engine-0.1.0.dist-info/RECORD +35 -0
- calendar_event_engine-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
import traceback
|
|
5
|
+
from urllib.error import HTTPError
|
|
6
|
+
|
|
7
|
+
from slack_sdk.webhook import WebhookClient
|
|
8
|
+
|
|
9
|
+
from calendar_event_engine.db_cache import SQLiteDB
|
|
10
|
+
from calendar_event_engine.filter import normalize_generic_event
|
|
11
|
+
from calendar_event_engine.logger import create_logger_from_designated_logger
|
|
12
|
+
from calendar_event_engine.parser.submission import get_runner_submission
|
|
13
|
+
from calendar_event_engine.publishers.abc_publisher import Publisher
|
|
14
|
+
from calendar_event_engine.scrapers.abc_scraper import Scraper
|
|
15
|
+
from calendar_event_engine.types.custom_scraper import CustomScraperJob
|
|
16
|
+
from calendar_event_engine.types.submission import GroupPackage, EventsToUploadFromCalendarID, GroupEventsKernel
|
|
17
|
+
from calendar_event_engine.types.submission_handlers import RunnerSubmission
|
|
18
|
+
from calendar_event_engine.scrapers.google_calendar.api import ExpiredToken
|
|
19
|
+
|
|
20
|
+
logger = create_logger_from_designated_logger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _runner(runner_submission: RunnerSubmission, custom_scrapers: dict[Publisher, list[CustomScraperJob]] = None):
|
|
24
|
+
continue_scraping = True
|
|
25
|
+
num_retries = 0
|
|
26
|
+
theres_an_expired_token = False
|
|
27
|
+
while continue_scraping and num_retries < 5:
|
|
28
|
+
try:
|
|
29
|
+
submitted_publishers: {Publisher: list[GroupPackage]} = runner_submission.publishers
|
|
30
|
+
for publisher in submitted_publishers.keys():
|
|
31
|
+
publisher: Publisher
|
|
32
|
+
publisher.connect()
|
|
33
|
+
group_package: GroupPackage
|
|
34
|
+
for group_package in submitted_publishers[publisher]:
|
|
35
|
+
logger.info(f"Reading Group Package: {group_package.package_name}")
|
|
36
|
+
|
|
37
|
+
for scraper_type in group_package.scraper_type_and_kernels.keys():
|
|
38
|
+
scraper: Scraper = runner_submission.respective_scrapers[scraper_type]
|
|
39
|
+
try:
|
|
40
|
+
scraper.connect_to_source()
|
|
41
|
+
group_event_kernels: list[GroupEventsKernel] = group_package.scraper_type_and_kernels[
|
|
42
|
+
scraper_type]
|
|
43
|
+
for event_kernel in group_event_kernels:
|
|
44
|
+
event_kernel: GroupEventsKernel
|
|
45
|
+
try:
|
|
46
|
+
events: list[EventsToUploadFromCalendarID] = scraper.retrieve_from_source(event_kernel)
|
|
47
|
+
except HTTPError as err:
|
|
48
|
+
if err.code == 404:
|
|
49
|
+
logger.warning(f"The following group is no longer available: {event_kernel.group_name}")
|
|
50
|
+
else:
|
|
51
|
+
raise
|
|
52
|
+
normalize_generic_event(events)
|
|
53
|
+
publisher.upload(events)
|
|
54
|
+
scraper.close_connection_to_source()
|
|
55
|
+
except ExpiredToken:
|
|
56
|
+
theres_an_expired_token = True
|
|
57
|
+
logger.warning("Expired token.json needs to be replaced. Will continue scraping other types")
|
|
58
|
+
continue
|
|
59
|
+
publisher.close()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if custom_scrapers:
|
|
63
|
+
for publisher in custom_scrapers.keys():
|
|
64
|
+
publisher.connect()
|
|
65
|
+
for scrap in custom_scrapers[publisher]:
|
|
66
|
+
try:
|
|
67
|
+
logger.info(f"Using custom scraper {scrap.scraper_name}")
|
|
68
|
+
scrap.custom_scraper.connect_to_source()
|
|
69
|
+
events = scrap.custom_scraper.retrieve_from_source(None)
|
|
70
|
+
normalize_generic_event(events)
|
|
71
|
+
publisher.upload(events)
|
|
72
|
+
except Exception as custom_err:
|
|
73
|
+
logger.error("Exception for custom scraper: " + scrap.scraper_name, custom_err)
|
|
74
|
+
publisher.close()
|
|
75
|
+
continue_scraping = False
|
|
76
|
+
except HTTPError as err:
|
|
77
|
+
if err.code == 500 and err.reason.lower() == 'Too many requests'.lower():
|
|
78
|
+
num_retries += 1
|
|
79
|
+
logger.warning("Going to sleep then retrying to scrape. Retry Num: " + num_retries)
|
|
80
|
+
time.sleep(120)
|
|
81
|
+
if theres_an_expired_token:
|
|
82
|
+
raise ExpiredToken
|
|
83
|
+
|
|
84
|
+
def _days_to_sleep(days):
|
|
85
|
+
now = datetime.datetime.now()
|
|
86
|
+
seconds_from_zero = (now.hour * 60 * 60)
|
|
87
|
+
time_at_2am = 2 * 60 * 60
|
|
88
|
+
time_to_sleep = time_at_2am
|
|
89
|
+
if seconds_from_zero > time_at_2am:
|
|
90
|
+
time_to_sleep = ((23 * 60 * 60) - seconds_from_zero) + time_at_2am
|
|
91
|
+
else:
|
|
92
|
+
time_to_sleep = time_at_2am - seconds_from_zero
|
|
93
|
+
|
|
94
|
+
return time_to_sleep + (60 * 60 * 24 * days)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _produce_slack_message(color, title, text, priority):
|
|
98
|
+
return {
|
|
99
|
+
"color": color,
|
|
100
|
+
"author_name": "CTEvent Scraper",
|
|
101
|
+
"author_icon": "https://ctgrassroots.org/favicon.ico",
|
|
102
|
+
"title": title,
|
|
103
|
+
"title_link": "google.com",
|
|
104
|
+
"text": text,
|
|
105
|
+
"fields": [
|
|
106
|
+
{
|
|
107
|
+
"title": "Priority",
|
|
108
|
+
"value": priority,
|
|
109
|
+
"short": "false"
|
|
110
|
+
}
|
|
111
|
+
],
|
|
112
|
+
"footer": "CTEvent Scraper",
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
def start_event_engine(remote_json_url: str, slack_webhook: WebhookClient = None, custom_scrapers: dict[Publisher, list[CustomScraperJob]] = None, test_mode: bool = False):
|
|
116
|
+
logger.info("Scraper Started")
|
|
117
|
+
sleeping = 2
|
|
118
|
+
while True:
|
|
119
|
+
#####################
|
|
120
|
+
# Create Submission #
|
|
121
|
+
#####################
|
|
122
|
+
|
|
123
|
+
cache_db: SQLiteDB = SQLiteDB(test_mode)
|
|
124
|
+
submission: RunnerSubmission = get_runner_submission(remote_json_url, test_mode, cache_db)
|
|
125
|
+
|
|
126
|
+
######################
|
|
127
|
+
# Execute Submission #
|
|
128
|
+
######################
|
|
129
|
+
time_to_sleep = _days_to_sleep(sleeping)
|
|
130
|
+
logger.info("Scraping")
|
|
131
|
+
try:
|
|
132
|
+
_runner(submission, custom_scrapers)
|
|
133
|
+
logger.info("Sleeping " + str(sleeping) + " Days Until Next Scrape")
|
|
134
|
+
except ExpiredToken:
|
|
135
|
+
time_to_sleep = _days_to_sleep(2)
|
|
136
|
+
logger.info("Sleeping " + str(sleeping) + " Days Until Next Scrape")
|
|
137
|
+
if slack_webhook is not None:
|
|
138
|
+
response = slack_webhook.send(attachments=[
|
|
139
|
+
_produce_slack_message("#e6e209", "Expired Token", "Replace token.json", "Medium")
|
|
140
|
+
])
|
|
141
|
+
|
|
142
|
+
except Exception as e:
|
|
143
|
+
logger.error("Unknown Error")
|
|
144
|
+
logger.error(e)
|
|
145
|
+
logger.error(traceback.format_exc())
|
|
146
|
+
logger.error("Going to Sleep for 7 days")
|
|
147
|
+
if slack_webhook is not None:
|
|
148
|
+
slack_webhook.send(attachments=[
|
|
149
|
+
_produce_slack_message("#ab1a13", "Event Scraper Unknown Error", "Check logs for error.", "High")
|
|
150
|
+
])
|
|
151
|
+
time_to_sleep = _days_to_sleep(7)
|
|
152
|
+
|
|
153
|
+
cache_db.close()
|
|
154
|
+
time.sleep(time_to_sleep)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
if __name__ == "__main__":
|
|
158
|
+
env_webhook = None if os.environ.get("SLACK_WEBHOOK") is None else WebhookClient(os.environ.get("SLACK_WEBHOOK"))
|
|
159
|
+
env_test_mode = False if "TEST_MODE" not in os.environ else True
|
|
160
|
+
submission_json_path = os.getenv("RUNNER_SUBMISSION_JSON_PATH")
|
|
161
|
+
if submission_json_path is None:
|
|
162
|
+
err = NotImplementedError("Did not find JSON submission path")
|
|
163
|
+
logger.error(err)
|
|
164
|
+
raise err
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
start_event_engine(submission_json_path, env_webhook, None, env_test_mode)
|
|
168
|
+
|
|
169
|
+
|
|
File without changes
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sqlite3
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from calendar_event_engine.logger import create_logger_from_designated_logger
|
|
6
|
+
|
|
7
|
+
logger = create_logger_from_designated_logger(__name__)
|
|
8
|
+
|
|
9
|
+
class UploadedEventRow:
|
|
10
|
+
uuid: str
|
|
11
|
+
id: str
|
|
12
|
+
title: str
|
|
13
|
+
date: str
|
|
14
|
+
groupID: str
|
|
15
|
+
groupName: str
|
|
16
|
+
|
|
17
|
+
def __init__(self, uuid: str, id: str, title: str, date: str, group_id: str, group_name: str):
|
|
18
|
+
"""_summary_
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
uuid (str): _description_
|
|
22
|
+
id (str): _description_
|
|
23
|
+
title (str): _description_
|
|
24
|
+
date (str): Has to be of format ISO8601 that is YYYY-MM-DD HH:MM:SS.SSS. Can also have T in the center if desired.
|
|
25
|
+
group_id (str): _description_
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
self.uuid = uuid
|
|
29
|
+
self.id = id
|
|
30
|
+
self.title = title
|
|
31
|
+
self.date = date
|
|
32
|
+
self.groupID = group_id
|
|
33
|
+
self.groupName = group_name
|
|
34
|
+
|
|
35
|
+
class UploadSource:
|
|
36
|
+
uuid: str
|
|
37
|
+
websiteURL: str
|
|
38
|
+
source: str
|
|
39
|
+
sourceType: str
|
|
40
|
+
|
|
41
|
+
def __init__(self, uuid:str, website_url: str, source:str, source_type: str):
|
|
42
|
+
self.uuid = uuid
|
|
43
|
+
self.websiteURL = website_url
|
|
44
|
+
self.source = source
|
|
45
|
+
self.sourceType = source_type
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class SQLiteDB:
|
|
50
|
+
sql_db_connection: sqlite3.Connection
|
|
51
|
+
uploaded_events_table_name = "uploaded_events"
|
|
52
|
+
event_source_table_name = "event_source"
|
|
53
|
+
|
|
54
|
+
allColumns = f"""{uploaded_events_table_name}.uuid, {uploaded_events_table_name}.id,
|
|
55
|
+
{uploaded_events_table_name}.title, {uploaded_events_table_name}.date,
|
|
56
|
+
{uploaded_events_table_name}.group_id, {uploaded_events_table_name}.group_name,
|
|
57
|
+
{event_source_table_name}.websiteURL, {event_source_table_name}.source, {event_source_table_name}.sourceType"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, in_memory_sq_lite: bool = False):
|
|
60
|
+
if in_memory_sq_lite:
|
|
61
|
+
self.sql_db_connection = sqlite3.connect(":memory:")
|
|
62
|
+
else:
|
|
63
|
+
cache_db_path = os.environ.get("CACHE_DB_PATH")
|
|
64
|
+
if cache_db_path is not None:
|
|
65
|
+
self.sql_db_connection = sqlite3.connect(cache_db_path + "/event_cache.db")
|
|
66
|
+
else:
|
|
67
|
+
self.sql_db_connection = sqlite3.connect("event_cache.db")
|
|
68
|
+
self.initialize_db()
|
|
69
|
+
|
|
70
|
+
def initialize_db(self) -> sqlite3.Connection:
|
|
71
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
72
|
+
db_cursor.execute(f"""CREATE TABLE IF NOT EXISTS {self.uploaded_events_table_name}
|
|
73
|
+
(uuid PRIMARY KEY, id, title text, date text, group_id, group_name)""")
|
|
74
|
+
db_cursor.execute(f"""CREATE TABLE IF NOT EXISTS {self.event_source_table_name}
|
|
75
|
+
(uuid, websiteURL text, source text, sourceType text,
|
|
76
|
+
FOREIGN KEY (uuid) REFERENCES uploaded_events(uuid) ON DELETE CASCADE)""" )
|
|
77
|
+
|
|
78
|
+
def close(self):
|
|
79
|
+
self.sql_db_connection.close()
|
|
80
|
+
|
|
81
|
+
# https://www.sqlite.org/lang_datefunc.html
|
|
82
|
+
# Uses built in date time function
|
|
83
|
+
def delete_all_month_old_events(self):
|
|
84
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
85
|
+
db_cursor.execute(f"DELETE FROM {self.uploaded_events_table_name} WHERE datetime(date) < datetime('now', '-1 month')")
|
|
86
|
+
|
|
87
|
+
self.sql_db_connection.commit()
|
|
88
|
+
|
|
89
|
+
def select_all_rows_with_calendar_id(self, source):
|
|
90
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
91
|
+
# Comma at the end of (groupID,) turns it into a tuple
|
|
92
|
+
res = db_cursor.execute(f"""SELECT {self.allColumns} FROM {self.uploaded_events_table_name}
|
|
93
|
+
INNER JOIN {self.event_source_table_name} ON
|
|
94
|
+
{self.uploaded_events_table_name}.uuid={self.event_source_table_name}.uuid
|
|
95
|
+
WHERE source = ?""", (source,))
|
|
96
|
+
return res
|
|
97
|
+
|
|
98
|
+
def get_last_event_date_for_source_id(self, calendar_id) -> datetime:
|
|
99
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
100
|
+
res = db_cursor.execute(f"""SELECT date FROM {self.uploaded_events_table_name}
|
|
101
|
+
INNER JOIN {self.event_source_table_name} ON
|
|
102
|
+
{self.uploaded_events_table_name}.uuid={self.event_source_table_name}.uuid
|
|
103
|
+
WHERE source = ?
|
|
104
|
+
ORDER BY date DESC LIMIT 1""", (calendar_id,))
|
|
105
|
+
# Conversion to ISO format does not like the Z, that represents UTC aka no time zone
|
|
106
|
+
# so using +00:00 is an equivalent to it
|
|
107
|
+
date_string = res.fetchone()[0]
|
|
108
|
+
logger.debug(f"Last date found for calendar ID {calendar_id}: {date_string}")
|
|
109
|
+
return datetime.fromisoformat(date_string)
|
|
110
|
+
|
|
111
|
+
def no_entries_with_source_id(self, calendar_id: str) -> bool:
|
|
112
|
+
res = self.select_all_rows_with_calendar_id(calendar_id)
|
|
113
|
+
return len(res.fetchall()) == 0
|
|
114
|
+
|
|
115
|
+
def entry_already_in_cache(self, date:str, title:str, source_id:str) -> bool:
|
|
116
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
117
|
+
res = db_cursor.execute(f"""SELECT {self.allColumns} FROM {self.uploaded_events_table_name}
|
|
118
|
+
INNER JOIN {self.event_source_table_name} ON
|
|
119
|
+
{self.uploaded_events_table_name}.uuid={self.event_source_table_name}.uuid
|
|
120
|
+
WHERE date = ? AND title = ? AND source = ?""", (date, title, source_id))
|
|
121
|
+
query = res.fetchall()
|
|
122
|
+
if len(query) > 0:
|
|
123
|
+
return True
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
def insert_uploaded_event(self, row_to_add: UploadedEventRow, event_source: UploadSource):
|
|
127
|
+
db_cursor: sqlite3.Cursor = self.sql_db_connection.cursor()
|
|
128
|
+
insert_row = (row_to_add.uuid, row_to_add.id, row_to_add.title, row_to_add.date, row_to_add.groupID, row_to_add.groupName)
|
|
129
|
+
event_source_row= (event_source.uuid, event_source.websiteURL, event_source.source, event_source.sourceType)
|
|
130
|
+
|
|
131
|
+
db_cursor.execute(f"INSERT INTO {self.uploaded_events_table_name} VALUES (?, ?, ?, ? , ?, ?)", insert_row)
|
|
132
|
+
db_cursor.execute(f"INSERT INTO {self.event_source_table_name} VALUES (?, ? , ?, ?)", event_source_row)
|
|
133
|
+
self.sql_db_connection.commit()
|
|
134
|
+
|
|
135
|
+
def select_all_from_upload_table(self) -> sqlite3.Cursor:
|
|
136
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
137
|
+
res = db_cursor.execute(f"SELECT * FROM {self.uploaded_events_table_name}")
|
|
138
|
+
return res
|
|
139
|
+
|
|
140
|
+
def select_all_from_event_source_table(self) -> sqlite3.Cursor:
|
|
141
|
+
db_cursor = self.sql_db_connection.cursor()
|
|
142
|
+
res = db_cursor.execute(f"SELECT * FROM {self.event_source_table_name}")
|
|
143
|
+
return res
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
import validators
|
|
3
|
+
|
|
4
|
+
from calendar_event_engine.types.generics import GenericEvent
|
|
5
|
+
from calendar_event_engine.types.submission import EventsToUploadFromCalendarID
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# https://www.geeksforgeeks.org/http-headers-content-type/
|
|
9
|
+
|
|
10
|
+
def normalize_generic_event(events_to_upload_list: list[EventsToUploadFromCalendarID]):
|
|
11
|
+
allowed_image_content_type_header = ["image/gif", "image/jpeg", "image/png"]
|
|
12
|
+
for events_to_upload in events_to_upload_list:
|
|
13
|
+
generic_event: GenericEvent
|
|
14
|
+
for generic_event in events_to_upload.events:
|
|
15
|
+
# If valid URL but not a picture, set to empty string
|
|
16
|
+
if validators.url(generic_event.picture):
|
|
17
|
+
picture_head = requests.head(generic_event.picture)
|
|
18
|
+
if picture_head.status_code != 200 and picture_head.headers.get("Content-type") not in allowed_image_content_type_header:
|
|
19
|
+
generic_event.picture = ""
|
|
20
|
+
|
|
21
|
+
if not validators.url(generic_event.online_address):
|
|
22
|
+
generic_event.online_address = ""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def create_logger_from_designated_logger(logger_name: str, codebase_logger_level: int=logging.INFO):
|
|
5
|
+
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
|
|
6
|
+
logger = logging.getLogger("codebase_logger")
|
|
7
|
+
logger.setLevel(codebase_logger_level)
|
|
8
|
+
return logger.getChild(logger_name)
|
|
9
|
+
|
|
10
|
+
|
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import urllib
|
|
3
|
+
|
|
4
|
+
from calendar_event_engine.types.generics import GenericAddress, GenericEvent
|
|
5
|
+
from calendar_event_engine.types.submission import ScraperTypes, GroupPackage, TimeInfo, GroupEventsKernel
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def none_if_not_present(x, dictionary):
|
|
9
|
+
return None if x not in dictionary else dictionary[x]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def retrieve_source_type(source_type_string) -> ScraperTypes:
|
|
13
|
+
match source_type_string:
|
|
14
|
+
case "STATIC":
|
|
15
|
+
return ScraperTypes.STATIC
|
|
16
|
+
case "GOOGLE_CAL":
|
|
17
|
+
return ScraperTypes.GOOGLE_CAL
|
|
18
|
+
case "ICAL":
|
|
19
|
+
return ScraperTypes.ICAL
|
|
20
|
+
case "CUSTOM":
|
|
21
|
+
return ScraperTypes.CUSTOM
|
|
22
|
+
raise TypeError("Expected a string that resolves to a scraper type, instead got: " + source_type_string)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_group_package(json_path: str) -> GroupPackage:
|
|
26
|
+
group_schema: dict = None
|
|
27
|
+
with urllib.request.urlopen(json_path) as f:
|
|
28
|
+
group_schema = json.load(f)
|
|
29
|
+
|
|
30
|
+
group_package: GroupPackage = GroupPackage({}, none_if_not_present("name", group_schema),
|
|
31
|
+
none_if_not_present("description", group_schema))
|
|
32
|
+
|
|
33
|
+
for group_name, group_info in group_schema["groupKernels"].items():
|
|
34
|
+
|
|
35
|
+
event_address = None if "defaultLocation" not in group_info else GenericAddress(**group_info["defaultLocation"])
|
|
36
|
+
event_kernel = GenericEvent(group_info["publisherInfo"], none_if_not_present("title", group_info),
|
|
37
|
+
none_if_not_present("beginsOn", group_info),
|
|
38
|
+
none_if_not_present("defaultDescription", group_info),
|
|
39
|
+
none_if_not_present("endsOn", group_info),
|
|
40
|
+
group_info["onlineAddress"], none_if_not_present("phoneAddress", group_info),
|
|
41
|
+
event_address)
|
|
42
|
+
|
|
43
|
+
calendar_ids = group_info["calendarIDs"]
|
|
44
|
+
scraper_type: ScraperTypes = retrieve_source_type(group_info["calendarType"])
|
|
45
|
+
time_info = None
|
|
46
|
+
if scraper_type == ScraperTypes.STATIC:
|
|
47
|
+
time_info = TimeInfo(group_info["defaultTimes"], group_info["endDate"])
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
if scraper_type not in group_package.scraper_type_and_kernels:
|
|
52
|
+
group_package.scraper_type_and_kernels[scraper_type] = []
|
|
53
|
+
|
|
54
|
+
group_package.scraper_type_and_kernels[scraper_type].append(
|
|
55
|
+
GroupEventsKernel(event_kernel, group_name, calendar_ids=calendar_ids,
|
|
56
|
+
scraper_type=scraper_type, json_source_url=json_path, time_info=time_info))
|
|
57
|
+
|
|
58
|
+
return group_package
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import urllib
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from calendar_event_engine.db_cache import SQLiteDB
|
|
8
|
+
from calendar_event_engine.logger import create_logger_from_designated_logger
|
|
9
|
+
from calendar_event_engine.parser.package import get_group_package
|
|
10
|
+
from calendar_event_engine.publishers.abc_publisher import Publisher
|
|
11
|
+
from calendar_event_engine.publishers.mobilizon.uploader import MobilizonUploader
|
|
12
|
+
from calendar_event_engine.scrapers.abc_scraper import Scraper
|
|
13
|
+
from calendar_event_engine.scrapers.google_calendar.scraper import GoogleCalendarScraper
|
|
14
|
+
from calendar_event_engine.scrapers.ical.scraper import ICALScraper
|
|
15
|
+
from calendar_event_engine.scrapers.statics.scraper import StaticScraper
|
|
16
|
+
from calendar_event_engine.types.submission import PublisherTypes, ScraperTypes, GroupPackage
|
|
17
|
+
from calendar_event_engine.types.submission_handlers import RunnerSubmission
|
|
18
|
+
|
|
19
|
+
logger = create_logger_from_designated_logger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_runner_submission(remote_submission_url: str, test_mode: bool, cache_db: SQLiteDB) -> RunnerSubmission:
|
|
23
|
+
json_submission: dict = requests.get(remote_submission_url).json()
|
|
24
|
+
|
|
25
|
+
publisher_package: dict[Publisher, list[GroupPackage]] = dict()
|
|
26
|
+
respective_scrapers: dict[ScraperTypes, Scraper] = dict()
|
|
27
|
+
for publisher in json_submission.keys():
|
|
28
|
+
publisher_instance: Publisher
|
|
29
|
+
match publisher:
|
|
30
|
+
case PublisherTypes.MOBILIZON.value:
|
|
31
|
+
publisher_instance = MobilizonUploader(test_mode, cache_db)
|
|
32
|
+
case _:
|
|
33
|
+
raise TypeError("Expected publisher that is accepted, instead got: " + publisher)
|
|
34
|
+
publisher_package[publisher_instance] = []
|
|
35
|
+
for group_package_source_path in json_submission[publisher]:
|
|
36
|
+
group_package: GroupPackage = get_group_package(group_package_source_path)
|
|
37
|
+
publisher_package[publisher_instance].append(group_package)
|
|
38
|
+
for scraper_type in list(group_package.scraper_type_and_kernels.keys()):
|
|
39
|
+
if scraper_type not in respective_scrapers:
|
|
40
|
+
logger.info(f"Creating scraper of type: {scraper_type}")
|
|
41
|
+
match scraper_type:
|
|
42
|
+
case ScraperTypes.GOOGLE_CAL:
|
|
43
|
+
respective_scrapers[scraper_type] = GoogleCalendarScraper(cache_db)
|
|
44
|
+
case ScraperTypes.STATIC:
|
|
45
|
+
respective_scrapers[scraper_type] = StaticScraper()
|
|
46
|
+
case ScraperTypes.ICAL:
|
|
47
|
+
respective_scrapers[scraper_type] = ICALScraper(cache_db)
|
|
48
|
+
|
|
49
|
+
return RunnerSubmission(cache_db, publisher_package, test_mode, respective_scrapers)
|
|
50
|
+
|
|
51
|
+
|
|
File without changes
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
|
|
4
|
+
from calendar_event_engine.types.submission import EventsToUploadFromCalendarID
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Publisher(ABC):
|
|
8
|
+
"""
|
|
9
|
+
The resulting website or remote destination where all scraped events will be stored.
|
|
10
|
+
"""
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def upload(self, events_to_upload: list[EventsToUploadFromCalendarID]) -> list[EventsToUploadFromCalendarID]:
|
|
13
|
+
"""
|
|
14
|
+
Upload events to the publishers, and returns a list of successfully uploaded events.
|
|
15
|
+
:param events_to_upload:
|
|
16
|
+
:return:
|
|
17
|
+
"""
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def update(self) -> None:
|
|
22
|
+
"""
|
|
23
|
+
Update an event that has already been uploaded to the publisher.
|
|
24
|
+
:return:
|
|
25
|
+
"""
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def connect(self) -> None:
|
|
30
|
+
"""
|
|
31
|
+
Connect to the external publisher.
|
|
32
|
+
:return:
|
|
33
|
+
"""
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
def close(self) -> None:
|
|
38
|
+
"""
|
|
39
|
+
Close the connection created with the publisher.
|
|
40
|
+
:return:
|
|
41
|
+
"""
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
@abstractmethod
|
|
45
|
+
def monitor(self) -> None:
|
|
46
|
+
"""
|
|
47
|
+
Used to check already scraped events if they have changed, and update them if needed.
|
|
48
|
+
"""
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
File without changes
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
from gql import Client
|
|
5
|
+
from gql.transport.aiohttp import AIOHTTPTransport
|
|
6
|
+
from gql.transport.requests import RequestsHTTPTransport
|
|
7
|
+
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
|
8
|
+
|
|
9
|
+
import aiohttp
|
|
10
|
+
from calendar_event_engine.logger import create_logger_from_designated_logger
|
|
11
|
+
from calendar_event_engine.publishers.mobilizon.gql_requests import EventGQL, AuthenticationGQL, ActorsGQL
|
|
12
|
+
from calendar_event_engine.publishers.mobilizon.types import MobilizonEvent, Actor
|
|
13
|
+
|
|
14
|
+
logger = create_logger_from_designated_logger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class retry_if_not_exception_type(retry_if_exception):
|
|
18
|
+
"""Retries except an exception has been raised of one or more types."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, exception_types=Exception):
|
|
21
|
+
self.exception_types = exception_types
|
|
22
|
+
super(retry_if_not_exception_type, self).__init__(
|
|
23
|
+
lambda e: not isinstance(e, exception_types))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _MobilizonClient:
|
|
29
|
+
class LoginTokens:
|
|
30
|
+
accessToken: str
|
|
31
|
+
refreshToken: str
|
|
32
|
+
|
|
33
|
+
def __init__(self, access_token: str, refresh_token: str):
|
|
34
|
+
self.accessToken = access_token
|
|
35
|
+
self.refreshToken = refresh_token
|
|
36
|
+
|
|
37
|
+
loginTokens: LoginTokens = None
|
|
38
|
+
client: Client = None
|
|
39
|
+
endpoint: str = None
|
|
40
|
+
headers: dict = None
|
|
41
|
+
|
|
42
|
+
def __init__(self, endpoint: str, email: str, password: str):
|
|
43
|
+
self.endpoint = endpoint
|
|
44
|
+
self.client = self._build_client()
|
|
45
|
+
data = self.publish(AuthenticationGQL.loginGQL(email, password))
|
|
46
|
+
login = data['login']
|
|
47
|
+
self.loginTokens = self.LoginTokens(login['accessToken'], login['refreshToken'])
|
|
48
|
+
self.client = self._build_client(self.loginTokens.accessToken)
|
|
49
|
+
|
|
50
|
+
def _build_client(self,bearer=None):
|
|
51
|
+
self.headers = dict()
|
|
52
|
+
if bearer is not None:
|
|
53
|
+
self.headers['Authorization'] = 'Bearer ' + bearer
|
|
54
|
+
|
|
55
|
+
transport = RequestsHTTPTransport(
|
|
56
|
+
url=self.endpoint,
|
|
57
|
+
headers=self.headers,
|
|
58
|
+
verify=True,
|
|
59
|
+
# retries=3,
|
|
60
|
+
)
|
|
61
|
+
return Client(transport=transport, fetch_schema_from_transport=True)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def refresh_token(self, refresh_token: str):
|
|
65
|
+
return self.publish(AuthenticationGQL.refreshTokenGQL(refresh_token))
|
|
66
|
+
|
|
67
|
+
def log_out(self):
|
|
68
|
+
return self.publish(AuthenticationGQL.logoutGQL(f'"{self.loginTokens.refreshToken}"')) # void
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# attempts at 0s, 2s, 4s, 8s
|
|
72
|
+
@retry(reraise=True, stop=stop_after_attempt(1),
|
|
73
|
+
wait=wait_exponential(multiplier=2))
|
|
74
|
+
def publish(self, query):
|
|
75
|
+
response = self.client.execute(query)
|
|
76
|
+
return response
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class MobilizonAPI:
|
|
80
|
+
_mobilizon_client: _MobilizonClient
|
|
81
|
+
bot_actor: Actor
|
|
82
|
+
|
|
83
|
+
def __init__(self, endpoint: str, email: str, password: str):
|
|
84
|
+
self._mobilizon_client = _MobilizonClient(endpoint, f'"{email}"', f'"{password}"')
|
|
85
|
+
self.bot_actor = Actor(**self.getActors()["identities"][0])
|
|
86
|
+
logger.info("Logged In Mobilizon")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def bot_created_event(self, event_type: MobilizonEvent) -> dict:
|
|
90
|
+
event_type.organizerActorId = self.bot_actor.id
|
|
91
|
+
|
|
92
|
+
event_id = self._mobilizon_client.publish(EventGQL.createEventGQL(event_type))
|
|
93
|
+
return {"id": event_id["createEvent"]["id"], "uuid": event_id["createEvent"]["uuid"]}
|
|
94
|
+
|
|
95
|
+
def upload_file(self, file_path: str) -> str:
|
|
96
|
+
with requests.get(file_path) as file:
|
|
97
|
+
variables = {"name": "automatically-uploaded-event-bot", "file": "uploaded-file", "actorID": self.bot_actor.id}
|
|
98
|
+
response = requests.post(self._mobilizon_client.endpoint,
|
|
99
|
+
data={
|
|
100
|
+
"query": EventGQL.uploadMediaRawGQL(),
|
|
101
|
+
"variables": json.dumps(variables)
|
|
102
|
+
},
|
|
103
|
+
files= {
|
|
104
|
+
"uploaded-file": file.content
|
|
105
|
+
},
|
|
106
|
+
headers={'Authorization': f'Bearer {self._mobilizon_client.loginTokens.accessToken}',
|
|
107
|
+
'accept': 'application/json'}
|
|
108
|
+
)
|
|
109
|
+
content = json.loads(response.content)
|
|
110
|
+
if content is None or response.status_code != 200 or "errors" in content:
|
|
111
|
+
return ""
|
|
112
|
+
return content["data"]["uploadMedia"]["id"]
|
|
113
|
+
|
|
114
|
+
def logout(self):
|
|
115
|
+
self._mobilizon_client.log_out()
|
|
116
|
+
logger.info("Logged Out")
|
|
117
|
+
|
|
118
|
+
def getActors(self):
|
|
119
|
+
return self._mobilizon_client.publish(ActorsGQL.getIdentities())
|
|
120
|
+
|
|
121
|
+
def getGroups(self):
|
|
122
|
+
return self._mobilizon_client.publish(ActorsGQL.getGroups('"eventbot"'))
|
|
123
|
+
|