email-to-calendar 20250826010803.dev0__py3-none-any.whl → 20251125165904.dev0__py3-none-any.whl

__init__.py

@@ -1,16 +1,19 @@
 import os
 import sys
 import logging
+from pathlib import Path
 
-if not os.path.isdir("../data"):
-    os.makedirs("../data")
+from src.util.env import get_settings
 
-db_file = os.environ.get("DB_FILE", "../data/emails.db")
+settings = get_settings()
 
-if not os.path.isdir("../logs"):
-    os.makedirs("../logs")
+db_file = Path(settings.DB_FILE)
 
-log_file = os.path.join(os.path.dirname(__file__), "../logs/emails.log")
+db_file.parent.mkdir(parents=True, exist_ok=True)
+
+log_file_path = Path(os.path.dirname(__file__), "../logs/emails.log")
+log_file_path.parent.mkdir(parents=True, exist_ok=True)
+log_file_path.touch(exist_ok=True)
 
 
 class StdoutFilter(logging.Filter):
@@ -43,7 +46,7 @@ stderr_handler.addFilter(StderrFilter())
 stderr_handler.setFormatter(logging.Formatter(log_format))
 
 # Handler for file (all levels)
-file_handler = logging.FileHandler(log_file)
+file_handler = logging.FileHandler(log_file_path)
 file_handler.setLevel(logging.DEBUG)
 file_handler.setFormatter(logging.Formatter(log_format))
 
@@ -52,8 +55,3 @@ logger.handlers.clear()
 logger.addHandler(stdout_handler)
 logger.addHandler(stderr_handler)
 logger.addHandler(file_handler)
-
-# Usage: from src import logger
-# logger.info("Info message")
-# logger.warning("Warning message")
-# logger.error("Error message")

db.py

@@ -1,17 +1,16 @@
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, declarative_base
+from sqlmodel import create_engine, Session
 
-from src import db_file
+from src.util.env import get_settings
 
-DATABASE_URL = f"sqlite:///{db_file}"
+settings = get_settings()
 
-engine = create_engine(DATABASE_URL, echo=False)
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-Base = declarative_base()
+DATABASE_URL = f"sqlite:///{settings.DB_FILE}"
+
+engine = create_engine(DATABASE_URL)
 
 
 def get_db():
-    db = SessionLocal()
+    db = Session(engine)
     try:
         yield db
     finally:

email_to_calendar-20251125165904.dev0.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: email-to-calendar
+Version: 20251125165904.dev0
+Summary: Takes emails from an IMAP server, parses the body, and creates event(s) in a CalDAV calendar
+Requires-Python: <4.0,>=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: apprise==1.9.5
+Requires-Dist: caldav==2.1.2
+Requires-Dist: imapclient==3.0.1
+Requires-Dist: pydantic-ai-slim[openai]==1.22.0
+Requires-Dist: pydantic-settings==2.12.0
+Requires-Dist: pydantic[email]==2.12.4
+Requires-Dist: sqlmodel==0.0.27
+Requires-Dist: tzlocal==5.3.1
+Dynamic: license-file
+
+# E-Mail to calendar Converter
+The point of this application is to search an IMAP account, look for emails based on certain criteria(s), and parse 
+the content, using regex, and automatically create calendar events in an iCal account.
+
+## TO-DO
+- [X] Get e-mails, and save ID to sqlite db to avoid duplicates
+- [X] Save calendar events to sqlite db to avoid duplicates
+- [X] Add config to backfill (check all emails from an optional certain date), or use most recent email
+  - [X] If using most recent, when new email arrives, remove events not present, and add new ones
+- [ ] If new email comes in with updated events, update event in calendar instead of creating a new one
+- [ ] Using email summary check for words like `Cancelled`, etc. to delete events
+- [ ] If event already exists, check if details have changed, and update if necessary
+- [ ] Investigate IMAP IDLE (push instead of poll)
+- [X] Make sure all day events are handled correctly
+- [ ] Add Docker Model Runner support
+- [ ] Add 'validate' function for events, and if it fails, have AI re-process that event
+
+
+## Environment Variables
+| Name | Description | Type | Default Value | Allowed Values |
+|------|-------------|------|---------------|----------------|
+|      |             |      |               |                |
+|      |             |      |               |                |
+|      |             |      |               |                |

email_to_calendar-20251125165904.dev0.dist-info/RECORD

@@ -0,0 +1,20 @@
+__init__.py,sha256=iI9fqY9bavUE4S2CxDks8-Q7OBGKl96xQVbYRmAm36E,1572
+db.py,sha256=apQrwaEp0ncFNn5WIoDi7ndMe4Q-VYRnXvCJT_iV0O0,295
+main.py,sha256=CZxXR163RvFC_7C8CKK39JDEOQJlpobK1Mi7y-fUeR8,6308
+email_to_calendar-20251125165904.dev0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+events/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+events/caldav.py,sha256=Tf-v_yuHr6SBzQ4f-AOdKGG_eoIVuedZ5AKCv-hKWG8,1713
+mail/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mail/mail.py,sha256=Kq50o2kpkovvfNcSnDQZTnOZ6TV7ZSR8KOIRvdXkGao,5825
+mail/mail_idle.py,sha256=o_lOVO9VPm88si3qvEQhttSH2N6cRZF7AWV2kU8uXDw,6860
+model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+model/email.py,sha256=GRpqoOQikfZAWofPOYpHGVIbkNVr3_XahMsorwsagis,3647
+model/event.py,sha256=WQPVKPAuebALqalZYHvl1sAmQ3BPnu6XDDmpvrgNTWM,3917
+util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+util/ai.py,sha256=Px1OU2VLbnXr9Fk7VHj66b0_-hyvrGKsInAmZigwIag,7367
+util/env.py,sha256=gTgydkmnEgOYvltMqt0hlYWErVf08ZUssbhHzA0GCiE,3780
+util/notifications.py,sha256=WKSgBfVeDOg1tqIEOwUgMgb10qxrzjsDvh5bmYi9-Hw,640
+email_to_calendar-20251125165904.dev0.dist-info/METADATA,sha256=y5QEaKOKPcheZdUjljcyyHrgQMarhSK1lyT_ZGAGC8I,1989
+email_to_calendar-20251125165904.dev0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+email_to_calendar-20251125165904.dev0.dist-info/top_level.txt,sha256=635ZTCyzc_2xtdJz_2Q50A96mBoIUrfORy5SdfGYabo,40
+email_to_calendar-20251125165904.dev0.dist-info/RECORD,,

events/caldav.py

@@ -1,15 +1,22 @@
 from caldav import DAVClient, Calendar
+from pydantic import AnyUrl
+
 from src import logger
 
 from src.model.event import Event
 
 
-def authenticate_caldav(url: str, username: str, password: str) -> DAVClient:
-    return DAVClient(url, username=username, password=password)
+def authenticate_caldav(url: AnyUrl, username: str, password: str) -> DAVClient:
+    return DAVClient(
+        url.encoded_string(),
+        username=username,
+        password=password,
+        headers={"User-Agent": "email-to-calendar/1.0"},
+    )
 
 
 def add_to_caldav(
-    url: str, username: str, password: str, calendar_name: str, events: list[Event]
+    url: AnyUrl, username: str, password: str, calendar_name: str, events: list[Event]
 ):
     with authenticate_caldav(url, username, password) as client:
         principal = client.principal()
@@ -30,6 +37,9 @@ def add_to_caldav(
                 logger.info(
                     f"Adding event {event.summary} to CalDAV calendar '{calendar_name}'"
                 )
+                if event.all_day:
+                    event.start = event.start.date()
+                    event.end = event.end.date()
                 calendar.add_event(
                     dtstart=event.start, dtend=event.end, summary=event.summary
                 )

mail/mail_idle.py

@@ -0,0 +1,187 @@
+import time
+import re
+import email
+from email import policy
+from imapclient import IMAPClient
+
+
+def idle_print_emails(
+    host,
+    port,
+    username,
+    password,
+    folder="INBOX",
+    idle_timeout=30,
+    payload_timeout=30,
+    retry_interval=1,
+):
+    """
+    Connects to IMAP server, enters IDLE loop and prints new messages to console.
+    Only prints a message once its payload is actually available. Retries fetching
+    the payload for up to `payload_timeout` seconds (checking every `retry_interval` seconds).
+    """
+
+    processed_uids = set()
+
+    def extract_text(msg):
+        if msg.is_multipart():
+            for part in msg.walk():
+                if (
+                    part.get_content_type() == "text/plain"
+                    and part.get_content_disposition() != "attachment"
+                ):
+                    return part.get_content()
+            for part in msg.walk():
+                if (
+                    part.get_content_type() == "text/html"
+                    and part.get_content_disposition() != "attachment"
+                ):
+                    return part.get_content()
+            return ""
+        else:
+            return msg.get_content()
+
+    def _raw_from_fetch_entry(data):
+        # fetch response keys vary; try common ones
+        for key in (
+            b"BODY.PEEK[]",
+            "BODY.PEEK[]",
+            b"BODY[]",
+            "BODY[]",
+            b"RFC822",
+            "RFC822",
+        ):
+            raw = data.get(key)
+            if raw:
+                return raw
+        return None
+
+    def fetch_payload_with_retry(client, uid):
+        start = time.time()
+        while time.time() - start < payload_timeout:
+            try:
+                fetch_data = client.fetch([uid], ["BODY.PEEK[]", "FLAGS"])
+            except Exception:
+                # transient fetch error, wait and retry
+                time.sleep(retry_interval)
+                continue
+
+            entry = fetch_data.get(uid) or fetch_data.get(int(uid)) or {}
+            raw = _raw_from_fetch_entry(entry)
+            if raw:
+                # normalize
+                if isinstance(raw, tuple):
+                    raw = raw[1]
+                if isinstance(raw, str):
+                    raw = raw.encode()
+                return raw
+            # not yet present, wait and retry
+            time.sleep(retry_interval)
+        return None
+
+    def process_uids(client, uids):
+        for uid in uids:
+            if uid in processed_uids:
+                continue
+            raw = fetch_payload_with_retry(client, uid)
+            if not raw:
+                # payload never became available within timeout; skip printing
+                print(
+                    f"UID {uid}: payload not available after {payload_timeout}s, skipping."
+                )
+                continue
+            try:
+                msg = email.message_from_bytes(raw, policy=policy.default)
+            except Exception as e:
+                print(f"UID {uid}: failed to parse message bytes:", e)
+                processed_uids.add(uid)
+                continue
+
+            subject = msg.get("Subject", "(no subject)")
+            _from = msg.get("From", "(no from)")
+            _to = msg.get("To", "(no to)")
+            date = msg.get("Date", "(no date)")
+            body = extract_text(msg) or "(no body)"
+
+            print("----- MESSAGE START -----")
+            print("UID:", uid)
+            print("From:", _from)
+            print("To:", _to)
+            print("Date:", date)
+            print("Subject:", subject)
+            print("Body:\n", body)
+            print("----- MESSAGE END -------")
+
+            processed_uids.add(uid)
+
+    with IMAPClient(host, port, use_uid=True, ssl=True) as client:
+        client.login(username, password)
+        client.select_folder(folder)
+        print("Connected and monitoring folder:", folder)
+
+        try:
+            while True:
+                print("Entering IDLE...")
+                started_idle = False
+                try:
+                    client.idle()
+                    started_idle = True
+                except Exception as e:
+                    # servers can send unsolicited FETCH while entering IDLE; extract UIDs and process
+                    msg = str(e)
+                    print("IDLE start error (handled):", msg)
+                    found = re.findall(r"UID (\d+)", msg)
+                    if found:
+                        uids = [int(x) for x in found]
+                        print("Processing UIDs from unsolicited response:", uids)
+                        process_uids(client, uids)
+
+                responses = []
+                if started_idle:
+                    try:
+                        responses = client.idle_check(timeout=idle_timeout)
+                    finally:
+                        try:
+                            client.idle_done()
+                        except Exception as e:
+                            print("Failed to exit IDLE:", e)
+
+                if responses:
+                    print("Server notifications:", responses)
+                    uids_from_responses = []
+                    for resp in responses:
+                        try:
+                            if (
+                                isinstance(resp, tuple)
+                                and len(resp) >= 3
+                                and resp[1] in (b"FETCH", "FETCH")
+                            ):
+                                nested = resp[2]
+                                if isinstance(nested, tuple):
+                                    for i in range(len(nested)):
+                                        if nested[i] == b"UID" or nested[i] == "UID":
+                                            maybe_uid = nested[i + 1]
+                                            uids_from_responses.append(int(maybe_uid))
+                        except Exception:
+                            continue
+
+                    if uids_from_responses:
+                        process_uids(client, uids_from_responses)
+                    else:
+                        # fallback: search for UNSEEN messages
+                        try:
+                            uids = client.search(["UNSEEN"])
+                            if not uids:
+                                print("No unseen messages found.")
+                            else:
+                                print(f"Found {len(uids)} unseen message(s): {uids}")
+                                process_uids(client, uids)
+                        except Exception as e:
+                            print("Error fetching or parsing messages:", e)
+
+                time.sleep(1)
+        except KeyboardInterrupt:
+            print("Interrupted by user, closing connection.")
+        except Exception as e:
+            print("Error in IDLE loop:", e)
+            raise

main.py

@@ -1,45 +1,133 @@
+import asyncio
+import datetime
 import os
+import time
 
 from sqlalchemy import inspect
+from sqlmodel import SQLModel
 
 from src import db_file, logger
 from src.events.caldav import add_to_caldav
-from src.model.event import Event
-from src.util import text
 from src.mail import mail
-from src.db import Base, engine, SessionLocal
+from src.db import Session, engine
 from src.model.email import EMail
+from src.model.event import Event
+from src.util.ai import parse_email
+from src.util.env import get_settings, Settings
+from src.util.notifications import send_success_notification
+
+
+async def populate_events(settings: Settings):
+    backfill = settings.BACKFILL
+    provider = settings.AI_PROVIDER
+    model = settings.AI_MODEL
+    ollama_host = settings.OLLAMA_HOST
+    ollama_port = settings.OLLAMA_PORT
+    ollama_secure = settings.OLLAMA_SECURE
+    open_ai_api_key = settings.OPEN_AI_API_KEY
+    max_retries = settings.AI_MAX_RETRIES
+    system_prompt = settings.AI_SYSTEM_PROMPT
+
+    if backfill:
+        logger.info("Backfilling events from all emails without events")
+        for email in EMail.get_without_events():
+            events = await parse_email(
+                email,
+                provider,
+                model,
+                ollama_host,
+                ollama_port,
+                ollama_secure,
+                open_ai_api_key,
+                max_retries,
+                system_prompt,
+            )
+            for event in events:
+                logger.debug(f"Backfilling event: {event}")
+                try:
+                    event.save()
+                except Exception as e:
+                    logger.error(f"Error saving event {event}: {e}", exc_info=True)
+                    for e in events:
+                        e.delete()
+        logger.info("Backfilled events from all emails")
+    else:
+        most_recent_email = EMail.get_most_recent_without_events()
+        if most_recent_email:
+            logger.info("Parsing most recent email with id %s", most_recent_email.id)
+            events = await parse_email(
+                most_recent_email,
+                provider,
+                model,
+                ollama_host,
+                ollama_port,
+                ollama_secure,
+                open_ai_api_key,
+                max_retries,
+                system_prompt,
+            )
+            for event in events:
+                logger.debug(f"Saving event: {event}")
+                try:
+                    event.save()
+                except Exception as e:
+                    logger.error(f"Error saving event {event}: {e}", exc_info=True)
+                    for e in events:
+                        e.delete()
+            logger.info(
+                "Parsed and saved events from most recent email with date %s",
+                most_recent_email.delivery_date,
+            )
+        else:
+            logger.info("No new emails without events to parse")
+
+    events = Event.get_not_in_calendar()
+
+    caldav_url = settings.CALDAV_URL
+    caldav_username = settings.CALDAV_USERNAME
+    caldav_password = settings.CALDAV_PASSWORD
+    calendar_name = settings.CALDAV_CALENDAR
+
+    if events:
+        logger.info("Adding %d new events to CalDAV calendar", len(events))
+        add_to_caldav(
+            caldav_url, caldav_username, caldav_password, calendar_name, events
+        )
+        if settings.APPRISE_URL:
+            send_success_notification(settings.APPRISE_URL, events)
 
 
 def main():
     logger.info("Starting email retrieval process")
+    settings = get_settings()
 
     # Create tables if they don't exist
-    Base.metadata.create_all(bind=engine)
+    SQLModel.metadata.create_all(engine)
 
-    imap_host = os.environ["IMAP_HOST"]
-    imap_port = int(os.environ["IMAP_PORT"])
-    imap_username = os.environ["IMAP_USERNAME"]
-    imap_password = os.environ["IMAP_PASSWORD"]
+    imap_host = settings.IMAP_HOST
+    imap_port = settings.IMAP_PORT
+    imap_username = settings.IMAP_USERNAME
+    imap_password = settings.IMAP_PASSWORD
+    mailbox = settings.IMAP_MAILBOX
+    ssl = settings.IMAP_SSL
 
-    from_email = os.environ["FILTER_FROM_EMAIL"]
-    subject = os.environ["FILTER_SUBJECT"]
-    backfill: bool = os.environ.get("BACKFILL", "false").lower() == "true"
+    from_email = settings.FILTER_FROM_EMAIL
+    subject = settings.FILTER_SUBJECT
 
     db_path = os.path.join(os.path.dirname(__file__), db_file)
     db_exists = os.path.exists(db_path)
     inspector = inspect(engine)
-    table_exists = inspector.has_table("emails")
+    table_exists = inspector.has_table("email")
     has_record = False
     if db_exists and table_exists:
         logger.info("Database and table exist, checking for records")
-        session = SessionLocal()
+        session = Session(engine)
         try:
             has_record = len(EMail.get_all()) > 0
         finally:
             session.close()
 
-    client = mail.authenticate(imap_host, imap_port, imap_username, imap_password)
+    client = mail.authenticate(imap_host, imap_port, imap_username, imap_password, ssl)
 
     try:
         if has_record:
@@ -55,6 +143,7 @@ def main():
                 from_email=from_email,
                 subject=subject,
                 since=most_recent_email.delivery_date,
+                mailbox=mailbox,
             )
         else:
             logger.info("No existing records found, retrieving all emails")
@@ -67,31 +156,23 @@ def main():
         for email in emails:
             email.save()
 
-        if backfill:
-            for email in EMail.get_all():
-                events: list[Event] = text.parse_email_events(email)
-                for event_obj in events:
-                    event_obj.save()
-            logger.info("Backfilled events from all emails")
-        else:
-            most_recent_email = EMail.get_most_recent()
-            events: list[Event] = text.parse_email_events(most_recent_email)
-            for event_obj in events:
-                event_obj.save()
-            logger.info(
-                "Parsed and saved events from most recent email with date %s",
-                most_recent_email.delivery_date,
-            )
-        events = Event.get_all()
-
-        caldav_url = os.environ["CALDAV_URL"]
-        caldav_username = os.environ["CALDAV_USERNAME"]
-        caldav_password = os.environ["CALDAV_PASSWORD"]
-        calendar_name = os.environ["CALDAV_CALENDAR"]
-
-        add_to_caldav(
-            caldav_url, caldav_username, caldav_password, calendar_name, events
-        )
+        while True:
+            logger.info("Starting event population process...")
+            start_time = datetime.datetime.now()
+            try:
+                asyncio.run(populate_events(settings))
+            except Exception as e:
+                logger.error("Error populating events: %s", e, e)
+            finally:
+                end_time = datetime.datetime.now()
+                duration = (end_time - start_time).total_seconds()
+                logger.info(
+                    "Event population process completed in %.2f seconds", duration
+                )
+                sleep_duration = settings.INTERVAL_MINUTES * 60
+                remaining_time = max(0, sleep_duration - int(duration))
+                logger.info("Sleeping for %d seconds", remaining_time)
+                time.sleep(remaining_time)
 
     except Exception as e:
         logger.error("An error occurred while retrieving emails: %s", e)