email-to-calendar 20250826010803.dev0__py3-none-any.whl → 20251210163203.dev0__py3-none-any.whl

model/email.py

@@ -2,10 +2,10 @@ from datetime import datetime
 import enum
 
 import tzlocal
-from sqlalchemy import Column, Integer, String, DateTime, Enum
-from sqlalchemy.orm import relationship
+from sqlmodel import SQLModel, Field, select
 
-from src.db import Base, SessionLocal
+from src.db import Session, engine
+from src.model.event import Event
 
 
 class EMailType(enum.Enum):
@@ -13,19 +13,16 @@ class EMailType(enum.Enum):
     HTML = "html"
 
 
-class EMail(Base):
-    __tablename__ = "emails"
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    subject = Column(String, nullable=False)
-    from_address = Column(String, nullable=False)
-    delivery_date = Column(DateTime, nullable=False)
-    body = Column(String, nullable=False)
-    retrieved_date = Column(
-        DateTime, nullable=False, default=lambda: datetime.now(tzlocal.get_localzone())
+class EMail(SQLModel, table=True):
+    id: int = Field(primary_key=True)
+    subject: str = Field(nullable=False)
+    from_address: str = Field(nullable=False)
+    delivery_date: datetime = Field(nullable=False)
+    body: str = Field(nullable=False)
+    retrieved_date: datetime = Field(
+        nullable=False, default=lambda: datetime.now(tzlocal.get_localzone())
     )
-    email_type = Column(Enum(EMailType), nullable=False)
-
-    events = relationship("Event", back_populates="email", cascade="all, delete-orphan")
+    email_type: EMailType = Field(nullable=False)
 
     def __repr__(self):
         return f"<EMail(id={self.id}, subject={self.subject}, from_address={self.from_address}, delivery_date={self.delivery_date})>"
@@ -34,7 +31,7 @@ class EMail(Base):
         return f"EMail(id={self.id}, subject={self.subject}, from_address={self.from_address}, delivery_date={self.delivery_date})"
 
     def save(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
             self.retrieved_date = datetime.now(tzlocal.get_localzone())
             session.merge(self)
@@ -43,50 +40,84 @@ class EMail(Base):
             session.close()
 
     def get(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return session.query(EMail).filter(EMail.id == self.id).first()
+            return session.exec(select(EMail).where(EMail.id == self.id)).first()
         finally:
             session.close()
 
     def delete(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            session.query(EMail).filter(EMail.id == self.id).delete()
+            session.delete(EMail.where(EMail.id == self.id))
             session.commit()
         finally:
             session.close()
 
     @staticmethod
     def get_by_id(email_id: int):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return session.query(EMail).filter(EMail.id == email_id).first()
+            return session.exec(select(EMail).where(EMail.id == email_id)).first()
         finally:
             session.close()
 
     @staticmethod
     def get_all():
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return session.query(EMail).all()
+            return session.exec(select(EMail)).all()
         finally:
             session.close()
 
     @staticmethod
     def get_by_delivery_date(delivery_date: datetime):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return (
-                session.query(EMail).filter(EMail.delivery_date == delivery_date).all()
-            )
+            return session.exec(
+                select(EMail).where(EMail.delivery_date == delivery_date)
+            ).all()
         finally:
             session.close()
 
     @staticmethod
     def get_most_recent():
-        session = SessionLocal()
+        session = Session(engine)
+        try:
+            return session.exec(
+                select(EMail).order_by(EMail.delivery_date.desc())
+            ).first()
+        finally:
+            session.close()
+
+    @staticmethod
+    def get_most_recent_without_events():
+        session = Session(engine)
+        try:
+            email: EMail = session.exec(
+                select(EMail).order_by(EMail.delivery_date.desc())
+            ).first()
+            # Check if this email has any associated events
+            if email:
+                event = session.exec(
+                    select(Event).where(Event.email_id == email.id)
+                ).first()
+                if event:
+                    return None  # The most recent email has associated events
+            return email
+        finally:
+            session.close()
+
+    @staticmethod
+    def get_without_events():
+        session = Session(engine)
         try:
-            return session.query(EMail).order_by(EMail.delivery_date.desc()).first()
+            # Use NOT EXISTS to find emails with no events referencing them
+            result = session.exec(
+                select(EMail).where(
+                    ~select(Event).where(Event.email_id == EMail.id).exists()
+                )
+            ).all()
+            return result
         finally:
             session.close()

model/event.py

@@ -1,32 +1,34 @@
 from datetime import datetime
 
 from sqlalchemy import (
-    Column,
-    Integer,
-    String,
-    DateTime,
-    Boolean,
     UniqueConstraint,
-    ForeignKey,
 )
-from sqlalchemy.orm import relationship
+from sqlmodel import SQLModel, Field, select
 
-from src.db import Base, SessionLocal
+from src.db import Session, engine
 
 
-class Event(Base):
-    __tablename__ = "events"
+class Event(SQLModel, table=True):
     __table_args__ = (
         UniqueConstraint("start", "end", "summary", name="uq_event_start_end_summary"),
     )
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    start = Column(DateTime, nullable=False)
-    end = Column(DateTime, nullable=False)
-    summary = Column(String, nullable=False)
-    email_id = Column(Integer, ForeignKey("emails.id"), nullable=True, index=True)
-    in_calendar = Column(Boolean, nullable=False, default=False)
-
-    email = relationship("EMail", back_populates="events")
+    id: int = Field(primary_key=True)
+    start: datetime = Field(
+        nullable=False,
+        description="The start date and time of the event, must be a Python datetime object, cannot be None",
+    )
+    end: datetime = Field(
+        nullable=False,
+        description="The end date and time of the event, must be a Python datetime object, cannot be None",
+    )
+    all_day: bool = Field(
+        nullable=False,
+        default=False,
+        description="Whether the event lasts all day or not",
+    )
+    summary: str = Field(nullable=False)
+    email_id: int = Field(foreign_key="email.id", nullable=True)
+    in_calendar: bool = Field(nullable=False, default=False)
 
     def __repr__(self):
         return f"<Event(id={self.id}, start={self.start}, end={self.end}, summary={self.summary})>"
@@ -35,98 +37,51 @@ class Event(Base):
         return f"Event(id={self.id}, start={self.start}, end={self.end}, summary={self.summary})"
 
     def save(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            # First check for exact duplicates (same start, end, and summary)
-            existing_exact = (
-                session.query(Event)
-                .filter_by(start=self.start, end=self.end, summary=self.summary)
-                .one_or_none()
-            )
-            if existing_exact:
-                # Preserve email linkage if existing record has it but new instance doesn't
-                if self.email_id is None and existing_exact.email_id is not None:
-                    self.email_id = existing_exact.email_id
-                self.id = existing_exact.id  # Update the ID to match the existing record
-                session.merge(self)
-                session.commit()
-                return
-
-            # Check for events with the same summary but different start/end times
-            if self.email_id is not None:
-                from src.model.email import EMail  # Import here to avoid circular import
-
-                # Get all events with the same summary
-                existing_events_with_same_summary = (
-                    session.query(Event)
-                    .filter_by(summary=self.summary)
-                    .filter(Event.email_id.isnot(None))
-                    .all()
-                )
-
-                if existing_events_with_same_summary:
-                    # Get the current email's delivery date
-                    current_email = session.query(EMail).filter_by(id=self.email_id).first()
-                    if current_email:
-                        current_delivery_date = current_email.delivery_date
-
-                        # Check if any existing events are from older emails
-                        events_to_remove = []
-                        for existing_event in existing_events_with_same_summary:
-                            existing_email = session.query(EMail).filter_by(id=existing_event.email_id).first()
-                            if existing_email and existing_email.delivery_date < current_delivery_date:
-                                events_to_remove.append(existing_event)
-
-                        # Remove older events with the same summary
-                        for event_to_remove in events_to_remove:
-                            session.delete(event_to_remove)
-
-                        # Also check if there are newer events with the same summary
-                        has_newer_event = False
-                        for existing_event in existing_events_with_same_summary:
-                            if existing_event not in events_to_remove:
-                                existing_email = session.query(EMail).filter_by(id=existing_event.email_id).first()
-                                if existing_email and existing_email.delivery_date > current_delivery_date:
-                                    has_newer_event = True
-                                    break
-
-                        # Only save the current event if there are no newer events with the same summary
-                        if not has_newer_event:
-                            session.merge(self)
-                            session.commit()
-                        # If there is a newer event, don't save the current event
-                    else:
-                        # If we can't get the email, proceed with normal save
-                        session.merge(self)
-                        session.commit()
+            if isinstance(self.start, str):
+                self.start = datetime.fromisoformat(self.start)
+            if isinstance(self.end, str):
+                self.end = datetime.fromisoformat(self.end)
+            if self.id is not None:
+                # Update existing event
+                existing_event = session.exec(
+                    select(Event).where(Event.id == self.id)
+                ).first()
+                if existing_event:
+                    existing_event.start = self.start
+                    existing_event.end = self.end
+                    existing_event.summary = self.summary
+                    existing_event.email_id = self.email_id
+                    existing_event.in_calendar = self.in_calendar
+                    session.add(existing_event)
                 else:
-                    # No existing events with same summary, proceed with normal save
-                    session.merge(self)
-                    session.commit()
+                    # If id is set but not found, treat as new
+                    session.add(self)
             else:
-                # No email_id, proceed with normal save
-                session.merge(self)
-                session.commit()
+                session.add(self)
+            session.commit()
+            session.flush()
         finally:
             session.close()
 
     def get(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return session.query(Event).filter(Event.id == self.id).first()
+            return session.exec(select(Event).where(Event.id == self.id)).first()
         finally:
             session.close()
 
     def delete(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            session.query(Event).filter(Event.id == self.id).delete()
+            session.delete(Event.where(Event.id == self.id))
             session.commit()
         finally:
             session.close()
 
     def save_to_caldav(self):
-        session = SessionLocal()
+        session = Session(engine)
         try:
             self.in_calendar = True
             session.merge(self)
@@ -136,28 +91,34 @@ class Event(Base):
 
     @staticmethod
     def get_by_id(event_id: int):
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return session.query(Event).filter(Event.id == event_id).first()
+            return session.exec(select(Event).where(Event.id == event_id)).first()
         finally:
             session.close()
 
     @staticmethod
     def get_all():
-        session = SessionLocal()
+        session = Session(engine)
         try:
-            return session.query(Event).all()
+            return session.exec(select(Event)).all()
         finally:
             session.close()
 
     @staticmethod
     def get_by_date(date: datetime):
-        session = SessionLocal()
+        session = Session(engine)
+        try:
+            return session.exec(
+                select(Event).where(Event.start == date or Event.end == date)
+            ).all()
+        finally:
+            session.close()
+
+    @staticmethod
+    def get_not_in_calendar():
+        session = Session(engine)
         try:
-            return (
-                session.query(Event)
-                .filter(Event.start == date or Event.end == date)
-                .all()
-            )
+            return session.exec(select(Event).where(Event.in_calendar == False)).all()
         finally:
             session.close()

util/ai.py

@@ -0,0 +1,163 @@
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+from pydantic_ai import Agent, RunContext
+from pydantic_ai.models.openai import OpenAIChatModel
+from pydantic_ai.providers.ollama import OllamaProvider
+from pydantic_ai.providers.openai import OpenAIProvider
+from sqlmodel import select
+
+from src import logger
+from src.db import Session, engine
+from src.model.email import EMail
+from src.model.event import Event
+from src.util.env import AIProvider, get_settings
+
+DEFAULT_SYSTEM_PROMPT = """You are an assistant that extracts calendar events from an email body. Produce only a single JSON object that strictly conforms to the schema below. Do not include any explanatory text, code fences, or comments—only the final JSON.
+Context you will receive via earlier messages:
+Current year to assume if no year is present
+The email ID to assign to the email_id field
+A list of existing events from the database
+Output schema
+Top-level object: { "events": [Event, ...] }
+Event object fields and types:
+id: integer or null
+If you can confidently match an existing DB event by start, end, and summary (case-insensitive, trimmed), set to that event’s id; otherwise null.
+start: string, ISO 8601 datetime (e.g., 2025-11-16T09:00:00 or 2025-11-16T09:00:00-05:00)
+end: string, ISO 8601 datetime
+all_day: boolean
+summary: string
+A concise title/description with all date/time tokens removed.
+email_id: integer (must be the provided email ID)
+in_calendar: boolean (always false for new extractions)
+Event extraction and inference rules
+Treat the email body as lines:
+Most non-blank lines are events; lines that are clearly headers (month or year) set context for following lines until superseded.
+Month/year headers: use them to resolve dates for subsequent event lines.
+Dates:
+A date may be on the same line as an event or inherited from the last resolvable date context above.
+Multi-day ranges (e.g., “Oct 10–12”, “10-12” under an October header) mean start is the first day at start time and end is the last day at end time.
+If no year is present, use the provided “current year” context.
+Times:
+Recognize formats such as: 12:50, 6:30, 9am, 8, 820, 130, 10-12, 10:30-12:30, 9am-11am, 9-11, 9-11am, 9am-11
+If a single time is provided for a single-day event, assume a 1-hour duration.
+If no time is present, the event is all-day: start 00:00:00, end 23:59:00, all_day = true.
+If a multi-day event provides only a start time, start at that time on the first day and end at 23:59:00 on the last day, all_day = false.
+Normalize 8 → 08:00:00; 820 → 08:20:00; 130 → 01:30:00; add seconds as :00 if missing.
+Respect am/pm; if none given and a 12-hour ambiguity exists, prefer a sensible local interpretation (e.g., 9 → 09:00).
+Summary:
+Remove all date/time expressions and markers from the line; keep a clear human title.
+Trim extra punctuation and whitespace.
+Deduplication and matching:
+Avoid emitting duplicates within this run (same start, end, summary).
+To fill id from DB, match by exact start, end, and normalized summary (case-insensitive, trimmed); otherwise id = null.
+Defaults:
+email_id must equal the provided email ID.
+in_calendar must be false.
+If a line cannot be reliably parsed into an event, you may skip it (do not invent events).
+Formatting requirements
+Emit exactly one JSON object with this shape: { "events": [ { "id": null or integer, "start": "YYYY-MM-DDTHH:MM:SS[±HH:MM]", "end": "YYYY-MM-DDTHH:MM:SS[±HH:MM]", "all_day": true|false, "summary": "string", "email_id": integer, "in_calendar": false }, ... ] }
+Key names must be lower_snake_case exactly as above.
+No markdown, code fences, or extra prose—only the JSON.
+Self-check before answering
+Validate that:
+All events have start < end, both in ISO 8601
+all_day is true only when start is 00:00:00 and end is 23:59:00 for that day (or the multi-day all-day case)
+email_id equals the provided number
+in_calendar is false for all events
+id is integer only when a clear DB match exists; otherwise null
+summary has no date/time remnants
+If any item fails validation, correct the output and re-validate before responding.
+If still invalid, correct again; keep correcting until the output matches the schema and rules."""
+
+@dataclass
+class AgentDependencies:
+    email: EMail
+    max_result_retries: int = get_settings().AI_MAX_RETRIES
+    db = Session(engine)
+
+
+class Events(BaseModel):
+    events: list[Event] = Field(description="A list of events parsed from the email")
+
+
+async def parse_email(
+    email: EMail,
+    provider: AIProvider,
+    model: str = "gpt-oss:20b",
+    ollama_host: str = "localhost",
+    ollama_port: int = 11434,
+    ollama_secure: bool = False,
+    open_ai_api_key: str = None,
+    max_retries: int = 3,
+    system_prompt: str = None,
+) -> list[Event]:
+    if provider == AIProvider.OLLAMA:
+        logger.info(
+            f"Creating events from email id {email.id}, using model: {model} at ollama host: {'https://' if ollama_secure else 'http://'}{ollama_host}:{ollama_port}"
+        )
+        ai_model = OpenAIChatModel(
+            model_name=model,
+            provider=OllamaProvider(
+                base_url=f"{'https://' if ollama_secure else 'http://'}{ollama_host}:{ollama_port}/v1"
+            ),
+        )
+    elif provider == AIProvider.OPENAI:
+        logger.info(
+            f"Creating events from email id {email.id}, using OpenAI model: {model}"
+        )
+        ai_model = OpenAIChatModel(
+            model_name=model, provider=OpenAIProvider(api_key=open_ai_api_key)
+        )
+    else:
+        raise ValueError(f"Unsupported AI provider: {provider}")
+
+    deps = AgentDependencies(email=email)
+
+    system_prompt = system_prompt or DEFAULT_SYSTEM_PROMPT
+
+    agent = Agent(
+        ai_model,
+        deps_type=AgentDependencies,
+        output_type=Events,
+        system_prompt=system_prompt,
+        retries=max_retries,
+    )
+
+    #@agent.system_prompt
+    #async def get_current_year(ctx: RunContext[AgentDependencies]):
+    #    return f"If there is no year provided, use this year: {ctx.deps.email.delivery_date.year}"
+
+
+    @agent.system_prompt
+    async def get_email_id(ctx: RunContext[AgentDependencies]):
+        return f"The email ID is {ctx.deps.email.id}. Use this ID for the email_id attribute of the event(s)."
+
+    @agent.system_prompt
+    async def get_events(ctx: RunContext[AgentDependencies]):
+        logger.info("Checking existing events in the database...")
+        events = ctx.deps.db.exec(select(Event)).all()
+        logger.debug("Found %d existing events in the database", len(events))
+        if not events:
+            return "There are no current events in the database, assume all events are new."
+        return f"The currents events in the database are: {events}"
+
+    logger.info("Generating events...")
+    start_time = datetime.now()
+
+    task = asyncio.create_task(agent.run(email.body, deps=deps))
+    while not task.done():
+        logger.debug("Waiting for AI to finish generating events...")
+        await asyncio.sleep(10)
+
+    result = await task
+    events: Events = result.output
+
+    end_time = datetime.now()
+    elapsed = (end_time - start_time).total_seconds()
+    logger.info(
+        f"Took {elapsed:.3f} seconds to generate {len(events.events)} events from email id: {email.id}"
+    )
+    return events.events

util/env.py

@@ -0,0 +1,97 @@
+from pathlib import Path
+from typing import Optional
+from enum import Enum
+
+from pydantic import (
+    Field,
+    ValidationError,
+    AnyUrl,
+    EmailStr,
+)
+from pydantic_settings import BaseSettings
+
+
+class AIProvider(str, Enum):
+    OLLAMA = "ollama"
+    OPENAI = "openai"
+
+
+class Settings(BaseSettings):
+    IMAP_HOST: str = Field(default=None, description="IMAP server host")
+    IMAP_PORT: int = Field(993, ge=1, le=65535, description="IMAP server port")
+    IMAP_USERNAME: str = Field(default=None, description="IMAP username")
+    IMAP_PASSWORD: str = Field(default=None, description="IMAP password")
+    IMAP_MAILBOX: str = Field("INBOX", description="IMAP mailbox to check")
+    IMAP_SSL: bool = Field(True, description="Whether to use SSL for IMAP connection")
+
+    FILTER_FROM_EMAIL: Optional[EmailStr] = Field(
+        None, description="Email address to filter messages from (optional)"
+    )
+    FILTER_SUBJECT: Optional[str] = Field(None, description="Subject filter (optional)")
+    BACKFILL: bool = Field(False, description="Whether to backfill all emails")
+    INTERVAL_MINUTES: int = Field(
+        15, ge=1, description="Interval in minutes to check for new emails"
+    )
+
+    CALDAV_URL: AnyUrl = Field(default=None, description="CalDAV server URL")
+    CALDAV_USERNAME: str = Field(default=None, description="CalDAV username")
+    CALDAV_PASSWORD: str = Field(default=None, description="CalDAV password")
+    CALDAV_CALENDAR: str = Field(default=None, description="CalDAV calendar name")
+
+    AI_PROVIDER: AIProvider = Field(
+        default=None, description="AI provider to use (ollama, openai, none)"
+    )
+
+    OLLAMA_SECURE: bool = Field(
+        False, description="Whether to use HTTPS for Ollama connection"
+    )
+    OLLAMA_HOST: str = Field("localhost", description="Ollama base URL")
+    OLLAMA_PORT: int = Field(11434, ge=1, le=65535, description="Ollama port")
+
+    OPEN_AI_API_KEY: Optional[str] = Field(
+        None, description="OpenAI API key (required if AI_PROVIDER is openai)"
+    )
+
+    AI_MODEL: str = Field(default=None, description="Model to use for parsing")
+    AI_MAX_RETRIES: int = Field(3, ge=0, description="Maximum retries for AI parsing")
+
+    AI_SYSTEM_PROMPT: Optional[str] = Field(
+        None, description="Custom system prompt for the AI model (optional)"
+    )
+    AI_SYSTEM_PROMPT_FILE: Optional[str] = Field(
+        None, description="Custom system prompt for the AI model (optional)"
+    )
+
+    DB_FILE: str = Field(f"{Path.cwd()}/data/emails.db", description="SQLite database file path")
+
+    APPRISE_URL: Optional[AnyUrl] = Field(
+        None, description="Apprise notification service URL (optional)"
+    )
+
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+        case_sensitive = True
+
+
+def get_settings() -> Settings:
+    try:
+        settings = Settings()
+        if settings.AI_PROVIDER == AIProvider.OLLAMA and not settings.AI_MODEL:
+            settings.AI_MODEL = "gpt-oss:20b"
+            assert settings.OLLAMA_HOST is not None
+            assert settings.OLLAMA_PORT is not None
+        elif settings.AI_PROVIDER == AIProvider.OPENAI and not settings.AI_MODEL:
+            settings.AI_MODEL = "gpt-5-mini"
+            assert settings.OPEN_AI_API_KEY is not None
+
+        if settings.AI_SYSTEM_PROMPT_FILE:
+            try:
+                with open(settings.AI_SYSTEM_PROMPT_FILE, "r") as f:
+                    settings.AI_SYSTEM_PROMPT = f.read()
+            except Exception as e:
+                raise ValueError(f"Error reading system prompt file: {e}")
+        return settings
+    except ValidationError as exc:
+        # Fail fast with a clear error so startup doesn't proceed with bad config
+        raise SystemExit(f"Environment validation error:\n{exc}")

util/notifications.py

@@ -0,0 +1,20 @@
+import apprise
+from pydantic import AnyUrl
+
+from src.model.event import Event
+
+
+def send_success_notification(apprise_url: AnyUrl, events: list[Event]):
+    # Create an Apprise instance
+    apobj = apprise.Apprise()
+
+    # Add all the notification services by their server url.
+    # A sample email notification:
+    apobj.add(apprise_url.encoded_string())
+
+    # Then notify these services any time you desire. The below would
+    # notify all the services loaded into our Apprise object.
+    apobj.notify(
+        body=f"The following new events were added to your calendar: {events}",
+        title="New Events Added to Calendar",
+    )