remdb 0.3.14__py3-none-any.whl → 0.3.157__py3-none-any.whl

rem/services/content/service.py

@@ -278,6 +278,7 @@ class ContentService:
         category: str | None = None,
         tags: list[str] | None = None,
         is_local_server: bool = False,
+        resource_type: str | None = None,
     ) -> dict[str, Any]:
         """
         Complete file ingestion pipeline: read → store → parse → chunk → embed.
@@ -322,6 +323,9 @@ class ContentService:
             category: Optional category tag (document, code, audio, etc.)
             tags: Optional list of tags
             is_local_server: True if running as local/stdio MCP server
+            resource_type: Optional resource type (case-insensitive). Supports:
+                - "resource", "resources", "Resource" → Resource (default)
+                - "domain-resource", "domain_resource", "DomainResource" → DomainResource
 
         Returns:
             dict with:
@@ -366,11 +370,32 @@ class ContentService:
         file_size = len(file_content)
         logger.info(f"Read {file_size} bytes from {file_uri} (source: {source_type})")
 
-        # Step 2: Write to internal storage (user-scoped)
+        # Step 1.5: Early schema detection for YAML/JSON files
+        # Skip File entity creation for schemas (agents/evaluators)
+        file_suffix = Path(file_name).suffix.lower()
+        if file_suffix in ['.yaml', '.yml', '.json']:
+            import yaml
+            import json
+            try:
+                content_text = file_content.decode('utf-8') if isinstance(file_content, bytes) else file_content
+                data = yaml.safe_load(content_text) if file_suffix in ['.yaml', '.yml'] else json.loads(content_text)
+                if isinstance(data, dict):
+                    json_schema_extra = data.get('json_schema_extra', {})
+                    kind = json_schema_extra.get('kind', '')
+                    if kind in ['agent', 'evaluator']:
+                        # Route directly to schema processing, skip File entity
+                        logger.info(f"Detected {kind} schema: {file_name}, routing to _process_schema")
+                        result = self.process_uri(file_uri)
+                        return await self._process_schema(result, file_uri, user_id)
+            except Exception as e:
+                logger.debug(f"Early schema detection failed for {file_name}: {e}")
+                # Fall through to standard file processing
+
+        # Step 2: Write to internal storage (public or user-scoped)
         file_id = str(uuid4())
         storage_uri, internal_key, content_type, _ = await fs_service.write_to_internal_storage(
             content=file_content,
-            tenant_id=user_id,  # Using user_id for storage scoping
+            tenant_id=user_id or "public",  # Storage path: public/ or user_id/
             file_name=file_name,
             file_id=file_id,
         )
@@ -379,7 +404,7 @@ class ContentService:
         # Step 3: Create File entity
         file_entity = File(
             id=file_id,
-            tenant_id=user_id,  # Set tenant_id to user_id (application scoped to user)
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
             name=file_name,
             uri=storage_uri,
@@ -418,6 +443,7 @@ class ContentService:
             processing_result = await self.process_and_save(
                 uri=storage_uri,
                 user_id=user_id,
+                resource_type=resource_type,
             )
             processing_status = processing_result.get("status", "completed")
             resources_created = processing_result.get("chunk_count", 0)
@@ -459,7 +485,12 @@ class ContentService:
             "message": f"File ingested and {processing_status}. Created {resources_created} resources.",
         }
 
-    async def process_and_save(self, uri: str, user_id: str | None = None) -> dict[str, Any]:
+    async def process_and_save(
+        self,
+        uri: str,
+        user_id: str | None = None,
+        resource_type: str | None = None,
+    ) -> dict[str, Any]:
         """
         Process file end-to-end: extract → markdown → chunk → save.
 
@@ -474,6 +505,8 @@ class ContentService:
         Args:
             uri: File URI (s3://bucket/key or local path)
             user_id: Optional user ID for multi-tenancy
+            resource_type: Optional resource type (case-insensitive). Defaults to "Resource".
+                Supports: resource, domain-resource, domain_resource, DomainResource, etc.
 
         Returns:
             dict with file metadata and chunk count
@@ -526,7 +559,7 @@ class ContentService:
             size_bytes=result["metadata"].get("size"),
             mime_type=result["metadata"].get("content_type"),
             processing_status="completed",
-            tenant_id=user_id or "default",  # Required field
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
         )
 
@@ -534,28 +567,66 @@ class ContentService:
             await self.file_repo.upsert(file)
             logger.info(f"Saved File: {filename}")
 
-        # Create Resource entities for each chunk
-        resources = [
-            Resource(
+        # Resolve resource model class from type parameter (case-insensitive)
+        from typing import cast, Type
+        from pydantic import BaseModel
+        from rem.utils.model_helpers import model_from_arbitrary_casing, get_table_name
+
+        resource_model: Type[BaseModel] = Resource  # Default
+        if resource_type:
+            try:
+                resource_model = model_from_arbitrary_casing(resource_type)
+                logger.info(f"Using resource model: {resource_model.__name__}")
+            except ValueError as e:
+                logger.warning(f"Invalid resource_type '{resource_type}', using default Resource: {e}")
+                resource_model = Resource
+
+        # Get table name for the resolved model
+        table_name = get_table_name(resource_model)
+
+        # Create resource entities for each chunk
+        resources: list[BaseModel] = [
+            resource_model(
                 name=f"{filename}#chunk-{i}",
                 uri=f"{uri}#chunk-{i}",
                 ordinal=i,
                 content=chunk,
                 category="document",
-                tenant_id=user_id or "default",  # Required field
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             for i, chunk in enumerate(chunks)
         ]
 
-        if self.resource_repo:
-            await self.resource_repo.upsert(
-                resources,
-                embeddable_fields=["content"],
-                generate_embeddings=True,
-            )
-            logger.info(f"Saved {len(resources)} Resource chunks")
-            logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
+        # Save resources to the appropriate table
+        if resources:
+            from rem.services.postgres import get_postgres_service
+
+            postgres = get_postgres_service()
+            if postgres:
+                await postgres.connect()
+                try:
+                    await postgres.batch_upsert(
+                        records=cast(list[BaseModel | dict], resources),
+                        model=resource_model,
+                        table_name=table_name,
+                        entity_key_field="name",
+                        embeddable_fields=["content"],
+                        generate_embeddings=True,
+                    )
+                    logger.info(f"Saved {len(resources)} {resource_model.__name__} chunks to {table_name}")
+                    logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
+                finally:
+                    await postgres.disconnect()
+            elif self.resource_repo:
+                # Fallback to injected repo (only works for default Resource)
+                await self.resource_repo.upsert(
+                    resources,
+                    embeddable_fields=["content"],
+                    generate_embeddings=True,
+                )
+                logger.info(f"Saved {len(resources)} Resource chunks")
+                logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
 
         return {
             "file": file.model_dump(),
@@ -595,8 +666,10 @@ class ContentService:
         # IMPORTANT: category field distinguishes agents from evaluators
         # - kind=agent → category="agent" (AI agents with tools/resources)
         # - kind=evaluator → category="evaluator" (LLM-as-a-Judge evaluators)
+        # User-scoped schemas: if user_id provided, scope to user's tenant
+        # System schemas: if no user_id, use "system" tenant for shared access
         schema_entity = Schema(
-            tenant_id=user_id or "default",
+            tenant_id=user_id or "system",
             user_id=user_id,
             name=name,
             spec=schema_data,
@@ -667,7 +740,7 @@ class ContentService:
             processor = EngramProcessor(postgres)
             result = await processor.process_engram(
                 data=data,
-                tenant_id=user_id or "default",
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             logger.info(f"✅ Engram processed: {result.get('resource_id')} with {len(result.get('moment_ids', []))} moments")

rem/services/email/__init__.py

@@ -0,0 +1,10 @@
+"""
+Email Service Module.
+
+Provides EmailService for sending transactional emails and passwordless login.
+"""
+
+from .service import EmailService
+from .templates import EmailTemplate, login_code_template
+
+__all__ = ["EmailService", "EmailTemplate", "login_code_template"]

rem/services/email/service.py

@@ -0,0 +1,459 @@
+"""
+Email Service.
+
+Provides methods for sending transactional emails via SMTP.
+Supports passwordless login via email codes.
+"""
+
+import uuid
+import random
+import string
+import smtplib
+import logging
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from datetime import datetime, timedelta, timezone
+from typing import Any, Optional, TYPE_CHECKING
+
+from .templates import EmailTemplate, login_code_template, welcome_template
+
+if TYPE_CHECKING:
+    from ..postgres import PostgresService
+
+logger = logging.getLogger(__name__)
+
+
+class EmailService:
+    """Service for sending transactional emails and passwordless login."""
+
+    def __init__(
+        self,
+        smtp_host: str | None = None,
+        smtp_port: int | None = None,
+        sender_email: str | None = None,
+        sender_name: str | None = None,
+        app_password: str | None = None,
+        use_tls: bool = True,
+        login_code_expiry_minutes: int = 10,
+    ):
+        """
+        Initialize EmailService.
+
+        If no arguments provided, uses settings from rem.settings.
+        This allows no-arg construction for simple usage.
+
+        Args:
+            smtp_host: SMTP server host
+            smtp_port: SMTP server port
+            sender_email: Sender email address
+            sender_name: Sender display name
+            app_password: SMTP app password
+            use_tls: Use TLS encryption
+            login_code_expiry_minutes: Login code expiry in minutes
+        """
+        # Import settings lazily to avoid circular imports
+        from ...settings import settings
+
+        self._smtp_host = smtp_host or settings.email.smtp_host
+        self._smtp_port = smtp_port or settings.email.smtp_port
+        self._sender_email = sender_email or settings.email.sender_email
+        self._sender_name = sender_name or settings.email.sender_name
+        self._app_password = app_password or settings.email.app_password
+        self._use_tls = use_tls
+        self._login_code_expiry_minutes = (
+            login_code_expiry_minutes
+            or settings.email.login_code_expiry_minutes
+        )
+
+        if not self._app_password:
+            logger.warning(
+                "Email app password not configured. "
+                "Set EMAIL__APP_PASSWORD to enable email sending."
+            )
+
+    @property
+    def is_configured(self) -> bool:
+        """Check if email service is properly configured."""
+        return bool(self._sender_email and self._app_password)
+
+    def _create_smtp_connection(self) -> smtplib.SMTP:
+        """Create and authenticate SMTP connection."""
+        server = smtplib.SMTP(self._smtp_host, self._smtp_port)
+
+        if self._use_tls:
+            server.starttls()
+
+        server.login(self._sender_email, self._app_password)
+        return server
+
+    def send_email(
+        self,
+        to_email: str,
+        template: EmailTemplate,
+        reply_to: Optional[str] = None,
+    ) -> bool:
+        """
+        Send an email using a template.
+
+        Args:
+            to_email: Recipient email address
+            template: EmailTemplate with subject and HTML body
+            reply_to: Optional reply-to address
+
+        Returns:
+            True if sent successfully, False otherwise
+        """
+        if not self.is_configured:
+            logger.error("Email service not configured. Cannot send email.")
+            return False
+
+        try:
+            # Create message
+            msg = MIMEMultipart("alternative")
+            msg["Subject"] = template.subject
+            msg["From"] = f"{self._sender_name} <{self._sender_email}>"
+            msg["To"] = to_email
+
+            if reply_to:
+                msg["Reply-To"] = reply_to
+
+            # Attach HTML body
+            html_part = MIMEText(template.html_body, "html", "utf-8")
+            msg.attach(html_part)
+
+            # Send email
+            with self._create_smtp_connection() as server:
+                server.sendmail(
+                    self._sender_email,
+                    to_email,
+                    msg.as_string(),
+                )
+
+            logger.info(f"Email sent successfully to {to_email}: {template.subject}")
+            return True
+
+        except smtplib.SMTPAuthenticationError as e:
+            logger.error(f"SMTP authentication failed: {e}")
+            return False
+        except smtplib.SMTPException as e:
+            logger.error(f"SMTP error sending email to {to_email}: {e}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected error sending email to {to_email}: {e}")
+            return False
+
+    @staticmethod
+    def generate_login_code() -> str:
+        """
+        Generate a 6-digit login code.
+
+        Returns:
+            6-digit numeric string
+        """
+        return "".join(random.choices(string.digits, k=6))
+
+    @staticmethod
+    def generate_user_id_from_email(email: str) -> str:
+        """
+        Generate a deterministic UUID from email address.
+
+        Uses UUID v5 with DNS namespace for consistency.
+        Same email always produces same UUID.
+
+        Args:
+            email: Email address
+
+        Returns:
+            UUID string
+        """
+        return str(uuid.uuid5(uuid.NAMESPACE_DNS, email.lower().strip()))
+
+    async def send_login_code(
+        self,
+        email: str,
+        db: "PostgresService | None" = None,
+        tenant_id: str = "default",
+        template_kwargs: dict | None = None,
+    ) -> dict:
+        """
+        Send a login code to an email address.
+
+        Access control logic:
+        1. If user exists and tier is BLOCKED -> reject with "Account blocked"
+        2. If user exists and tier is not BLOCKED -> allow (send code)
+        3. If user doesn't exist -> check trusted_email_domains setting:
+           - If domain is trusted (or no restrictions) -> create user and send code
+           - If domain is not trusted -> reject with "Domain not allowed"
+
+        This method:
+        1. Generates a 6-digit login code
+        2. Checks user existence and access permissions
+        3. Upserts the user with login code in metadata
+        4. Sends the code via email
+
+        Args:
+            email: User's email address
+            db: PostgresService instance for repository operations
+            tenant_id: Tenant identifier for multi-tenancy
+            template_kwargs: Additional arguments for template customization
+
+        Returns:
+            Dict with status and details:
+            {
+                "success": bool,
+                "email": str,
+                "user_id": str,
+                "code_sent": bool,
+                "expires_at": str (ISO format),
+                "error": str (if failed)
+            }
+        """
+        from ...settings import settings
+
+        email = email.lower().strip()
+        code = self.generate_login_code()
+        user_id = self.generate_user_id_from_email(email)
+        expires_at = datetime.now(timezone.utc) + timedelta(
+            minutes=self._login_code_expiry_minutes
+        )
+
+        result = {
+            "success": False,
+            "email": email,
+            "user_id": user_id,
+            "code_sent": False,
+            "expires_at": expires_at.isoformat(),
+        }
+
+        # Check user access and upsert login code using repository
+        if db:
+            try:
+                access_result = await self._check_and_upsert_user_login_code(
+                    db=db,
+                    email=email,
+                    user_id=user_id,
+                    code=code,
+                    expires_at=expires_at,
+                    tenant_id=tenant_id,
+                    settings=settings,
+                )
+                if not access_result["allowed"]:
+                    result["error"] = access_result["error"]
+                    return result
+            except Exception as e:
+                logger.error(f"Failed to upsert user login code: {e}")
+                result["error"] = "Failed to store login code"
+                return result
+
+        # Send the email with branding from settings
+        # Merge settings.email.template_kwargs with any explicit overrides
+        kwargs = {**settings.email.template_kwargs, **(template_kwargs or {})}
+        template = login_code_template(code=code, email=email, **kwargs)
+        sent = self.send_email(to_email=email, template=template)
+
+        if sent:
+            result["success"] = True
+            result["code_sent"] = True
+            logger.info(
+                f"Login code sent to {email}, "
+                f"user_id={user_id}, expires at {expires_at.isoformat()}"
+            )
+        else:
+            result["error"] = "Failed to send email"
+
+        return result
+
+    async def _check_and_upsert_user_login_code(
+        self,
+        db: "PostgresService",
+        email: str,
+        user_id: str,
+        code: str,
+        expires_at: datetime,
+        tenant_id: str = "default",
+        settings: Any = None,
+    ) -> dict:
+        """
+        Check user access and upsert login code in metadata using repository pattern.
+
+        Access control logic:
+        1. If user exists and tier is BLOCKED -> reject
+        2. If user exists and tier is not BLOCKED -> allow and update
+        3. If user doesn't exist -> check trusted_email_domains:
+           - If domain is trusted -> create user
+           - If domain is not trusted -> reject
+
+        Args:
+            db: PostgresService instance
+            email: User's email
+            user_id: Deterministic UUID from email
+            code: Generated login code
+            expires_at: Code expiration datetime
+            tenant_id: Tenant identifier
+            settings: Settings instance for domain checking
+
+        Returns:
+            Dict with {"allowed": bool, "error": str | None}
+        """
+        from ...models.entities import User, UserTier
+        from ..postgres.repository import Repository
+
+        now = datetime.now(timezone.utc)
+        login_metadata = {
+            "login_code": code,
+            "login_code_expires_at": expires_at.isoformat(),
+            "login_code_sent_at": now.isoformat(),
+        }
+
+        # Use repository pattern for User operations
+        user_repo = Repository(User, db=db)
+
+        # Try to get existing user first
+        existing_user = await user_repo.get_by_id(user_id, tenant_id=tenant_id)
+
+        if existing_user:
+            # Check if user is blocked
+            if existing_user.tier == UserTier.BLOCKED:
+                logger.warning(f"Blocked user attempted login: {email}")
+                return {"allowed": False, "error": "Account is blocked"}
+
+            # User exists and is not blocked - merge login code into existing metadata
+            existing_user.metadata = {**(existing_user.metadata or {}), **login_metadata}
+            existing_user.email = email  # Ensure email is current
+            await user_repo.upsert(existing_user)
+            return {"allowed": True, "error": None}
+        else:
+            # New user - check if domain is trusted
+            if settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+                if not settings.email.is_domain_trusted(email):
+                    email_domain = email.split("@")[-1]
+                    logger.warning(f"Untrusted domain attempted signup: {email_domain}")
+                    return {"allowed": False, "error": "Email domain not allowed for signup"}
+
+            # Domain is trusted (or no restrictions) - create new user
+            # Users from trusted domains get admin role
+            user_role = None
+            if settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+                if settings.email.is_domain_trusted(email):
+                    user_role = "admin"
+                    logger.info(f"New user {email} assigned admin role (trusted domain)")
+
+            new_user = User(
+                id=uuid.UUID(user_id),
+                tenant_id=tenant_id,
+                name=email.split("@")[0],  # Default name from email
+                email=email,
+                role=user_role,
+                metadata=login_metadata,
+            )
+            await user_repo.upsert(new_user)
+            return {"allowed": True, "error": None}
+
+    async def verify_login_code(
+        self,
+        email: str,
+        code: str,
+        db: "PostgresService",
+        tenant_id: str = "default",
+    ) -> dict:
+        """
+        Verify a login code for an email address.
+
+        On success, clears the code from metadata and returns user info.
+
+        Args:
+            email: User's email address
+            code: The login code to verify
+            db: PostgresService instance
+            tenant_id: Tenant identifier
+
+        Returns:
+            Dict with verification result:
+            {
+                "valid": bool,
+                "user_id": str (if valid),
+                "email": str,
+                "error": str (if invalid)
+            }
+        """
+        from ...models.entities import User
+        from ..postgres.repository import Repository
+
+        email = email.lower().strip()
+        user_id = self.generate_user_id_from_email(email)
+
+        result = {
+            "valid": False,
+            "email": email,
+        }
+
+        try:
+            # Use repository pattern for User operations
+            user_repo = Repository(User, db=db)
+
+            # Get user by deterministic ID
+            user = await user_repo.get_by_id(user_id, tenant_id=tenant_id)
+
+            if not user:
+                result["error"] = "User not found"
+                return result
+
+            metadata = user.metadata or {}
+            stored_code = metadata.get("login_code")
+            expires_at_str = metadata.get("login_code_expires_at")
+
+            if not stored_code or not expires_at_str:
+                result["error"] = "No login code requested"
+                return result
+
+            # Check expiration
+            expires_at = datetime.fromisoformat(expires_at_str)
+            if datetime.now(timezone.utc) > expires_at:
+                result["error"] = "Login code expired"
+                return result
+
+            # Check code match
+            if stored_code != code:
+                result["error"] = "Invalid login code"
+                return result
+
+            # Code is valid - clear it from metadata
+            user.metadata = {
+                k: v for k, v in metadata.items()
+                if k not in ("login_code", "login_code_expires_at", "login_code_sent_at")
+            }
+            await user_repo.upsert(user)
+
+            result["valid"] = True
+            result["user_id"] = str(user.id)
+            logger.info(f"Login code verified for {email}, user_id={result['user_id']}")
+
+        except Exception as e:
+            logger.error(f"Error verifying login code: {e}")
+            result["error"] = "Verification failed"
+
+        return result
+
+    async def send_welcome_email(
+        self,
+        email: str,
+        name: Optional[str] = None,
+        template_kwargs: dict | None = None,
+    ) -> bool:
+        """
+        Send a welcome email to a new user.
+
+        Args:
+            email: User's email address
+            name: Optional user's name
+            template_kwargs: Additional arguments for template customization
+
+        Returns:
+            True if sent successfully
+        """
+        from ...settings import settings
+
+        # Merge settings.email.template_kwargs with any explicit overrides
+        kwargs = {**settings.email.template_kwargs, **(template_kwargs or {})}
+        template = welcome_template(name=name, **kwargs)
+        return self.send_email(to_email=email, template=template)