remdb 0.3.146__py3-none-any.whl → 0.3.181__py3-none-any.whl

rem/auth/providers/email.py

@@ -0,0 +1,215 @@
+"""
+Email Authentication Provider.
+
+Passwordless authentication using email verification codes.
+Unlike OAuth providers, this handles the full flow internally.
+
+Flow:
+1. User requests login with email address
+2. System generates code, upserts user, sends email
+3. User enters code
+4. System verifies code and creates session
+
+Design:
+- Uses EmailService for sending codes
+- Creates users with deterministic UUID from email hash
+- Stores challenge in user metadata
+- No external OAuth dependencies
+"""
+
+from typing import TYPE_CHECKING
+from pydantic import BaseModel, Field
+from loguru import logger
+
+from ...services.email import EmailService
+
+if TYPE_CHECKING:
+    from ...services.postgres import PostgresService
+
+
+class EmailAuthResult(BaseModel):
+    """Result of email authentication operations."""
+
+    success: bool = Field(description="Whether operation succeeded")
+    email: str = Field(description="Email address")
+    user_id: str | None = Field(default=None, description="User ID if authenticated")
+    error: str | None = Field(default=None, description="Error message if failed")
+    message: str | None = Field(default=None, description="User-friendly message")
+
+
+class EmailAuthProvider:
+    """
+    Email-based passwordless authentication provider.
+
+    Handles the complete email login flow:
+    1. send_code() - Generate and send verification code
+    2. verify_code() - Verify code and return user info
+    """
+
+    def __init__(
+        self,
+        email_service: EmailService | None = None,
+        template_kwargs: dict | None = None,
+    ):
+        """
+        Initialize EmailAuthProvider.
+
+        Args:
+            email_service: EmailService instance (creates new one if not provided)
+            template_kwargs: Customization for email templates (colors, branding, etc.)
+        """
+        self._email_service = email_service or EmailService()
+        self._template_kwargs = template_kwargs or {}
+
+    @property
+    def is_configured(self) -> bool:
+        """Check if email auth is properly configured."""
+        return self._email_service.is_configured
+
+    async def send_code(
+        self,
+        email: str,
+        db: "PostgresService",
+        tenant_id: str = "default",
+    ) -> EmailAuthResult:
+        """
+        Send a verification code to an email address.
+
+        Creates user if not exists (using deterministic UUID from email).
+        Stores code in user metadata.
+
+        Args:
+            email: Email address to send code to
+            db: PostgresService instance
+            tenant_id: Tenant identifier
+
+        Returns:
+            EmailAuthResult with success status
+        """
+        if not self.is_configured:
+            return EmailAuthResult(
+                success=False,
+                email=email,
+                error="Email service not configured",
+                message="Email login is not available. Please try another method.",
+            )
+
+        try:
+            result = await self._email_service.send_login_code(
+                email=email,
+                db=db,
+                tenant_id=tenant_id,
+                template_kwargs=self._template_kwargs,
+            )
+
+            if result["success"]:
+                return EmailAuthResult(
+                    success=True,
+                    email=email,
+                    user_id=result["user_id"],
+                    message=f"Verification code sent to {email}. Check your inbox.",
+                )
+            else:
+                return EmailAuthResult(
+                    success=False,
+                    email=email,
+                    error=result.get("error", "Failed to send code"),
+                    message="Failed to send verification code. Please try again.",
+                )
+
+        except Exception as e:
+            logger.error(f"Error sending login code: {e}")
+            return EmailAuthResult(
+                success=False,
+                email=email,
+                error=str(e),
+                message="An error occurred. Please try again.",
+            )
+
+    async def verify_code(
+        self,
+        email: str,
+        code: str,
+        db: "PostgresService",
+        tenant_id: str = "default",
+    ) -> EmailAuthResult:
+        """
+        Verify a login code and authenticate user.
+
+        Args:
+            email: Email address
+            code: 6-digit verification code
+            db: PostgresService instance
+            tenant_id: Tenant identifier
+
+        Returns:
+            EmailAuthResult with user_id if successful
+        """
+        try:
+            result = await self._email_service.verify_login_code(
+                email=email,
+                code=code,
+                db=db,
+                tenant_id=tenant_id,
+            )
+
+            if result["valid"]:
+                return EmailAuthResult(
+                    success=True,
+                    email=email,
+                    user_id=result["user_id"],
+                    message="Successfully authenticated!",
+                )
+            else:
+                error = result.get("error", "Invalid code")
+                # User-friendly error messages
+                if error == "Login code expired":
+                    message = "Your code has expired. Please request a new one."
+                elif error == "Invalid login code":
+                    message = "Invalid code. Please check and try again."
+                elif error == "No login code requested":
+                    message = "No code was requested for this email. Please request a new code."
+                elif error == "User not found":
+                    message = "Email not found. Please request a login code first."
+                else:
+                    message = "Verification failed. Please try again."
+
+                return EmailAuthResult(
+                    success=False,
+                    email=email,
+                    error=error,
+                    message=message,
+                )
+
+        except Exception as e:
+            logger.error(f"Error verifying login code: {e}")
+            return EmailAuthResult(
+                success=False,
+                email=email,
+                error=str(e),
+                message="An error occurred. Please try again.",
+            )
+
+    def get_user_dict(self, email: str, user_id: str) -> dict:
+        """
+        Create a user dict for session storage.
+
+        Compatible with OAuth user format for consistent session handling.
+
+        Args:
+            email: User's email
+            user_id: User's UUID
+
+        Returns:
+            User dict for session
+        """
+        return {
+            "id": user_id,
+            "email": email,
+            "email_verified": True,  # Email is verified through code
+            "name": email.split("@")[0],  # Use email prefix as name
+            "provider": "email",
+            "tenant_id": "default",
+            "tier": "free",  # Email users start at free tier
+            "roles": ["user"],
+        }

rem/cli/commands/ask.py

@@ -75,7 +75,7 @@ async def run_agent_streaming(
     """
     Run agent in streaming mode using agent.iter() with usage limits.
 
-    Design Pattern (from carrier):
+    Design Pattern:
     - Use agent.iter() for complete execution with tool call visibility
     - run_stream() stops after first output, missing tool calls
     - Stream tool call markers: [Calling: tool_name]

rem/cli/commands/db.py

@@ -333,64 +333,120 @@ def rebuild_cache(connection: str | None):
 
 @click.command()
 @click.argument("file_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--table", "-t", default=None, help="Target table name (required for non-YAML formats)")
 @click.option("--user-id", default=None, help="User ID to scope data privately (default: public/shared)")
 @click.option("--dry-run", is_flag=True, help="Show what would be loaded without loading")
-def load(file_path: Path, user_id: str | None, dry_run: bool):
+def load(file_path: Path, table: str | None, user_id: str | None, dry_run: bool):
     """
-    Load data from YAML file into database.
+    Load data from file into database.
 
-    File format:
-        - table: resources
-          key_field: name
-          rows:
-            - name: Example
-              content: Test data...
+    Supports YAML with embedded metadata, or any tabular format via Polars
+    (jsonl, parquet, csv, json, arrow, etc.). For non-YAML formats, use --table.
 
     Examples:
-        rem db load rem/tests/data/graph_seed.yaml
-        rem db load data.yaml --user-id my-user  # Private to user
-        rem db load data.yaml --dry-run
+        rem db load data.yaml                        # YAML with metadata
+        rem db load data.jsonl -t resources          # Any Polars-supported format
     """
-    asyncio.run(_load_async(file_path, user_id, dry_run))
+    asyncio.run(_load_async(file_path, table, user_id, dry_run))
 
 
-async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
+def _load_dataframe_from_file(file_path: Path) -> "pl.DataFrame":
+    """Load any Polars-supported file format into a DataFrame."""
+    import polars as pl
+
+    suffix = file_path.suffix.lower()
+
+    if suffix in {".jsonl", ".ndjson"}:
+        return pl.read_ndjson(file_path)
+    elif suffix in {".parquet", ".pq"}:
+        return pl.read_parquet(file_path)
+    elif suffix == ".csv":
+        return pl.read_csv(file_path)
+    elif suffix == ".json":
+        return pl.read_json(file_path)
+    elif suffix in {".ipc", ".arrow"}:
+        return pl.read_ipc(file_path)
+    else:
+        raise ValueError(f"Unsupported file format: {suffix}. Use any Polars-supported format.")
+
+
+async def _load_async(file_path: Path, table: str | None, user_id: str | None, dry_run: bool):
     """Async implementation of load command."""
+    import polars as pl
     import yaml
     from ...models.core.inline_edge import InlineEdge
-    from ...models.entities import Resource, Moment, User, Message, SharedSession, Schema
+    from ...models.entities import SharedSession
     from ...services.postgres import get_postgres_service
+    from ...utils.model_helpers import get_table_name
+    from ... import get_model_registry
 
     logger.info(f"Loading data from: {file_path}")
     scope_msg = f"user: {user_id}" if user_id else "public"
     logger.info(f"Scope: {scope_msg}")
 
-    # Load YAML file
-    with open(file_path) as f:
-        data = yaml.safe_load(f)
+    suffix = file_path.suffix.lower()
+    is_yaml = suffix in {".yaml", ".yml"}
 
-    if not isinstance(data, list):
-        logger.error("YAML must be a list of table definitions")
-        raise click.Abort()
-
-    if dry_run:
-        logger.info("DRY RUN - Would load:")
-        logger.info(yaml.dump(data, default_flow_style=False))
-        return
-
-    # Map table names to model classes
-    # CoreModel subclasses use Repository.upsert()
+    # Build MODEL_MAP dynamically from registry
+    registry = get_model_registry()
+    registry.register_core_models()
     MODEL_MAP = {
-        "users": User,
-        "moments": Moment,
-        "resources": Resource,
-        "messages": Message,
-        "schemas": Schema,
+        get_table_name(model): model
+        for model in registry.get_model_classes().values()
     }
 
     # Non-CoreModel tables that need direct SQL insertion
     DIRECT_INSERT_TABLES = {"shared_sessions"}
 
+    # Parse file based on format
+    if is_yaml:
+        # YAML with embedded metadata
+        with open(file_path) as f:
+            data = yaml.safe_load(f)
+
+        if not isinstance(data, list):
+            logger.error("YAML must be a list of table definitions")
+            raise click.Abort()
+
+        if dry_run:
+            logger.info("DRY RUN - Would load:")
+            logger.info(yaml.dump(data, default_flow_style=False))
+            return
+
+        table_defs = data
+    else:
+        # Polars-supported format - require --table
+        if not table:
+            logger.error(f"For {suffix} files, --table is required. Example: rem db load {file_path.name} -t resources")
+            raise click.Abort()
+
+        try:
+            df = _load_dataframe_from_file(file_path)
+        except Exception as e:
+            logger.error(f"Failed to load file: {e}")
+            raise click.Abort()
+
+        rows = df.to_dicts()
+
+        if dry_run:
+            logger.info(f"DRY RUN - Would load {len(rows)} rows to table '{table}':")
+            logger.info(f"Columns: {list(df.columns)}")
+
+            # Validate first row against model if table is known
+            if table in MODEL_MAP and rows:
+                from ...utils.model_helpers import validate_data_for_model
+                result = validate_data_for_model(MODEL_MAP[table], rows[0])
+                if result.extra_fields:
+                    logger.warning(f"Unknown fields (ignored): {result.extra_fields}")
+                if result.valid:
+                    logger.success(f"Sample row validates OK. Required: {result.required_fields or '(none)'}")
+                else:
+                    result.log_errors("Sample row")
+            return
+
+        # Wrap as single table definition
+        table_defs = [{"table": table, "rows": rows}]
+
     # Connect to database
     pg = get_postgres_service()
     if not pg:
@@ -399,23 +455,24 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
 
     await pg.connect()
 
+    # Start embedding worker for generating embeddings
+    if pg.embedding_worker:
+        await pg.embedding_worker.start()
+
     try:
         total_loaded = 0
 
-        for table_def in data:
+        for table_def in table_defs:
             table_name = table_def["table"]
-            key_field = table_def.get("key_field", "id")
             rows = table_def.get("rows", [])
 
             # Handle direct insert tables (non-CoreModel)
             if table_name in DIRECT_INSERT_TABLES:
                 for row_data in rows:
-                    # Add tenant_id if not present
                     if "tenant_id" not in row_data:
                         row_data["tenant_id"] = "default"
 
                     if table_name == "shared_sessions":
-                        # Insert shared_session directly
                         await pg.fetch(
                             """INSERT INTO shared_sessions
                                (session_id, owner_user_id, shared_with_user_id, tenant_id)
@@ -434,16 +491,13 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                 logger.warning(f"Unknown table: {table_name}, skipping")
                 continue
 
-            model_class = MODEL_MAP[table_name]  # Type is inferred from MODEL_MAP
+            model_class = MODEL_MAP[table_name]
 
-            for row_data in rows:
-                # Add user_id and tenant_id only if explicitly provided
-                # Default is public (None) - data is shared/visible to all
-                # Pass --user-id to scope data privately to a specific user
-                if "user_id" not in row_data and user_id is not None:
-                    row_data["user_id"] = user_id
+            for row_idx, row_data in enumerate(rows):
+                # user_id stays NULL for public data (accessible by any user)
+                # Only set tenant_id for scoping - the --user-id flag controls tenant scope
                 if "tenant_id" not in row_data and user_id is not None:
-                    row_data["tenant_id"] = row_data.get("user_id", user_id)
+                    row_data["tenant_id"] = user_id
 
                 # Convert graph_edges to InlineEdge format if present
                 if "graph_edges" in row_data:
@@ -452,30 +506,40 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                         for edge in row_data["graph_edges"]
                     ]
 
-                # Convert any ISO timestamp strings with Z suffix to naive datetime
-                # This handles fields like starts_timestamp, ends_timestamp, etc.
+                # Convert ISO timestamp strings
                 from ...utils.date_utils import parse_iso
                 for key, value in list(row_data.items()):
                     if isinstance(value, str) and (key.endswith("_timestamp") or key.endswith("_at")):
                         try:
                             row_data[key] = parse_iso(value)
                         except (ValueError, TypeError):
-                            pass  # Not a valid datetime string, leave as-is
+                            pass
 
-                # Create model instance and upsert via repository
                 from ...services.postgres.repository import Repository
+                from ...utils.model_helpers import validate_data_for_model
+
+                result = validate_data_for_model(model_class, row_data)
+                if not result.valid:
+                    result.log_errors(f"Row {row_idx + 1} ({table_name})")
+                    raise click.Abort()
 
-                instance = model_class(**row_data)
-                repo = Repository(model_class, table_name, pg)  # Type inferred from MODEL_MAP
-                await repo.upsert(instance)  # type: ignore[arg-type]
+                repo = Repository(model_class, table_name, pg)
+                await repo.upsert(result.instance)  # type: ignore[arg-type]
                 total_loaded += 1
 
-                # Log based on model type
-                name = getattr(instance, 'name', getattr(instance, 'id', '?'))
+                name = getattr(result.instance, 'name', getattr(result.instance, 'id', '?'))
                 logger.success(f"Loaded {table_name[:-1]}: {name}")
 
         logger.success(f"Data loaded successfully! Total rows: {total_loaded}")
 
+        # Wait for embeddings to complete
+        if pg.embedding_worker and pg.embedding_worker.running:
+            queue_size = pg.embedding_worker.task_queue.qsize()
+            if queue_size > 0:
+                logger.info(f"Waiting for {queue_size} embeddings to complete...")
+            await pg.embedding_worker.stop()
+            logger.success("Embeddings generated successfully")
+
     finally:
         await pg.disconnect()
 

rem/models/entities/__init__.py

@@ -39,6 +39,7 @@ from .shared_session import (
     SharedWithMeResponse,
     SharedWithMeSummary,
 )
+from .subscriber import Subscriber, SubscriberOrigin, SubscriberStatus
 from .user import User, UserTier
 
 __all__ = [
@@ -56,6 +57,9 @@ __all__ = [
     "FeedbackCategory",
     "User",
     "UserTier",
+    "Subscriber",
+    "SubscriberStatus",
+    "SubscriberOrigin",
     "File",
     "Moment",
     "Schema",