remdb 0.3.7__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/utils/date_utils.py

@@ -0,0 +1,228 @@
+"""
+Centralized datetime utilities for consistent UTC-naive datetime handling.
+
+IMPORTANT: REM uses UTC-naive datetimes throughout the codebase.
+PostgreSQL stores TIMESTAMP WITHOUT TIME ZONE, so all Python datetime
+operations should use UTC-naive datetimes to avoid comparison errors.
+
+Convention:
+- All timestamps are implicitly UTC
+- Use utc_now() instead of datetime.utcnow() or datetime.now(timezone.utc)
+- Use parse_iso() to parse ISO format strings (handles "Z" suffix)
+- Use to_iso() to format datetimes as ISO strings
+
+See CLAUDE.md Section 1 (Datetime Convention) for details.
+"""
+
+from datetime import UTC, datetime, timedelta
+from typing import Optional
+
+
+def utc_now() -> datetime:
+    """
+    Get current UTC time as a naive datetime.
+
+    Returns:
+        UTC-naive datetime representing current time.
+
+    Example:
+        >>> now = utc_now()
+        >>> now.tzinfo is None
+        True
+    """
+    return datetime.now(UTC).replace(tzinfo=None)
+
+
+def to_iso(dt: datetime) -> str:
+    """
+    Convert datetime to ISO 8601 format string.
+
+    Args:
+        dt: Datetime to format (should be UTC-naive)
+
+    Returns:
+        ISO format string (e.g., "2024-01-15T10:30:00")
+
+    Example:
+        >>> dt = datetime(2024, 1, 15, 10, 30, 0)
+        >>> to_iso(dt)
+        '2024-01-15T10:30:00'
+    """
+    return dt.isoformat()
+
+
+def to_iso_with_z(dt: datetime) -> str:
+    """
+    Convert datetime to ISO 8601 format with Z suffix.
+
+    Use this when interfacing with external APIs that expect
+    the Z suffix to indicate UTC.
+
+    Args:
+        dt: Datetime to format (should be UTC-naive)
+
+    Returns:
+        ISO format string with Z suffix (e.g., "2024-01-15T10:30:00Z")
+    """
+    return dt.isoformat() + "Z"
+
+
+def parse_iso(iso_string: str) -> datetime:
+    """
+    Parse ISO 8601 format string to UTC-naive datetime.
+
+    Handles:
+    - Standard ISO format: "2024-01-15T10:30:00"
+    - Z suffix: "2024-01-15T10:30:00Z"
+    - Timezone offset: "2024-01-15T10:30:00+00:00" (converts to naive)
+    - Microseconds: "2024-01-15T10:30:00.123456"
+
+    Args:
+        iso_string: ISO format datetime string
+
+    Returns:
+        UTC-naive datetime
+
+    Raises:
+        ValueError: If string cannot be parsed
+
+    Example:
+        >>> parse_iso("2024-01-15T10:30:00Z")
+        datetime.datetime(2024, 1, 15, 10, 30)
+        >>> parse_iso("2024-01-15T10:30:00+00:00")
+        datetime.datetime(2024, 1, 15, 10, 30)
+    """
+    # Handle Z suffix (replace with +00:00 for fromisoformat)
+    if iso_string.endswith("Z"):
+        iso_string = iso_string[:-1] + "+00:00"
+
+    # Parse the ISO string
+    dt = datetime.fromisoformat(iso_string)
+
+    # Convert to naive UTC if timezone-aware
+    if dt.tzinfo is not None:
+        # Convert to UTC and strip timezone
+        from datetime import timezone
+        dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
+
+    return dt
+
+
+def parse_iso_safe(iso_string: Optional[str], default: Optional[datetime] = None) -> Optional[datetime]:
+    """
+    Safely parse ISO string, returning default on failure.
+
+    Args:
+        iso_string: ISO format string or None
+        default: Default value if parsing fails
+
+    Returns:
+        Parsed datetime or default value
+    """
+    if not iso_string:
+        return default
+    try:
+        return parse_iso(iso_string)
+    except (ValueError, TypeError):
+        return default
+
+
+def format_timestamp(dt: Optional[datetime] = None) -> str:
+    """
+    Format datetime for display/logging.
+
+    Args:
+        dt: Datetime to format (defaults to current UTC time)
+
+    Returns:
+        Formatted string like "2024-01-15 10:30:00 UTC"
+    """
+    if dt is None:
+        dt = utc_now()
+    return dt.strftime("%Y-%m-%d %H:%M:%S") + " UTC"
+
+
+def format_timestamp_compact(dt: Optional[datetime] = None) -> str:
+    """
+    Format datetime as compact string for filenames/IDs.
+
+    Args:
+        dt: Datetime to format (defaults to current UTC time)
+
+    Returns:
+        Formatted string like "20240115_103000"
+    """
+    if dt is None:
+        dt = utc_now()
+    return dt.strftime("%Y%m%d_%H%M%S")
+
+
+def format_timestamp_for_experiment(dt: Optional[datetime] = None) -> str:
+    """
+    Format datetime for experiment names.
+
+    Args:
+        dt: Datetime to format (defaults to current UTC time)
+
+    Returns:
+        Formatted string like "20240115-103000"
+    """
+    if dt is None:
+        dt = utc_now()
+    return dt.strftime("%Y%m%d-%H%M%S")
+
+
+def days_ago(days: int) -> datetime:
+    """
+    Get datetime N days ago from now.
+
+    Args:
+        days: Number of days ago
+
+    Returns:
+        UTC-naive datetime
+    """
+    return utc_now() - timedelta(days=days)
+
+
+def hours_ago(hours: int) -> datetime:
+    """
+    Get datetime N hours ago from now.
+
+    Args:
+        hours: Number of hours ago
+
+    Returns:
+        UTC-naive datetime
+    """
+    return utc_now() - timedelta(hours=hours)
+
+
+def is_within_hours(dt: datetime, hours: int) -> bool:
+    """
+    Check if datetime is within N hours of now.
+
+    Args:
+        dt: Datetime to check (should be UTC-naive)
+        hours: Number of hours
+
+    Returns:
+        True if dt is within the time window
+    """
+    cutoff = hours_ago(hours)
+    return dt >= cutoff
+
+
+def is_within_days(dt: datetime, days: int) -> bool:
+    """
+    Check if datetime is within N days of now.
+
+    Args:
+        dt: Datetime to check (should be UTC-naive)
+        days: Number of days
+
+    Returns:
+        True if dt is within the time window
+    """
+    cutoff = days_ago(days)
+    return dt >= cutoff

rem/utils/embeddings.py

@@ -20,7 +20,6 @@ Usage:
     embeddings = generate_embeddings("openai:text-embedding-3-small", texts)
 """
 
-import os
 from typing import Any, cast
 
 import requests
@@ -31,6 +30,16 @@ from tenacity import (
     wait_exponential,
 )
 
+from rem.utils.constants import (
+    HTTP_TIMEOUT_LONG,
+    OPENAI_EMBEDDING_DIMS_SMALL,
+    OPENAI_EMBEDDING_DIMS_LARGE,
+    VOYAGE_EMBEDDING_DIMS,
+    RETRY_BACKOFF_MULTIPLIER,
+    RETRY_BACKOFF_MIN,
+    RETRY_BACKOFF_MAX,
+)
+
 
 class EmbeddingError(Exception):
     """Base exception for embedding generation errors."""
@@ -166,7 +175,11 @@ def _create_retry_decorator(max_retries: int):
     return retry(
         retry=retry_if_exception_type(RateLimitError),
         stop=stop_after_attempt(max_retries),
-        wait=wait_exponential(multiplier=1, min=1, max=60),
+        wait=wait_exponential(
+            multiplier=RETRY_BACKOFF_MULTIPLIER,
+            min=RETRY_BACKOFF_MIN,
+            max=RETRY_BACKOFF_MAX,
+        ),
         reraise=True,
     )
 
@@ -234,7 +247,7 @@ def _generate_openai_embeddings(
     }
 
     try:
-        response = requests.post(url, json=payload, headers=headers, timeout=60)
+        response = requests.post(url, json=payload, headers=headers, timeout=HTTP_TIMEOUT_LONG)
 
         # Handle rate limits
         if response.status_code == 429:
@@ -334,7 +347,7 @@ def _generate_voyage_embeddings(
     }
 
     try:
-        response = requests.post(url, json=payload, headers=headers, timeout=60)
+        response = requests.post(url, json=payload, headers=headers, timeout=HTTP_TIMEOUT_LONG)
 
         # Handle rate limits
         if response.status_code == 429:

rem/utils/files.py

@@ -0,0 +1,167 @@
+"""
+File utilities for consistent file handling throughout REM.
+
+Provides context managers and helpers for temporary file operations,
+ensuring proper cleanup and consistent patterns.
+"""
+
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Generator, Optional
+
+from loguru import logger
+
+
+@contextmanager
+def temp_file_from_bytes(
+    content: bytes,
+    suffix: str = "",
+    prefix: str = "rem_",
+    dir: Optional[str] = None,
+) -> Generator[Path, None, None]:
+    """
+    Create a temporary file from bytes, yield path, cleanup automatically.
+
+    This context manager ensures proper cleanup of temporary files even
+    if an exception occurs during processing.
+
+    Args:
+        content: Bytes to write to the temporary file
+        suffix: File extension (e.g., ".pdf", ".wav")
+        prefix: Prefix for the temp file name
+        dir: Directory for temp file (uses system temp if None)
+
+    Yields:
+        Path to the temporary file
+
+    Example:
+        >>> with temp_file_from_bytes(pdf_bytes, suffix=".pdf") as tmp_path:
+        ...     result = process_pdf(tmp_path)
+        # File is automatically cleaned up after the block
+
+    Note:
+        The file is created with delete=False so we control cleanup.
+        This allows the file to be read by external processes.
+    """
+    tmp_path: Optional[Path] = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            suffix=suffix,
+            prefix=prefix,
+            dir=dir,
+            delete=False,
+        ) as tmp:
+            tmp.write(content)
+            tmp_path = Path(tmp.name)
+
+        yield tmp_path
+
+    finally:
+        if tmp_path is not None:
+            try:
+                tmp_path.unlink(missing_ok=True)
+            except Exception as e:
+                logger.warning(f"Failed to cleanup temp file {tmp_path}: {e}")
+
+
+@contextmanager
+def temp_file_empty(
+    suffix: str = "",
+    prefix: str = "rem_",
+    dir: Optional[str] = None,
+) -> Generator[Path, None, None]:
+    """
+    Create an empty temporary file, yield path, cleanup automatically.
+
+    Useful when you need to write to a file after creation or when
+    an external process will write to the file.
+
+    Args:
+        suffix: File extension
+        prefix: Prefix for the temp file name
+        dir: Directory for temp file
+
+    Yields:
+        Path to the empty temporary file
+    """
+    tmp_path: Optional[Path] = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            suffix=suffix,
+            prefix=prefix,
+            dir=dir,
+            delete=False,
+        ) as tmp:
+            tmp_path = Path(tmp.name)
+
+        yield tmp_path
+
+    finally:
+        if tmp_path is not None:
+            try:
+                tmp_path.unlink(missing_ok=True)
+            except Exception as e:
+                logger.warning(f"Failed to cleanup temp file {tmp_path}: {e}")
+
+
+@contextmanager
+def temp_directory(
+    prefix: str = "rem_",
+    dir: Optional[str] = None,
+) -> Generator[Path, None, None]:
+    """
+    Create a temporary directory, yield path, cleanup automatically.
+
+    Args:
+        prefix: Prefix for the temp directory name
+        dir: Parent directory for temp directory
+
+    Yields:
+        Path to the temporary directory
+    """
+    import shutil
+
+    tmp_dir: Optional[Path] = None
+    try:
+        tmp_dir = Path(tempfile.mkdtemp(prefix=prefix, dir=dir))
+        yield tmp_dir
+
+    finally:
+        if tmp_dir is not None:
+            try:
+                shutil.rmtree(tmp_dir, ignore_errors=True)
+            except Exception as e:
+                logger.warning(f"Failed to cleanup temp directory {tmp_dir}: {e}")
+
+
+def ensure_parent_exists(path: Path) -> Path:
+    """
+    Ensure parent directory exists, creating if necessary.
+
+    Args:
+        path: File path whose parent should exist
+
+    Returns:
+        The original path (for chaining)
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def safe_delete(path: Path) -> bool:
+    """
+    Safely delete a file, returning success status.
+
+    Args:
+        path: Path to delete
+
+    Returns:
+        True if deleted or didn't exist, False on error
+    """
+    try:
+        path.unlink(missing_ok=True)
+        return True
+    except Exception as e:
+        logger.warning(f"Failed to delete {path}: {e}")
+        return False

rem/utils/mime_types.py

@@ -0,0 +1,158 @@
+"""
+Centralized MIME type mappings for file format detection.
+
+Provides bidirectional mappings between file extensions and MIME types.
+Use these constants throughout the codebase instead of inline dictionaries.
+"""
+
+# Extension to MIME type mapping (extension includes leading dot)
+EXTENSION_TO_MIME: dict[str, str] = {
+    # Images
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+    ".tiff": "image/tiff",
+    ".svg": "image/svg+xml",
+    # Documents
+    ".pdf": "application/pdf",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".doc": "application/msword",
+    ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    ".ppt": "application/vnd.ms-powerpoint",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ".xls": "application/vnd.ms-excel",
+    # Audio
+    ".wav": "audio/wav",
+    ".mp3": "audio/mpeg",
+    ".m4a": "audio/x-m4a",
+    ".flac": "audio/flac",
+    ".ogg": "audio/ogg",
+    ".aac": "audio/aac",
+    # Video
+    ".mp4": "video/mp4",
+    ".webm": "video/webm",
+    ".avi": "video/x-msvideo",
+    ".mov": "video/quicktime",
+    # Text/Code
+    ".txt": "text/plain",
+    ".md": "text/markdown",
+    ".markdown": "text/markdown",
+    ".json": "application/json",
+    ".yaml": "application/x-yaml",
+    ".yml": "application/x-yaml",
+    ".xml": "application/xml",
+    ".html": "text/html",
+    ".css": "text/css",
+    ".js": "application/javascript",
+    ".py": "text/x-python",
+    ".ts": "application/typescript",
+    ".csv": "text/csv",
+}
+
+# MIME type to extension mapping (reverse of above, preferring shorter extensions)
+MIME_TO_EXTENSION: dict[str, str] = {
+    # Images
+    "image/png": ".png",
+    "image/jpeg": ".jpg",
+    "image/gif": ".gif",
+    "image/webp": ".webp",
+    "image/bmp": ".bmp",
+    "image/tiff": ".tiff",
+    "image/svg+xml": ".svg",
+    # Documents
+    "application/pdf": ".pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+    "application/msword": ".doc",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+    "application/vnd.ms-powerpoint": ".ppt",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+    "application/vnd.ms-excel": ".xls",
+    # Audio
+    "audio/wav": ".wav",
+    "audio/mpeg": ".mp3",
+    "audio/x-m4a": ".m4a",
+    "audio/mp4": ".m4a",
+    "audio/flac": ".flac",
+    "audio/ogg": ".ogg",
+    "audio/aac": ".aac",
+    # Video
+    "video/mp4": ".mp4",
+    "video/webm": ".webm",
+    "video/x-msvideo": ".avi",
+    "video/quicktime": ".mov",
+    # Text/Code
+    "text/plain": ".txt",
+    "text/markdown": ".md",
+    "application/json": ".json",
+    "application/x-yaml": ".yaml",
+    "application/xml": ".xml",
+    "text/html": ".html",
+    "text/css": ".css",
+    "application/javascript": ".js",
+    "text/x-python": ".py",
+    "application/typescript": ".ts",
+    "text/csv": ".csv",
+}
+
+# Grouped by category for convenience
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".svg"}
+DOCUMENT_EXTENSIONS = {".pdf", ".docx", ".doc", ".pptx", ".ppt", ".xlsx", ".xls"}
+AUDIO_EXTENSIONS = {".wav", ".mp3", ".m4a", ".flac", ".ogg", ".aac"}
+VIDEO_EXTENSIONS = {".mp4", ".webm", ".avi", ".mov"}
+TEXT_EXTENSIONS = {".txt", ".md", ".markdown", ".json", ".yaml", ".yml", ".xml", ".html", ".css", ".js", ".py", ".ts", ".csv"}
+
+
+def get_extension(mime_type: str, default: str = ".bin") -> str:
+    """
+    Get file extension for a MIME type.
+
+    Args:
+        mime_type: MIME type string (e.g., "image/png")
+        default: Default extension if MIME type not found
+
+    Returns:
+        File extension with leading dot (e.g., ".png")
+    """
+    return MIME_TO_EXTENSION.get(mime_type, default)
+
+
+def get_mime_type(extension: str, default: str = "application/octet-stream") -> str:
+    """
+    Get MIME type for a file extension.
+
+    Args:
+        extension: File extension with or without leading dot
+        default: Default MIME type if extension not found
+
+    Returns:
+        MIME type string (e.g., "image/png")
+    """
+    # Normalize extension to have leading dot
+    ext = extension if extension.startswith(".") else f".{extension}"
+    return EXTENSION_TO_MIME.get(ext.lower(), default)
+
+
+def is_image(extension_or_mime: str) -> bool:
+    """Check if extension or MIME type represents an image."""
+    if extension_or_mime.startswith("."):
+        return extension_or_mime.lower() in IMAGE_EXTENSIONS
+    return extension_or_mime.startswith("image/")
+
+
+def is_audio(extension_or_mime: str) -> bool:
+    """Check if extension or MIME type represents audio."""
+    if extension_or_mime.startswith("."):
+        return extension_or_mime.lower() in AUDIO_EXTENSIONS
+    return extension_or_mime.startswith("audio/")
+
+
+def is_document(extension_or_mime: str) -> bool:
+    """Check if extension or MIME type represents a document."""
+    if extension_or_mime.startswith("."):
+        return extension_or_mime.lower() in DOCUMENT_EXTENSIONS
+    # Check common document MIME types
+    doc_mimes = {"application/pdf", "application/msword"}
+    return extension_or_mime in doc_mimes or "officedocument" in extension_or_mime