remdb 0.3.7__py3-none-any.whl

rem/services/fs/__init__.py

@@ -0,0 +1,62 @@
+"""
+File system abstraction layer for REM.
+
+Provides unified interface for:
+- S3 operations (via S3Provider)
+- Local file operations (via LocalProvider)
+- Columnar data with Polars
+- Multiple file formats (yaml, json, csv, text, pdf, images, etc.)
+- Standardized path naming conventions
+
+Usage:
+    from rem.services.fs import FS, generate_presigned_url, get_uploads_path
+
+    fs = FS()
+
+    # Read from S3 or local
+    data = fs.read("s3://bucket/file.csv")
+    data = fs.read("/local/path/file.parquet")
+
+    # Write to S3 or local
+    fs.write("s3://bucket/output.json", {"key": "value"})
+
+    # Generate presigned URLs for S3
+    url = generate_presigned_url("s3://bucket/file.pdf", expiry=3600)
+
+    # Use standardized paths
+    upload_dir = get_uploads_path(user_id="user-123")
+    fs.write(f"{upload_dir}/data.json", {"key": "value"})
+"""
+
+from rem.services.fs.provider import FS
+from rem.services.fs.service import FileSystemService
+from rem.services.fs.s3_provider import S3Provider, generate_presigned_url
+from rem.services.fs.local_provider import LocalProvider
+from rem.services.fs.paths import (
+    get_rem_home,
+    get_base_uri,
+    get_uploads_path,
+    get_versioned_path,
+    get_user_path,
+    get_temp_path,
+    ensure_dir_exists,
+    join_path,
+)
+
+__all__ = [
+    # Core providers
+    "FS",
+    "FileSystemService",
+    "S3Provider",
+    "LocalProvider",
+    "generate_presigned_url",
+    # Path conventions
+    "get_rem_home",
+    "get_base_uri",
+    "get_uploads_path",
+    "get_versioned_path",
+    "get_user_path",
+    "get_temp_path",
+    "ensure_dir_exists",
+    "join_path",
+]

rem/services/fs/examples.py

@@ -0,0 +1,206 @@
+"""
+Example usage of the REM file system service.
+
+Run these examples to test the fs service functionality.
+"""
+
+from rem.services.fs import FS, generate_presigned_url
+
+
+def example_basic_operations():
+    """Basic read/write operations."""
+    fs = FS()
+
+    # Write and read JSON
+    data = {"name": "rem", "version": "0.1.0", "features": ["s3", "local", "polars"]}
+    fs.write("/tmp/test.json", data)
+    loaded = fs.read("/tmp/test.json")
+    print(f"JSON: {loaded}")
+
+    # Write and read YAML
+    config = {"database": {"host": "localhost", "port": 5432}, "cache": {"enabled": True}}
+    fs.write("/tmp/config.yaml", config)
+    loaded_config = fs.read("/tmp/config.yaml")
+    print(f"YAML: {loaded_config}")
+
+    # Write and read text
+    fs.write("/tmp/readme.md", "# Hello REM\n\nFile system abstraction.")
+    text = fs.read("/tmp/readme.md")
+    print(f"Text: {text[:50]}...")
+
+
+def example_columnar_data():
+    """Columnar data with Polars."""
+    try:
+        import polars as pl
+    except ImportError:
+        print("Polars not installed - skipping columnar examples")
+        print("Install with: uv add polars")
+        return
+
+    fs = FS()
+
+    # Create sample dataframe
+    df = pl.DataFrame({
+        "id": [1, 2, 3, 4, 5],
+        "name": ["Alice", "Bob", "Carol", "Dave", "Eve"],
+        "score": [95.5, 87.2, 92.8, 78.3, 88.9],
+    })
+
+    # Write to CSV
+    fs.write("/tmp/data.csv", df)
+    csv_df = fs.read("/tmp/data.csv", use_polars=True)
+    print(f"CSV DataFrame:\n{csv_df}")
+
+    # Write to Parquet
+    fs.write("/tmp/data.parquet", df)
+    parquet_df = fs.read("/tmp/data.parquet")
+    print(f"Parquet DataFrame:\n{parquet_df}")
+
+
+def example_image_operations():
+    """Image read/write operations."""
+    try:
+        from PIL import Image
+        import numpy as np
+    except ImportError:
+        print("Pillow not installed - skipping image examples")
+        print("Install with: uv add pillow")
+        return
+
+    fs = FS()
+
+    # Create sample image
+    img_array = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8)
+    img = Image.fromarray(img_array)
+
+    # Write image
+    fs.write("/tmp/test.png", img)
+
+    # Read image
+    loaded_img = fs.read_image("/tmp/test.png")
+    print(f"Image: {loaded_img.size} {loaded_img.mode}")
+
+
+def example_file_operations():
+    """File listing, copying, deleting."""
+    fs = FS()
+
+    # Create test files
+    fs.write("/tmp/rem_test/file1.txt", "Content 1")
+    fs.write("/tmp/rem_test/file2.txt", "Content 2")
+    fs.write("/tmp/rem_test/subdir/file3.txt", "Content 3")
+
+    # List files
+    files = fs.ls("/tmp/rem_test/")
+    print(f"Files: {files}")
+
+    # List directories
+    dirs = fs.ls_dirs("/tmp/rem_test/")
+    print(f"Directories: {dirs}")
+
+    # Copy file
+    fs.copy("/tmp/rem_test/file1.txt", "/tmp/rem_test/file1_copy.txt")
+
+    # Check existence
+    exists = fs.exists("/tmp/rem_test/file1_copy.txt")
+    print(f"Copy exists: {exists}")
+
+    # Delete files (with safety limit)
+    deleted = fs.delete("/tmp/rem_test/", limit=10)
+    print(f"Deleted {len(deleted)} files")
+
+
+def example_s3_operations():
+    """S3 operations (requires configured S3 bucket)."""
+    fs = FS()
+
+    # NOTE: Update bucket name to your test bucket
+    test_bucket = "s3://rem-test-bucket"
+
+    try:
+        # Write to S3
+        data = {"timestamp": "2025-01-01T00:00:00Z", "event": "test"}
+        fs.write(f"{test_bucket}/test.json", data)
+
+        # Read from S3
+        loaded = fs.read(f"{test_bucket}/test.json")
+        print(f"S3 JSON: {loaded}")
+
+        # Generate presigned URL
+        url = generate_presigned_url(f"{test_bucket}/test.json", expiry=3600)
+        print(f"Presigned URL: {url[:100]}...")
+
+        # Copy from S3 to local
+        fs.copy(f"{test_bucket}/test.json", "/tmp/downloaded.json")
+        print("Downloaded from S3 to /tmp/downloaded.json")
+
+        # Upload from local to S3
+        fs.write("/tmp/upload.txt", "Upload test")
+        fs.copy("/tmp/upload.txt", f"{test_bucket}/uploaded.txt")
+        print("Uploaded to S3")
+
+        # List S3 files
+        files = fs.ls(f"{test_bucket}/")
+        print(f"S3 files: {files[:5]}")  # First 5
+
+    except Exception as e:
+        print(f"S3 operations failed (bucket may not exist): {e}")
+        print("Configure S3 settings in .env and create test bucket")
+
+
+def example_presigned_urls():
+    """Generate presigned URLs for S3 access."""
+    # Download URL
+    download_url = generate_presigned_url(
+        "s3://rem-bucket/document.pdf",
+        expiry=3600  # 1 hour
+    )
+    print(f"Download URL: {download_url[:100]}...")
+
+    # Upload URL
+    upload_url = generate_presigned_url(
+        "s3://rem-bucket/upload.pdf",
+        expiry=300,  # 5 minutes
+        for_upload=True
+    )
+    print(f"Upload URL (PUT): {upload_url[:100]}...")
+
+
+def main():
+    """Run all examples."""
+    print("=" * 60)
+    print("REM File System Service Examples")
+    print("=" * 60)
+
+    print("\n1. Basic Operations")
+    print("-" * 60)
+    example_basic_operations()
+
+    print("\n2. Columnar Data (Polars)")
+    print("-" * 60)
+    example_columnar_data()
+
+    print("\n3. Image Operations")
+    print("-" * 60)
+    example_image_operations()
+
+    print("\n4. File Operations")
+    print("-" * 60)
+    example_file_operations()
+
+    print("\n5. S3 Operations")
+    print("-" * 60)
+    example_s3_operations()
+
+    print("\n6. Presigned URLs")
+    print("-" * 60)
+    example_presigned_urls()
+
+    print("\n" + "=" * 60)
+    print("Examples complete!")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

rem/services/fs/examples_paths.py

@@ -0,0 +1,204 @@
+"""
+Examples demonstrating REM filesystem path conventions.
+
+Run with:
+    python -m rem.services.fs.examples_paths
+"""
+
+from datetime import datetime, date
+from rem.services.fs import (
+    FS,
+    get_rem_home,
+    get_base_uri,
+    get_uploads_path,
+    get_versioned_path,
+    get_user_path,
+    get_temp_path,
+    ensure_dir_exists,
+    join_path,
+)
+
+
+def example_basic_paths():
+    """Demonstrate basic path generation."""
+    print("=== Basic Paths ===\n")
+
+    # REM home directory
+    rem_home = get_rem_home()
+    print(f"REM_HOME: {rem_home}")
+
+    # Base URI (auto-detects local vs S3 based on environment)
+    base_local = get_base_uri(use_s3=False)
+    print(f"Base URI (local): {base_local}")
+
+    base_s3 = get_base_uri(use_s3=True)
+    print(f"Base URI (S3): {base_s3}")
+    print()
+
+
+def example_uploads_paths():
+    """Demonstrate uploads path generation."""
+    print("=== Uploads Paths ===\n")
+
+    # System uploads (no user_id)
+    system_upload = get_uploads_path()
+    print(f"System upload (today): {system_upload}")
+
+    # User-specific uploads
+    user_upload = get_uploads_path(user_id="user-123")
+    print(f"User upload (user-123): {user_upload}")
+
+    # With specific date
+    specific_date = date(2025, 1, 15)
+    dated_upload = get_uploads_path(user_id="user-456", dt=specific_date)
+    print(f"Dated upload (2025-01-15): {dated_upload}")
+
+    # With time included
+    now = datetime.now()
+    timed_upload = get_uploads_path(user_id="user-789", dt=now, include_time=True)
+    print(f"Timed upload (with hour/min): {timed_upload}")
+
+    # S3 version
+    s3_upload = get_uploads_path(user_id="user-123", use_s3=True)
+    print(f"S3 upload: {s3_upload}")
+    print()
+
+
+def example_versioned_paths():
+    """Demonstrate versioned resource paths."""
+    print("=== Versioned Paths ===\n")
+
+    # Schemas
+    schema_path = get_versioned_path("schemas", "user-schema")
+    print(f"Schema path: {schema_path}")
+
+    # Agents
+    agent_path = get_versioned_path("agents", "query-agent", version="v2")
+    print(f"Agent path (v2): {agent_path}")
+
+    # Tools
+    tool_path = get_versioned_path("tools", "web-scraper")
+    print(f"Tool path: {tool_path}")
+
+    # Datasets
+    dataset_path = get_versioned_path("datasets", "training-data")
+    print(f"Dataset path: {dataset_path}")
+    print()
+
+
+def example_user_paths():
+    """Demonstrate user-scoped paths."""
+    print("=== User Paths ===\n")
+
+    # User root
+    user_root = get_user_path("user-123")
+    print(f"User root: {user_root}")
+
+    # User documents
+    user_docs = get_user_path("user-123", "documents")
+    print(f"User documents: {user_docs}")
+
+    # User images
+    user_images = get_user_path("user-456", "images")
+    print(f"User images: {user_images}")
+    print()
+
+
+def example_temp_paths():
+    """Demonstrate temporary file paths."""
+    print("=== Temp Paths ===\n")
+
+    # Default temp
+    temp_default = get_temp_path()
+    print(f"Default temp: {temp_default}")
+
+    # Processing temp
+    temp_processing = get_temp_path("processing")
+    print(f"Processing temp: {temp_processing}")
+
+    # Conversion temp
+    temp_conversion = get_temp_path("conversion")
+    print(f"Conversion temp: {temp_conversion}")
+    print()
+
+
+def example_practical_usage():
+    """Demonstrate practical usage with FS."""
+    print("=== Practical Usage ===\n")
+
+    fs = FS()
+
+    # Get upload path for today
+    upload_dir = get_uploads_path(user_id="user-123")
+    print(f"Upload directory: {upload_dir}")
+
+    # Ensure directory exists (local only)
+    ensure_dir_exists(upload_dir)
+
+    # Write file to upload directory
+    file_path = join_path(upload_dir, "test-data.json", is_s3=False)
+    print(f"Writing to: {file_path}")
+
+    data = {
+        "message": "Hello from REM",
+        "timestamp": datetime.now().isoformat(),
+        "user_id": "user-123",
+    }
+    fs.write(file_path, data)
+    print(f"✓ Written successfully")
+
+    # Read it back
+    read_data = fs.read(file_path)
+    print(f"✓ Read back: {read_data}")
+
+    # List files in directory
+    files = fs.ls(upload_dir)
+    print(f"✓ Files in directory: {files}")
+    print()
+
+
+def example_path_components():
+    """Demonstrate path structure breakdown."""
+    print("=== Path Structure ===\n")
+
+    example_path = get_uploads_path(
+        user_id="user-123", dt=datetime(2025, 1, 19, 14, 30), include_time=True
+    )
+
+    print(f"Full path: {example_path}")
+    print("\nBreakdown:")
+    print("  base_uri/rem/v1/uploads/user-123/2025/01/19/14_30")
+    print("  │        │  │  │       │        │    │  │  │")
+    print("  │        │  │  │       │        │    │  │  └─ minute")
+    print("  │        │  │  │       │        │    │  └──── hour")
+    print("  │        │  │  │       │        │    └─────── day")
+    print("  │        │  │  │       │        └──────────── month")
+    print("  │        │  │  │       └───────────────────── year")
+    print("  │        │  │  └───────────────────────────── user_id (or 'system')")
+    print("  │        │  └──────────────────────────────── category")
+    print("  │        └─────────────────────────────────── version")
+    print("  └──────────────────────────────────────────── namespace")
+    print()
+
+
+def main():
+    """Run all examples."""
+    print("\n" + "=" * 70)
+    print("REM Filesystem Path Conventions Examples")
+    print("=" * 70 + "\n")
+
+    example_basic_paths()
+    example_uploads_paths()
+    example_versioned_paths()
+    example_user_paths()
+    example_temp_paths()
+    example_path_components()
+    example_practical_usage()
+
+    print("=" * 70)
+    print("Done! Check $REM_HOME/fs/rem/v1/uploads/ for created files")
+    print("=" * 70 + "\n")
+
+
+if __name__ == "__main__":
+    main()