geek-cafe-saas-sdk 0.6.0__tar.gz → 0.7.0__tar.gz

geek_cafe_saas_sdk-0.7.0/LINEAGE_IMPLEMENTATION_SUMMARY.md

@@ -0,0 +1,335 @@
+# File Lineage Implementation Summary
+
+## Overview
+
+File lineage tracking has been successfully implemented in the Geek Cafe SaaS SDK. This feature allows tracking of file transformations through data processing pipelines.
+
+**Implementation Date:** October 2025
+
+---
+
+## What Was Added
+
+### 1. File Model Extensions
+
+**File:** `src/geek_cafe_saas_sdk/domains/files/models/file.py`
+
+**New Fields:**
+- `file_role`: str - Role in lineage chain ("standalone", "original", "main", "derived")
+- `original_file_id`: str | None - Root file in lineage chain
+- `parent_file_id`: str | None - Immediate parent file
+- `transformation_type`: str | None - Type of transformation ("convert", "clean", "process")
+- `transformation_operation`: str | None - Specific operation name
+- `transformation_metadata`: dict | None - Additional operation details
+- `derived_file_count`: int - Number of files derived from this one
+
+**New Properties:**
+- All fields have getter/setter properties with validation
+
+**New Helper Methods:**
+- `has_lineage()` - Check if file participates in lineage
+- `is_original()` - Check if this is an original file
+- `is_main()` - Check if this is a main file
+- `is_derived()` - Check if this is a derived file
+- `is_standalone()` - Check if this is a standalone file (no lineage)
+- `increment_derived_count()` - Increment derived file counter
+
+---
+
+### 2. FileLineageService
+
+**File:** `src/geek_cafe_saas_sdk/domains/files/services/file_lineage_service.py`
+
+**Purpose:** Helper service for managing file lineage and transformations.
+
+**Methods:**
+
+#### `create_main_file()`
+Create a main file from an original file (e.g., XLS → CSV conversion).
+
+**Parameters:**
+- `tenant_id`, `user_id`, `original_file_id`
+- `file_name`, `file_data`, `mime_type`
+- `transformation_operation`, `transformation_metadata`
+- `directory_id` (optional)
+
+**Returns:** ServiceResult with main File
+
+#### `create_derived_file()`
+Create a derived file from a main file (e.g., data cleaning).
+
+**Parameters:**
+- `tenant_id`, `user_id`, `main_file_id`
+- `file_name`, `file_data`
+- `transformation_operation`, `transformation_metadata`
+- `directory_id` (optional)
+
+**Returns:** ServiceResult with derived File
+
+**Note:** Derived files are ALWAYS created from main files (non-chained).
+
+#### `get_lineage()`
+Get complete lineage for a file.
+
+**Returns:** Dictionary with:
+- `selected`: The selected file
+- `main`: Main file (if exists)
+- `original`: Original file (if exists)
+- `all_derived`: List of all derived files (if viewing main)
+
+#### `list_derived_files()`
+List all files derived from a main file.
+
+**Returns:** ServiceResult with list of derived Files
+
+#### `prepare_lineage_bundle()`
+Prepare bundle of files for lineage (selected, main, original).
+
+**Returns:** Dictionary with files and transformation chain metadata
+
+#### `download_lineage_bundle()`
+Download all files in lineage chain with their data.
+
+**Returns:** Dictionary with file objects and binary data
+
+---
+
+### 3. FileSystemService Updates
+
+**File:** `src/geek_cafe_saas_sdk/domains/files/services/file_system_service.py`
+
+**Changes to `create()` method:**
+
+Added optional lineage parameters:
+- `file_role`
+- `parent_file_id`
+- `original_file_id`
+- `transformation_type`
+- `transformation_operation`
+- `transformation_metadata`
+
+These parameters are set on the File model when provided.
+
+---
+
+### 4. Service Exports
+
+**File:** `src/geek_cafe_saas_sdk/domains/files/services/__init__.py`
+
+Added export for `FileLineageService` along with existing services:
+- FileSystemService
+- DirectoryService
+- FileVersionService
+- FileShareService
+- S3FileService
+- **FileLineageService** (NEW)
+
+---
+
+### 5. Model Exports
+
+**File:** `src/geek_cafe_saas_sdk/domains/files/models/__init__.py`
+
+Added exports for all file models:
+- File
+- Directory
+- FileVersion
+- FileShare
+
+---
+
+### 6. Example Code
+
+**File:** `examples/file_lineage_example.py`
+
+Complete working example demonstrating:
+1. Upload original file
+2. Convert to main file
+3. Create derived files (data cleaning)
+4. Query lineage
+5. Prepare lineage bundle
+6. Download complete bundle
+
+---
+
+## Usage Example
+
+```python
+from geek_cafe_saas_sdk.domains.files.services import (
+    FileSystemService,
+    S3FileService,
+    FileLineageService
+)
+
+# Initialize services
+file_service = FileSystemService(...)
+lineage_service = FileLineageService(
+    file_service=file_service,
+    s3_service=s3_service
+)
+
+# 1. Upload original file
+original = file_service.create(
+    tenant_id="tenant-123",
+    user_id="user-456",
+    file_name="data.xls",
+    file_data=xls_bytes,
+    mime_type="application/vnd.ms-excel",
+    file_role="original"  # Mark as original
+)
+
+# 2. Convert to main file
+main = lineage_service.create_main_file(
+    tenant_id="tenant-123",
+    user_id="user-456",
+    original_file_id=original.data.file_id,
+    file_name="data.csv",
+    file_data=csv_bytes,
+    mime_type="text/csv",
+    transformation_operation="xls_to_csv"
+)
+
+# 3. Create derived files (cleaning)
+for version in range(1, 55):
+    cleaned = lineage_service.create_derived_file(
+        tenant_id="tenant-123",
+        user_id="user-456",
+        main_file_id=main.data.file_id,
+        file_name=f"data_clean_v{version}.csv",
+        file_data=cleaned_bytes,
+        transformation_operation=f"data_cleaning_v{version}"
+    )
+
+# 4. Get lineage for lineage
+lineage = lineage_service.get_lineage(
+    file_id="cleaned_v54_id",
+    tenant_id="tenant-123",
+    user_id="user-456"
+)
+
+# 5. Prepare bundle (selected, main, original)
+bundle = lineage_service.prepare_lineage_bundle(
+    selected_file_id="cleaned_v54_id",
+    tenant_id="tenant-123",
+    user_id="user-456"
+)
+
+# Use bundle.data['selected_file'], bundle.data['main_file'], etc.
+```
+
+---
+
+## Key Features
+
+### ✅ Non-Chained Transformations
+Derived files are ALWAYS created from the main file, never from other derived files. This matches your exact use case.
+
+### ✅ Seamless Integration
+Files with and without lineage work together. Lineage is optional.
+
+### ✅ Backward Compatible
+Existing files automatically have `file_role="standalone"`. No migration needed.
+
+### ✅ Included in Queries
+When listing files, lineage information is automatically present on each file object.
+
+### ✅ Complete Audit Trail
+Track every transformation with operation names and metadata.
+
+---
+
+## Architecture
+
+```
+Original File (data.xls)
+      ↓ convert
+Main File (data.csv)
+      ↓ clean (not chained)
+      ├── Derived v1
+      ├── Derived v2
+      ├── Derived v3
+      └── Derived v54
+
+User selects v54 → System bundles: v54 + main + original
+```
+
+---
+
+## Files Modified/Created
+
+### Modified:
+1. `src/geek_cafe_saas_sdk/domains/files/models/file.py` - Added lineage fields
+2. `src/geek_cafe_saas_sdk/domains/files/services/file_system_service.py` - Added lineage parameters
+3. `src/geek_cafe_saas_sdk/domains/files/services/__init__.py` - Added exports
+4. `src/geek_cafe_saas_sdk/domains/files/models/__init__.py` - Added exports
+
+### Created:
+1. `src/geek_cafe_saas_sdk/domains/files/services/file_lineage_service.py` - New service
+2. `examples/file_lineage_example.py` - Usage example
+3. `docs/help/file-system/07-file-lineage.md` - User guide
+4. `docs/help/file-system/08-lineage-implementation.md` - Implementation guide
+5. `LINEAGE_IMPLEMENTATION_SUMMARY.md` - This file
+
+---
+
+## Documentation
+
+Complete documentation added to `docs/help/file-system/`:
+
+1. **07-file-lineage.md** - User guide for file lineage
+   - Concepts and use cases
+   - Creating lineage chains
+   - Querying lineage
+   - Bundling for lineage
+   - UI integration examples
+
+2. **08-lineage-implementation.md** - Implementation guide
+   - Architecture decision
+   - Complete code for File model extensions
+   - Complete FileLineageService implementation
+   - Usage examples
+   - Migration guide
+
+---
+
+## Testing Recommendations
+
+1. **Unit Tests** - Test FileLineageService methods
+   - `create_main_file()` with valid/invalid original files
+   - `create_derived_file()` with valid/invalid main files
+   - `get_lineage()` with various file roles
+   - `list_derived_files()` with multiple versions
+
+2. **Integration Tests** - Test full pipeline
+   - Upload → Convert → Clean → Bundle
+   - Verify lineage relationships
+   - Verify bundle contents
+
+3. **Edge Cases**
+   - Creating main from non-original file (should fail)
+   - Creating derived from non-main file (should fail)
+   - Getting lineage for standalone file
+   - Getting lineage for file with missing parent/original
+
+---
+
+## Next Steps
+
+1. **Run Tests** - Ensure all tests pass with new fields
+2. **Update Database** - No migration needed (fields have defaults)
+3. **Deploy** - Deploy updated code to your environment
+4. **Monitor** - Monitor file creation with lineage fields
+5. **Document Internal** - Add to your internal docs/wikis
+
+---
+
+## Support
+
+For questions or issues:
+- Review `docs/help/file-system/07-file-lineage.md`
+- Check `examples/file_lineage_example.py`
+- See `docs/help/file-system/08-lineage-implementation.md`
+
+---
+
+**Implementation Status:** ✅ Complete and Ready for Testing

{geek_cafe_saas_sdk-0.6.0 → geek_cafe_saas_sdk-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geek_cafe_saas_sdk
-Version: 0.6.0
+Version: 0.7.0
 Summary: Base Reusable Services for SaaS
 Project-URL: Homepage, https://github.com/geekcafe/geek-cafe-services
 Project-URL: Documentation, https://github.com/geekcafe/geek-cafe-services/blob/main/README.md
@@ -87,20 +87,20 @@ Description-Content-Type: text/markdown
 <!-- COVERAGE-BADGE:START -->
 ## Test Coverage
 
-![Tests](https://img.shields.io/badge/tests-1127%20passed-brightgreen)
-![Coverage](https://img.shields.io/badge/coverage-82.9%25-green)
+![Tests](https://img.shields.io/badge/tests-1145%20passed-brightgreen)
+![Coverage](https://img.shields.io/badge/coverage-83.1%25-green)
 
-**Overall Coverage:** 82.9% (9325/11242 statements)
+**Overall Coverage:** 83.1% (9646/11611 statements)
 
 ### Coverage Summary
 
 | Metric | Value |
 |--------|-------|
-| Total Statements | 11,242 |
-| Covered Statements | 9,325 |
-| Missing Statements | 1,917 |
-| Coverage Percentage | 82.9% |
-| Total Tests | 1127 |
+| Total Statements | 11,611 |
+| Covered Statements | 9,646 |
+| Missing Statements | 1,965 |
+| Coverage Percentage | 83.1% |
+| Total Tests | 1145 |
 | Test Status | ✅ All Passing |
 
 ### Files Needing Attention (< 80% coverage)
@@ -118,7 +118,7 @@ Description-Content-Type: text/markdown
 | 63.4% | 86 | `domains/communities/services/community_service.py` |
 | 64.0% | 41 | `domains/files/services/s3_file_service.py` |
 
-*... and 20 more files with < 80% coverage*
+*... and 22 more files with < 80% coverage*
 
 ### Running Tests
 
@@ -130,7 +130,7 @@ Description-Content-Type: text/markdown
 open reports/coverage/index.html
 ```
 
-*Last updated: 2025-10-15 22:38:06*
+*Last updated: 2025-10-16 13:22:11*
 
 ---
 

{geek_cafe_saas_sdk-0.6.0 → geek_cafe_saas_sdk-0.7.0}/README.md

@@ -12,20 +12,20 @@
 <!-- COVERAGE-BADGE:START -->
 ## Test Coverage
 
-![Tests](https://img.shields.io/badge/tests-1127%20passed-brightgreen)
-![Coverage](https://img.shields.io/badge/coverage-82.9%25-green)
+![Tests](https://img.shields.io/badge/tests-1145%20passed-brightgreen)
+![Coverage](https://img.shields.io/badge/coverage-83.1%25-green)
 
-**Overall Coverage:** 82.9% (9325/11242 statements)
+**Overall Coverage:** 83.1% (9646/11611 statements)
 
 ### Coverage Summary
 
 | Metric | Value |
 |--------|-------|
-| Total Statements | 11,242 |
-| Covered Statements | 9,325 |
-| Missing Statements | 1,917 |
-| Coverage Percentage | 82.9% |
-| Total Tests | 1127 |
+| Total Statements | 11,611 |
+| Covered Statements | 9,646 |
+| Missing Statements | 1,965 |
+| Coverage Percentage | 83.1% |
+| Total Tests | 1145 |
 | Test Status | ✅ All Passing |
 
 ### Files Needing Attention (< 80% coverage)
@@ -43,7 +43,7 @@
 | 63.4% | 86 | `domains/communities/services/community_service.py` |
 | 64.0% | 41 | `domains/files/services/s3_file_service.py` |
 
-*... and 20 more files with < 80% coverage*
+*... and 22 more files with < 80% coverage*
 
 ### Running Tests
 
@@ -55,7 +55,7 @@
 open reports/coverage/index.html
 ```
 
-*Last updated: 2025-10-15 22:38:06*
+*Last updated: 2025-10-16 13:22:11*
 
 ---
 

geek_cafe_saas_sdk-0.7.0/examples/file_lineage_example.py

@@ -0,0 +1,240 @@
+"""
+Example: Using File Lineage System
+
+This example demonstrates how to track file transformations through
+a data processing pipeline.
+
+Geek Cafe, LLC
+MIT License. See Project Root for the license information.
+"""
+
+from boto3_assist.dynamodb.dynamodb import DynamoDB
+from boto3_assist.s3.s3_connection import S3Connection
+from boto3_assist.s3.s3_object import S3Object
+from boto3_assist.s3.s3_bucket import S3Bucket
+
+from geek_cafe_saas_sdk.domains.files.services import (
+    FileSystemService,
+    S3FileService,
+    FileLineageService
+)
+
+
+def setup_services():
+    """Initialize file system services."""
+    db = DynamoDB()
+    connection = S3Connection()
+    
+    s3_service = S3FileService(
+        s3_object=S3Object(connection=connection),
+        s3_bucket=S3Bucket(connection=connection),
+        default_bucket="my-files-bucket"
+    )
+    
+    file_service = FileSystemService(
+        dynamodb=db,
+        table_name="files-table",
+        s3_service=s3_service,
+        default_bucket="my-files-bucket"
+    )
+    
+    lineage_service = FileLineageService(
+        file_service=file_service,
+        s3_service=s3_service
+    )
+    
+    return file_service, lineage_service
+
+
+def example_data_pipeline():
+    """
+    Example: Data processing pipeline with lineage tracking.
+    
+    Flow:
+    1. User uploads original file (measurements.xls)
+    2. System converts to main file (measurements.csv)
+    3. Data cleaning produces derived files (v1, v2, v3...)
+    4. User selects derived file for lineage
+    5. System bundles selected + main + original
+    """
+    
+    file_service, lineage_service = setup_services()
+    
+    tenant_id = "tenant-123"
+    user_id = "user-456"
+    
+    # Step 1: Upload original file
+    print("Step 1: Uploading original file...")
+    with open("measurements.xls", "rb") as f:
+        xls_data = f.read()
+    
+    original_result = file_service.create(
+        tenant_id=tenant_id,
+        user_id=user_id,
+        file_name="measurements.xls",
+        file_data=xls_data,
+        mime_type="application/vnd.ms-excel",
+        file_role="original"  # Mark as original
+    )
+    
+    if not original_result.success:
+        print(f"Error: {original_result.message}")
+        return
+    
+    original_file = original_result.data
+    print(f"✓ Original file uploaded: {original_file.file_id}")
+    
+    # Step 2: Convert to main file (XLS → CSV)
+    print("\nStep 2: Converting to CSV...")
+    
+    # Simulate conversion (replace with actual conversion logic)
+    csv_data = convert_xls_to_csv(xls_data)
+    
+    main_result = lineage_service.create_main_file(
+        tenant_id=tenant_id,
+        user_id=user_id,
+        original_file_id=original_file.file_id,
+        file_name="measurements.csv",
+        file_data=csv_data,
+        mime_type="text/csv",
+        transformation_operation="xls_to_csv",
+        transformation_metadata={
+            'converter_version': '1.0',
+            'source_format': 'xls',
+            'target_format': 'csv'
+        }
+    )
+    
+    if not main_result.success:
+        print(f"Error: {main_result.message}")
+        return
+    
+    main_file = main_result.data
+    print(f"✓ Main file created: {main_file.file_id}")
+    print(f"  Role: {main_file.file_role}")
+    print(f"  Operation: {main_file.transformation_operation}")
+    
+    # Step 3: Create derived files (data cleaning)
+    print("\nStep 3: Creating cleaned versions...")
+    
+    derived_files = []
+    for version in range(1, 4):
+        # Simulate data cleaning
+        cleaned_data = perform_data_cleaning(csv_data, version)
+        
+        derived_result = lineage_service.create_derived_file(
+            tenant_id=tenant_id,
+            user_id=user_id,
+            main_file_id=main_file.file_id,
+            file_name=f"measurements_clean_v{version}.csv",
+            file_data=cleaned_data,
+            transformation_operation=f"data_cleaning_v{version}",
+            transformation_metadata={
+                'cleaning_version': version,
+                'operations': ['remove_nulls', 'normalize_units'],
+                'rows_processed': 1000
+            }
+        )
+        
+        if derived_result.success:
+            derived_file = derived_result.data
+            derived_files.append(derived_file)
+            print(f"✓ Derived v{version} created: {derived_file.file_id}")
+    
+    # Step 4: Get lineage for a derived file
+    print("\nStep 4: Getting lineage for derived v2...")
+    
+    selected_file_id = derived_files[1].file_id  # v2
+    lineage_result = lineage_service.get_lineage(
+        file_id=selected_file_id,
+        tenant_id=tenant_id,
+        user_id=user_id
+    )
+    
+    if lineage_result.success:
+        lineage = lineage_result.data
+        print(f"Selected: {lineage['selected'].file_name}")
+        print(f"Main: {lineage['main'].file_name}")
+        print(f"Original: {lineage['original'].file_name}")
+        print(f"All derived versions: {len(lineage['all_derived'])}")
+    
+    # Step 5: Prepare lineage bundle
+    print("\nStep 5: Preparing lineage bundle...")
+    
+    bundle_result = lineage_service.prepare_lineage_bundle(
+        selected_file_id=selected_file_id,
+        tenant_id=tenant_id,
+        user_id=user_id
+    )
+    
+    if bundle_result.success:
+        bundle = bundle_result.data
+        print("Bundle contents:")
+        print(f"  - Selected: {bundle['selected_file'].file_name}")
+        print(f"  - Main: {bundle['main_file'].file_name}")
+        print(f"  - Original: {bundle['original_file'].file_name}")
+        print(f"\nTransformation chain:")
+        for step in bundle['metadata']['transformation_chain']:
+            print(f"  {step['step']}. {step['type']}: {step['file_name']}")
+            if 'operation' in step:
+                print(f"     Operation: {step['operation']}")
+    
+    # Step 6: Download complete bundle
+    print("\nStep 6: Downloading complete bundle...")
+    
+    download_result = lineage_service.download_lineage_bundle(
+        selected_file_id=selected_file_id,
+        tenant_id=tenant_id,
+        user_id=user_id
+    )
+    
+    if download_result.success:
+        download_bundle = download_result.data
+        
+        # Save each file
+        if download_bundle['original']:
+            save_file('output/original/', download_bundle['original'])
+        
+        if download_bundle['main']:
+            save_file('output/main/', download_bundle['main'])
+        
+        if download_bundle['selected']:
+            save_file('output/processed/', download_bundle['selected'])
+        
+        print("✓ Bundle downloaded successfully!")
+
+
+def convert_xls_to_csv(xls_data: bytes) -> bytes:
+    """Simulate XLS to CSV conversion."""
+    # In production, use a library like openpyxl or pandas
+    # For example:
+    # import pandas as pd
+    # df = pd.read_excel(io.BytesIO(xls_data))
+    # return df.to_csv().encode('utf-8')
+    
+    return b"mock,csv,data\n1,2,3\n4,5,6"
+
+
+def perform_data_cleaning(csv_data: bytes, version: int) -> bytes:
+    """Simulate data cleaning."""
+    # In production, implement actual cleaning logic
+    # For example: remove nulls, normalize units, etc.
+    
+    return csv_data + f"\n# Cleaned v{version}".encode('utf-8')
+
+
+def save_file(directory: str, file_bundle: dict):
+    """Save file to disk."""
+    import os
+    
+    os.makedirs(directory, exist_ok=True)
+    
+    file_path = os.path.join(directory, file_bundle['file'].file_name)
+    with open(file_path, 'wb') as f:
+        f.write(file_bundle['data'])
+    
+    print(f"  Saved: {file_path}")
+
+
+if __name__ == "__main__":
+    example_data_pipeline()

{geek_cafe_saas_sdk-0.6.0 → geek_cafe_saas_sdk-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "geek_cafe_saas_sdk"
-version = "0.6.0"
+version = "0.7.0"
 description = "Base Reusable Services for SaaS"
 authors = [{ name = "Eric Wilson", email = "eric.wilson@geekcafe.com" }]
 requires-python = ">=3.8"

{geek_cafe_saas_sdk-0.6.0 → geek_cafe_saas_sdk-0.7.0}/src/geek_cafe_saas_sdk/__init__.py

@@ -1,9 +1,9 @@
 """
 Geek Cafe Services - Base Reusable Services for SaaS
 
-Version 0.2.0 adds Lambda Handler Wrappers for reducing boilerplate code.
+Version 0.6.0 adds File System Service
 """
-__version__ = "0.6.0"
+__version__ = "0.7.0"
 
 # Import main modules for easier access
 from . import services