amplify-excel-migrator 1.1.5__py3-none-any.whl → 1.2.15__py3-none-any.whl

amplify_excel_migrator/core/config.py

@@ -0,0 +1,98 @@
+"""Configuration management for Amplify Excel Migrator."""
+
+import json
+import logging
+from getpass import getpass
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigManager:
+    """Manages configuration loading, saving, and user prompts."""
+
+    DEFAULT_CONFIG_DIR = Path.home() / ".amplify-migrator"
+    DEFAULT_CONFIG_FILE = "config.json"
+
+    SENSITIVE_KEYS = {"password", "ADMIN_PASSWORD"}
+
+    def __init__(self, config_path: Optional[str] = None):
+        if config_path:
+            self.config_path = Path(config_path)
+        else:
+            self.config_path = self.DEFAULT_CONFIG_DIR / self.DEFAULT_CONFIG_FILE
+
+        self._config: Dict[str, Any] = {}
+
+    def load(self) -> Dict[str, Any]:
+        if not self.config_path.exists():
+            logger.debug(f"Config file not found at {self.config_path}")
+            return {}
+
+        try:
+            with open(self.config_path, "r") as f:
+                self._config = json.load(f)
+                logger.debug(f"Loaded configuration from {self.config_path}")
+                return self._config
+        except Exception as e:
+            logger.warning(f"Failed to load cached config: {e}")
+            return {}
+
+    def save(self, config: Dict[str, Any]) -> None:
+        self.config_path.parent.mkdir(parents=True, exist_ok=True)
+
+        sanitized_config = {k: v for k, v in config.items() if k not in self.SENSITIVE_KEYS}
+
+        with open(self.config_path, "w") as f:
+            json.dump(sanitized_config, f, indent=2)
+
+        self._config = sanitized_config
+        logger.info(f"✅ Configuration saved to {self.config_path}")
+
+    def get(self, key: str, default: Any = None) -> Any:
+        if not self._config:
+            self.load()
+        return self._config.get(key, default)
+
+    def set(self, key: str, value: Any) -> None:
+        if not self._config:
+            self.load()
+        self._config[key] = value
+
+    def update(self, updates: Dict[str, Any]) -> None:
+        if not self._config:
+            self.load()
+        self._config.update(updates)
+        self.save(self._config)
+
+    def prompt_for_value(self, prompt_text: str, default: str = "", secret: bool = False) -> str:
+        if default:
+            display_prompt = f"{prompt_text} [{default}]: "
+        else:
+            display_prompt = f"{prompt_text}: "
+
+        if secret:
+            value = getpass(display_prompt)
+        else:
+            value = input(display_prompt)
+
+        return value.strip() if value.strip() else default
+
+    def get_or_prompt(self, key: str, prompt_text: str, default: str = "", secret: bool = False) -> str:
+        if not self._config:
+            self.load()
+
+        if key in self._config:
+            return self._config[key]
+
+        return self.prompt_for_value(prompt_text, default, secret)
+
+    def exists(self) -> bool:
+        return self.config_path.exists()
+
+    def clear(self) -> None:
+        self._config = {}
+        if self.config_path.exists():
+            self.config_path.unlink()
+            logger.info(f"Configuration cleared from {self.config_path}")

amplify_excel_migrator/data/__init__.py

@@ -0,0 +1,7 @@
+"""Data processing components for Excel migration."""
+
+from .excel_reader import ExcelReader
+from .transformer import DataTransformer
+from .validator import RecordValidator
+
+__all__ = ["ExcelReader", "DataTransformer", "RecordValidator"]

amplify_excel_migrator/data/excel_reader.py

@@ -0,0 +1,23 @@
+"""Excel file reading functionality."""
+
+import logging
+from typing import Dict, Any
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+class ExcelReader:
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+
+    def read_all_sheets(self) -> Dict[str, pd.DataFrame]:
+        logger.info(f"Reading Excel file: {self.file_path}")
+        all_sheets = pd.read_excel(self.file_path, sheet_name=None)
+        logger.info(f"Loaded {len(all_sheets)} sheets from Excel")
+        return all_sheets
+
+    def read_sheet(self, sheet_name: str) -> pd.DataFrame:
+        logger.info(f"Reading sheet '{sheet_name}' from Excel file: {self.file_path}")
+        return pd.read_excel(self.file_path, sheet_name=sheet_name)

amplify_excel_migrator/data/transformer.py

@@ -0,0 +1,119 @@
+"""Data transformation from Excel rows to Amplify records."""
+
+import logging
+import re
+from typing import Dict, Any, Optional, List, Tuple
+
+import pandas as pd
+
+from amplify_excel_migrator.schema import FieldParser
+
+logger = logging.getLogger(__name__)
+
+
+class DataTransformer:
+    def __init__(self, field_parser: FieldParser):
+        self.field_parser = field_parser
+
+    def transform_rows_to_records(
+        self,
+        df: pd.DataFrame,
+        parsed_model_structure: Dict[str, Any],
+        primary_field: str,
+        fk_lookup_cache: Dict[str, Dict[str, str]],
+    ) -> Tuple[List[Dict], Dict[str, Dict], List[Dict]]:
+        records = []
+        row_dict_by_primary = {}
+        failed_rows = []
+        row_count = 0
+
+        for row_tuple in df.itertuples(index=False, name="Row"):
+            row_count += 1
+            row_dict = {col: getattr(row_tuple, col) for col in df.columns}
+            primary_field_value = row_dict.get(primary_field, f"Row {row_count}")
+
+            row_dict_by_primary[str(primary_field_value)] = row_dict.copy()
+
+            try:
+                record = self.transform_row_to_record(row_dict, parsed_model_structure, fk_lookup_cache)
+                if record:
+                    records.append(record)
+            except Exception as e:
+                error_msg = str(e)
+                logger.error(f"Error transforming row {row_count} ({primary_field}={primary_field_value}): {error_msg}")
+                failed_rows.append(
+                    {
+                        "primary_field": primary_field,
+                        "primary_field_value": primary_field_value,
+                        "error": f"Parsing error: {error_msg}",
+                        "original_row": row_dict,
+                    }
+                )
+
+        logger.info(f"Prepared {len(records)} records for upload")
+
+        return records, row_dict_by_primary, failed_rows
+
+    def transform_row_to_record(
+        self, row_dict: Dict, parsed_model_structure: Dict[str, Any], fk_lookup_cache: Dict[str, Dict[str, str]]
+    ) -> Optional[Dict]:
+        model_record = {}
+
+        for field in parsed_model_structure["fields"]:
+            input_value = self.parse_input(row_dict, field, fk_lookup_cache)
+            if input_value is not None:
+                model_record[field["name"]] = input_value
+
+        return model_record
+
+    def parse_input(
+        self,
+        row_dict: Dict,
+        field: Dict[str, Any],
+        fk_lookup_cache: Dict[str, Dict[str, str]],
+    ) -> Any:
+        field_name = field["name"][:-2] if field["is_id"] else field["name"]
+
+        if field_name not in row_dict or pd.isna(row_dict[field_name]):
+            if field["is_required"]:
+                raise ValueError(f"Required field '{field_name}' is missing")
+            return None
+
+        value = self.field_parser.clean_input(row_dict[field_name])
+
+        if field["is_id"]:
+            return self._resolve_foreign_key(field, value, fk_lookup_cache)
+        elif field["is_list"] and field["is_scalar"]:
+            return self.field_parser.parse_scalar_array(field, field_name, row_dict[field_name])
+        else:
+            return self.field_parser.parse_field_input(field, field_name, value)
+
+    @staticmethod
+    def _resolve_foreign_key(
+        field: Dict[str, Any], value: Any, fk_lookup_cache: Dict[str, Dict[str, str]]
+    ) -> Optional[str]:
+        if "related_model" in field:
+            related_model = field["related_model"]
+        else:
+            related_model = (temp := field["name"][:-2])[0].upper() + temp[1:]
+
+        if related_model in fk_lookup_cache:
+            lookup_dict = fk_lookup_cache[related_model]["lookup"]
+            record_id = lookup_dict.get(str(value))
+
+            if record_id:
+                return record_id
+            elif field["is_required"]:
+                raise ValueError(f"{related_model}: {value} does not exist")
+            return None
+        else:
+            if field["is_required"]:
+                raise ValueError(f"No pre-fetched data for {related_model}, cannot resolve FK for required field")
+            logger.warning(f"No pre-fetched data for {related_model}, skipping optional FK field")
+            return None
+
+    @staticmethod
+    def to_camel_case(s: str) -> str:
+        s_with_spaces = re.sub(r"(?<!^)(?=[A-Z])", " ", s)
+        parts = re.split(r"[\s_\-]+", s_with_spaces.strip())
+        return parts[0].lower() + "".join(word.capitalize() for word in parts[1:])

amplify_excel_migrator/data/validator.py

@@ -0,0 +1,48 @@
+"""Record validation functionality."""
+
+import logging
+from typing import Dict, Any, List
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+class RecordValidator:
+    @staticmethod
+    def validate_required_fields(row_dict: Dict, parsed_model_structure: Dict[str, Any]) -> List[str]:
+        errors = []
+
+        for field in parsed_model_structure["fields"]:
+            if not field["is_required"]:
+                continue
+
+            field_name = field["name"][:-2] if field["is_id"] else field["name"]
+
+            if field_name not in row_dict or pd.isna(row_dict[field_name]):
+                errors.append(f"Required field '{field_name}' is missing")
+
+        return errors
+
+    @staticmethod
+    def validate_foreign_key(
+        field: Dict[str, Any], value: Any, fk_lookup_cache: Dict[str, Dict[str, str]]
+    ) -> List[str]:
+        errors = []
+
+        if "related_model" in field:
+            related_model = field["related_model"]
+        else:
+            related_model = (temp := field["name"][:-2])[0].upper() + temp[1:]
+
+        if related_model not in fk_lookup_cache:
+            if field["is_required"]:
+                errors.append(f"No pre-fetched data for required foreign key {related_model}")
+            return errors
+
+        lookup_dict = fk_lookup_cache[related_model]["lookup"]
+        if str(value) not in lookup_dict:
+            if field["is_required"]:
+                errors.append(f"{related_model}: {value} does not exist")
+
+        return errors

amplify_excel_migrator/graphql/__init__.py

@@ -0,0 +1,8 @@
+"""GraphQL module for query and mutation building."""
+
+from .query_builder import QueryBuilder
+from .mutation_builder import MutationBuilder
+from .client import GraphQLClient, AuthenticationError, GraphQLError
+from .executor import QueryExecutor
+
+__all__ = ["QueryBuilder", "MutationBuilder", "GraphQLClient", "AuthenticationError", "GraphQLError", "QueryExecutor"]

amplify_excel_migrator/graphql/client.py

@@ -0,0 +1,137 @@
+"""GraphQL HTTP client for making requests to GraphQL APIs."""
+
+import logging
+import sys
+from typing import Dict, Any, Optional
+
+import aiohttp
+import requests
+
+from amplify_excel_migrator.auth import AuthenticationProvider
+
+logger = logging.getLogger(__name__)
+
+
+class AuthenticationError(Exception):
+    """Raised when authentication is required but not completed"""
+
+    pass
+
+
+class GraphQLError(Exception):
+    """Raised when GraphQL query returns errors"""
+
+    pass
+
+
+class GraphQLClient:
+    def __init__(self, api_endpoint: str, auth_provider: Optional[AuthenticationProvider] = None):
+        self.api_endpoint = api_endpoint
+        self.auth_provider = auth_provider
+
+    def request(
+        self, query: str, variables: Optional[Dict[str, Any]] = None, context: Optional[str] = None
+    ) -> Optional[Dict[str, Any]]:
+        if not self.auth_provider or not self.auth_provider.is_authenticated():
+            raise AuthenticationError("Not authenticated. Call authenticate() on the auth provider first.")
+
+        id_token = self.auth_provider.get_id_token()
+        headers = {"Authorization": id_token, "Content-Type": "application/json"}
+
+        payload = {"query": query, "variables": variables or {}}
+
+        context_msg = f" [{context}]" if context else ""
+
+        try:
+            response = requests.post(self.api_endpoint, headers=headers, json=payload)
+
+            if response.status_code == 200:
+                result = response.json()
+
+                if "errors" in result:
+                    raise GraphQLError(f"GraphQL errors{context_msg}: {result['errors']}")
+
+                return result
+            else:
+                logger.error(f"HTTP Error {response.status_code}{context_msg}: {response.text}")
+                return None
+
+        except requests.exceptions.ConnectionError:
+            logger.error(
+                f"Connection error{context_msg}: Unable to connect to API endpoint. Check your internet connection or the API endpoint URL."
+            )
+            sys.exit(1)
+
+        except requests.exceptions.Timeout as e:
+            logger.error(f"Request timeout{context_msg}: {e}")
+            return None
+
+        except requests.exceptions.HTTPError as e:
+            logger.error(f"HTTP error{context_msg}: {e}")
+            return None
+
+        except GraphQLError as e:
+            logger.error(str(e))
+            return None
+
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request error{context_msg}: {e}")
+            return None
+
+    async def request_async(
+        self,
+        session: aiohttp.ClientSession,
+        query: str,
+        variables: Optional[Dict[str, Any]] = None,
+        context: Optional[str] = None,
+    ) -> Optional[Dict[str, Any]]:
+        if not self.auth_provider or not self.auth_provider.is_authenticated():
+            raise AuthenticationError("Not authenticated. Call authenticate() on the auth provider first.")
+
+        id_token = self.auth_provider.get_id_token()
+        headers = {"Authorization": id_token, "Content-Type": "application/json"}
+
+        payload = {"query": query, "variables": variables or {}}
+
+        context_msg = f" [{context}]" if context else ""
+
+        try:
+            async with session.post(self.api_endpoint, headers=headers, json=payload) as response:
+                if response.status == 200:
+                    result = await response.json()
+
+                    if "errors" in result:
+                        raise GraphQLError(f"GraphQL errors{context_msg}: {result['errors']}")
+
+                    return result
+                else:
+                    text = await response.text()
+                    error_msg = f"HTTP Error {response.status}{context_msg}: {text}"
+                    logger.error(error_msg)
+                    raise aiohttp.ClientError(error_msg)
+
+        except aiohttp.ServerTimeoutError as e:
+            error_msg = f"Request timeout{context_msg}: {e}"
+            logger.error(error_msg)
+            raise aiohttp.ServerTimeoutError(error_msg)
+
+        except aiohttp.ClientConnectionError as e:
+            error_msg = f"Connection error{context_msg}: Unable to connect to API endpoint. {e}"
+            logger.error(error_msg)
+            raise aiohttp.ClientConnectionError(error_msg)
+
+        except aiohttp.ClientResponseError as e:
+            error_msg = f"HTTP response error{context_msg}: {e}"
+            logger.error(error_msg)
+            raise aiohttp.ClientResponseError(
+                request_info=e.request_info, history=e.history, status=e.status, message=error_msg
+            )
+
+        except GraphQLError as e:
+            logger.error(str(e))
+            raise
+
+        except aiohttp.ClientError as e:
+            error_msg = f"Client error{context_msg}: {e}"
+            logger.error(error_msg)
+            raise aiohttp.ClientError(error_msg)