poly-hammer-utils 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

poly_hammer_utils/constants.py

@@ -1,5 +1,8 @@
+import os
 from pathlib import Path
 
 RESOURCES_FOLDER = Path(__file__).parent / 'resources'
 
-BLENDER_STARTUP_SCRIPT = RESOURCES_FOLDER / 'scripts' / 'blender' / 'startup.py'
+BLENDER_STARTUP_SCRIPT = RESOURCES_FOLDER / 'scripts' / 'blender' / 'startup.py'
+
+ENVIRONMENT = os.environ.get('ENVIRONMENT', 'staging')

poly_hammer_utils/extension/meta_human_dna.py

@@ -3,10 +3,17 @@ import logging
 import zipfile
 import tempfile
 from pathlib import Path
+from poly_hammer_utils.constants import ENVIRONMENT
 from poly_hammer_utils.utilities import download_and_unzip_to_folder, download_release_file
 from poly_hammer_utils.github.release import GitHubRelease
 from poly_hammer_utils.extension.packager import package_extension, get_addon_version
-from poly_hammer_utils.extension.server import update_extension_index
+from poly_hammer_utils.extension.server import update_extension_index, sync_extensions_from_s3
+
+BLENDER_EXTENSION_SERVER_S3_BUCKET = 'poly-hammer-portal-staging-app-data'
+if ENVIRONMENT == 'production':
+    BLENDER_EXTENSION_SERVER_S3_BUCKET = 'poly-hammer-portal-production-app-data'
+
+BLENDER_EXTENSION_SERVER_S3_FOLDER = 'products/blender-extensions/meta_human_dna'
 
 logger = logging.getLogger(__name__)
 
@@ -119,7 +126,14 @@ def create_release(addon_folder: Path, releases_folder: Path):
 
     addon_version = get_addon_version(source_folder=addon_folder)
 
-    # Create the .zip files for the addon's various platforms
+    # First, sync existing extensions from S3
+    sync_extensions_from_s3(
+        repo_folder=releases_folder,
+        bucket=BLENDER_EXTENSION_SERVER_S3_BUCKET,
+        s3_prefix=BLENDER_EXTENSION_SERVER_S3_FOLDER,
+    )
+
+    # Create the new .zip files for the addon's various platforms
     addon_zip_files = package_extension(
         source_folder=addon_folder,
         output_folder=releases_folder,
@@ -137,6 +151,7 @@ def create_release(addon_folder: Path, releases_folder: Path):
         auth_token=token
     )
 
+    # Update each addon .zip to include the rig logic and core bindings
     for addon_zip_file in addon_zip_files:
         platform, arch = parse_blender_extension_zip_info(url=addon_zip_file)
         files_to_zip = rig_logic_files + core_files
@@ -157,11 +172,11 @@ def create_release(addon_folder: Path, releases_folder: Path):
                 else:
                     logger.error(f"Error: {file_path} not found")
 
-    # Generate the extension index and update it and upload addons to S3
+    # Generate the updated extension index and upload it and the addons .zip file to S3
     update_extension_index(
         repo_folder=releases_folder,
-        bucket=os.environ['BLENDER_EXTENSION_SERVER_S3_BUCKET'],
-        s3_folder=os.environ['BLENDER_EXTENSION_SERVER_S3_FOLDER'],
+        bucket=BLENDER_EXTENSION_SERVER_S3_BUCKET,
+        s3_folder=BLENDER_EXTENSION_SERVER_S3_FOLDER,
         blender_version=os.environ['BLENDER_VERSION'],
         docker=True
     )

poly_hammer_utils/extension/server.py

@@ -1,9 +1,10 @@
 import os
 import sys
-import json
 import logging
 import boto3
+import httpx
 from pathlib import Path
+from poly_hammer_utils.constants import ENVIRONMENT
 from poly_hammer_utils.utilities import shell, get_blender_executable
 
 logger = logging.getLogger(__name__)
@@ -12,25 +13,27 @@ ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
 SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
 REGION = os.environ.get('AWS_REGION')
 
-def merge_extension_index_from_s3(
-        local_index_path: Path,
+BASE_URL = 'https://api.portal.staging.polyhammer.com'
+if ENVIRONMENT == 'production':
+    BASE_URL = 'https://api.portal.polyhammer.com'
+
+PORTAL_API_KEY = os.environ.get('PORTAL_API_KEY')
+
+def sync_extensions_from_s3(
+        repo_folder: Path,
         bucket: str,
-        s3_key: str,
-        output_path: Path | None = None
-    ) -> dict:
+        s3_prefix: str = ''
+    ) -> list[Path]:
     """
-    Reads a JSON index file from S3, merges it with a local JSON index file,
-    and optionally writes the merged result to an output path.
+    Downloads all .zip extension files from an S3 folder to a local directory.
 
     Args:
-        local_index_path (Path): Path to the local JSON file to merge.
+        repo_folder (Path): Local directory to download files to.
         bucket (str): The S3 bucket name.
-        s3_key (str): The object key (path) in the S3 bucket.
-        output_path (Path, optional): Path to write the merged JSON. 
-            If None, overwrites the local_index_path.
+        s3_prefix (str, optional): Prefix (folder path) in S3. Defaults to ''.
 
     Returns:
-        dict: The merged JSON data.
+        list[Path]: List of local file paths that were downloaded.
     """
     s3_client = boto3.client(
         's3',
@@ -39,33 +42,31 @@ def merge_extension_index_from_s3(
         region_name=REGION,
     )
 
-    # Read the JSON file from S3
-    logger.info(f'Reading index from s3://{bucket}/{s3_key}')
-    try:
-        response = s3_client.get_object(Bucket=bucket, Key=s3_key)
-        s3_data = json.loads(response['Body'].read().decode('utf-8'))
-    except s3_client.exceptions.NoSuchKey:
-        logger.warning(f'S3 key "{s3_key}" not found, starting with empty data')
-        s3_data = {}
-    except Exception as e:
-        logger.warning(f'Failed to read from S3: {e}, starting with empty data')
-        s3_data = {}
+    repo_folder.mkdir(parents=True, exist_ok=True)
+
+    # List all objects in the S3 prefix
+    prefix = f'{s3_prefix}/' if s3_prefix and not s3_prefix.endswith('/') else s3_prefix
+    logger.info(f'Listing objects in s3://{bucket}/{prefix}')
 
-    # Read the local JSON file
-    logger.info(f'Reading local index from "{local_index_path}"')
-    with open(local_index_path, 'r', encoding='utf-8') as f:
-        local_data = json.load(f)
+    downloaded_files = []
+    paginator = s3_client.get_paginator('list_objects_v2')
 
-    # Merge the data (local data takes precedence)
-    merged_data = {**s3_data, **local_data}
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+        for obj in page.get('Contents', []):
+            key = obj['Key']
+            filename = key.split('/')[-1]
 
-    # Write the merged result
-    output = output_path or local_index_path
-    logger.info(f'Writing merged index to "{output}"')
-    with open(output, 'w', encoding='utf-8') as f:
-        json.dump(merged_data, f, indent=2)
+            # Only download .zip files
+            if not filename.endswith('.zip'):
+                continue
 
-    return merged_data
+            local_path = repo_folder / filename
+            logger.info(f'Downloading s3://{bucket}/{key} to "{local_path}"')
+            s3_client.download_file(bucket, key, str(local_path))
+            downloaded_files.append(local_path)
+
+    logger.info(f'Successfully downloaded {len(downloaded_files)} file(s) from S3')
+    return downloaded_files
 
 
 def upload_extensions_to_s3(
@@ -74,10 +75,12 @@ def upload_extensions_to_s3(
         s3_prefix: str = ''
     ) -> list[str]:
     """
-    Uploads extension .zip files and/or index files to S3.
+    Uploads extension .zip files and the index.json to S3.
+    Only uploads .zip files if they are newer than the S3 version.
+    Always uploads the index.json file.
 
     Args:
-        file_paths (list[Path]): List of file paths to upload.
+        repo_folder (Path): Local directory containing files to upload.
         bucket (str): The S3 bucket name.
         s3_prefix (str, optional): Prefix (folder path) in S3. Defaults to ''.
 
@@ -91,14 +94,39 @@ def upload_extensions_to_s3(
         region_name=REGION,
     )
 
+    # Build a map of S3 object keys to their last modified times
+    prefix = f'{s3_prefix}/' if s3_prefix and not s3_prefix.endswith('/') else s3_prefix
+    s3_objects = {}
+    paginator = s3_client.get_paginator('list_objects_v2')
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+        for obj in page.get('Contents', []):
+            s3_objects[obj['Key']] = obj['LastModified']
+
     uploaded_keys = []
-    # Only upload .json and .zip files
+    # Collect .json and .zip files
     file_paths = list(repo_folder.glob('*.json')) + list(repo_folder.glob('*.zip'))
+
     for file_path in file_paths:
         if not file_path.exists():
             logger.warning(f'File "{file_path}" does not exist, skipping')
             continue
 
+        # Build the S3 key
+        s3_key = f'{s3_prefix}/{file_path.name}' if s3_prefix else file_path.name
+        s3_key = s3_key.lstrip('/')
+
+        # For .zip files, check if local file is newer than S3 version
+        if file_path.suffix == '.zip' and s3_key in s3_objects:
+            import datetime
+            local_mtime = datetime.datetime.fromtimestamp(
+                file_path.stat().st_mtime, 
+                tz=datetime.timezone.utc
+            )
+            s3_mtime = s3_objects[s3_key]
+            if local_mtime <= s3_mtime:
+                logger.info(f'Skipping "{file_path.name}" (not newer than S3 version)')
+                continue
+
         # Determine content type
         if file_path.suffix == '.zip':
             content_type = 'application/zip'
@@ -107,10 +135,6 @@ def upload_extensions_to_s3(
         else:
             content_type = 'application/octet-stream'
 
-        # Build the S3 key
-        s3_key = f'{s3_prefix}/{file_path.name}' if s3_prefix else file_path.name
-        s3_key = s3_key.lstrip('/')
-
         logger.info(f'Uploading "{file_path}" to s3://{bucket}/{s3_key}')
         s3_client.upload_file(
             Filename=str(file_path),
@@ -147,6 +171,34 @@ def generate_extension_index(
 
     shell(command)
 
+def trigger_poly_hammer_portal_extension_index_refresh() -> httpx.Response:
+    """Trigger sync of all extensions from S3."""
+    logger.info("Triggering extension sync...")
+    
+    url = f"{BASE_URL}/api/v1/admin/extensions/sync"
+    headers = {"X-Admin-API-Key": PORTAL_API_KEY}
+
+    response = httpx.post(url, headers=headers, timeout=60.0)
+
+    if response.status_code == 200:
+        data = response.json()
+        logger.info("Sync complete!")
+        logger.info(f"Extensions processed: {data['extensions_processed']}")
+        logger.info(f"Platforms synced: {data['total_platforms_synced']}")
+        
+        if data.get("errors"):
+            logger.error(f"Errors: {data['errors']}")
+        
+        if data.get("details"):
+            logger.info("Extension details:")
+            for ext in data["details"]:
+                logger.info(f"- {ext['extension_id']}: {ext['versions_synced']} versions, {ext['platforms_synced']} platforms")
+    else:
+        logger.error(f"Failed with status {response.status_code}")
+        logger.error(f"Response: {response.text}")
+
+    return response
+
 def update_extension_index(
         repo_folder: Path, 
         bucket: str,
@@ -154,23 +206,30 @@ def update_extension_index(
         blender_version: str = '4.5',
         docker: bool = False,
     ):
+    """
+    Syncs extensions from S3, regenerates the index locally, and uploads
+    any new or modified files back to S3.
+
+    Args:
+        repo_folder (Path): Local directory for the extension repository.
+        bucket (str): The S3 bucket name.
+        s3_folder (str): The folder path in S3.
+        blender_version (str, optional): Blender version to use. Defaults to '4.5'.
+        docker (bool, optional): Whether to use Docker for index generation. Defaults to False.
+    """
+    # Step 1: Generate the extension index from all local .zip files
     generate_extension_index(
         repo_folder=repo_folder,
         blender_version=blender_version,
         docker=docker,
     )
 
-    local_index_path = repo_folder / 'index.json'
-    merge_extension_index_from_s3(
-        local_index_path=local_index_path,
-        bucket=bucket,
-        s3_key=f'{s3_folder}/index.json',
-        output_path=local_index_path,
-    )
-
+    # Step 2: Upload new/modified .zip files and the updated index.json back to S3
     upload_extensions_to_s3(
         repo_folder=repo_folder,
         bucket=bucket,
         s3_prefix=s3_folder,
     )
 
+    # Step 3: Trigger the Poly Hammer Portal to refresh its extension index
+    trigger_poly_hammer_portal_extension_index_refresh()

{poly_hammer_utils-0.0.23.dist-info → poly_hammer_utils-0.0.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: poly-hammer-utils
-Version: 0.0.23
+Version: 0.0.25
 Summary: 
 Author: Poly Hammer
 Author-email: info@polyhammer.com
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: PyGithub (==2.5.0)
 Requires-Dist: boto3 (>=1.42.35,<2.0.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
 Requires-Dist: python-dotenv (>=1.1.1,<2.0.0)
 Requires-Dist: requirements-parser (>=0.11.0,<0.12.0)
 Requires-Dist: tomlkit (>=0.14.0,<0.15.0)

{poly_hammer_utils-0.0.23.dist-info → poly_hammer_utils-0.0.25.dist-info}/RECORD

@@ -1,16 +1,16 @@
 poly_hammer_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 poly_hammer_utils/addon/packager.py,sha256=apoZhQpd2QeRucFq53u-L1FPmN4MYuiNrjTETAE9NQg,8435
 poly_hammer_utils/addon/release.py,sha256=EZirS8_c9S1OEZoOHStgciu_C7vMxBc_-O5P1fM9Qu4,9074
-poly_hammer_utils/constants.py,sha256=Q5cCA6H-AJYWdbndCwk3fIoBPHx_9NTgjvY7p-e0grA,162
-poly_hammer_utils/extension/meta_human_dna.py,sha256=XPH9w3q3ZALPKRGZ7nHULMHPSr4XYZ5b4FTUOUW56a8,7103
+poly_hammer_utils/constants.py,sha256=4XGiwn7ip50V1zvmpVD9YZYmE6wPhj2Ark2AXQuJvSc,228
+poly_hammer_utils/extension/meta_human_dna.py,sha256=HP5BGYhxwjlPBAPnJxqtKjVI5uWyK3ZTFFLDJ3jNoaw,7740
 poly_hammer_utils/extension/packager.py,sha256=btYzyU_fa2Qaa6R9eZSojQ5uy3nQ0kDxCf49LnTZrHM,2115
-poly_hammer_utils/extension/server.py,sha256=Zu0ao-RBSYo2Do6d3ZGQeBkUUmMTUwWXR0rQ9Y6x7ig,5420
+poly_hammer_utils/extension/server.py,sha256=vuDMqEehgFiFiUy-R28oKzijHo5wIJzW0l66wwPU6I8,8072
 poly_hammer_utils/github/release.py,sha256=P4vrNvdgWuzJI0mRye09BP6vL5b7kQ5fpuDDnEqJbwg,9951
 poly_hammer_utils/helpers.py,sha256=f_2fNrydsbIvwD2LuoBj2gjETMP7dFzJtq1_7WMBwNI,1655
 poly_hammer_utils/launch.py,sha256=Nc9FlFuGtMSjgSLsk6MMMj-FECsOGtfUQlZaUZt_rGU,469
 poly_hammer_utils/resources/scripts/blender/startup.py,sha256=eUd0VwAHjTwIscVKCbk_0-F7p0jgIEsVYdzpIjlzOK0,2282
 poly_hammer_utils/resources/scripts/unreal/init_unreal.py,sha256=1-d9IU8ZSAIS3MUANrsGx4ZmqNJ5f8S2k8XJVg0Bghs,693
 poly_hammer_utils/utilities.py,sha256=7dmZo02HXbWhfmwDHkgVUE_gyq4e-gqb4OdBc2bokbs,6491
-poly_hammer_utils-0.0.23.dist-info/METADATA,sha256=QmD8swWvEzq5a80-OocO_Z9d39Psh9Cz3GwDeB9nblY,894
-poly_hammer_utils-0.0.23.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-poly_hammer_utils-0.0.23.dist-info/RECORD,,
+poly_hammer_utils-0.0.25.dist-info/METADATA,sha256=4qTBctRnioUIPFOWKdlMakIu8peOj5qzyrzs1FQoJo4,934
+poly_hammer_utils-0.0.25.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+poly_hammer_utils-0.0.25.dist-info/RECORD,,