azureml-registry-tools 0.1.0a1__py3-none-any.whl → 0.1.0a2__py3-none-any.whl

azureml/registry/_cli/registry_syndication_cli.py

@@ -0,0 +1,230 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import argparse
+import json
+from azureml.registry._rest_client.registry_management_client import RegistryManagementClient
+from azureml.registry._rest_client.arm_client import ArmClient
+from azureml.registry.mgmt.create_manifest import generate_syndication_manifest
+from azureml.registry.mgmt.syndication_manifest import SyndicationManifest
+
+
+def syndication_manifest_show(registry_name: str) -> dict:
+    """Show the current syndication manifest for the specified registry and region.
+
+    Args:
+        registry_name (str): Name of the AzureML registry.
+
+    Returns:
+        dict: The manifest data.
+    """
+    return json.dumps(RegistryManagementClient(registry_name=registry_name).get_manifest())
+
+
+def syndication_manifest_set(manifest_value: str, folder: str, dry_run: bool) -> None:
+    """Set the syndication manifest for the specified registry and region.
+
+    Args:
+        manifest_value (str): Manifest value as a string (JSON or similar).
+        folder (str): Path to the manifest folder.
+        dry_run (bool): If True, do not perform any changes.
+    """
+    if manifest_value is not None:
+        # for inline manifest value allow different casing of keys
+        manifest = SyndicationManifest.from_dto(json.loads(manifest_value), normalize_keys=True)
+    else:
+        # Folder structure should have proper casing
+        manifest = generate_syndication_manifest(folder)
+    dto = manifest.to_dto()
+    if dry_run:
+        print(f"Dry run: Would set manifest to {dto}")
+    else:
+        client = RegistryManagementClient(registry_name=manifest.registry_name)
+        client.create_or_update_manifest(dto)
+
+
+def syndication_manifest_delete(registry_name: str, dry_run: bool) -> None:
+    """Delete the syndication manifest for the specified registry.
+
+    Args:
+        registry_name (str): Name of the AzureML registry.
+        dry_run (bool): If True, do not perform any changes.
+    """
+    if dry_run:
+        print(f"Dry run: Would delete manifest for registry {registry_name}")
+    else:
+        RegistryManagementClient(registry_name=registry_name).delete_manifest()
+
+
+def syndication_target_show(registry_name: str) -> object:
+    """Show the current syndication target(s) for the specified registry and region.
+
+    Args:
+        registry_name (str): Name of the AzureML registry.
+
+    Returns:
+        list or str: List of syndicated registries or 'None'.
+    """
+    discovery = RegistryManagementClient(registry_name=registry_name).discovery()
+    arm_resource_id = f"/subscriptions/{discovery.get('subscriptionId')}/resourceGroups/{discovery.get('resourceGroup')}/providers/Microsoft.MachineLearningServices/registries/{discovery.get('registryName')}"
+    return ArmClient().get_resource(resource_id=arm_resource_id).get("properties", {}).get("syndicatedRegistries", "None")
+
+
+def syndication_target_set(registry_name: str, registry_ids: list, dry_run: bool) -> None:
+    """Set the syndication target(s) for the specified registry and region.
+
+    Args:
+        registry_name (str): Name of the AzureML registry.
+        registry_ids (list): List of registry IDs to set as syndicated targets.
+        dry_run (bool): If True, do not perform any changes.
+    """
+    discovery = RegistryManagementClient(registry_name=registry_name).discovery()
+    arm_resource_id = f"/subscriptions/{discovery.get('subscriptionId')}/resourceGroups/{discovery.get('resourceGroup')}/providers/Microsoft.MachineLearningServices/registries/{discovery.get('registryName')}"
+    arm_client = ArmClient()
+    resource = arm_client.get_resource(resource_id=arm_resource_id)
+    resource["properties"]["syndicatedRegistries"] = registry_ids
+    if dry_run:
+        print(f"Dry run: Would set {registry_ids} as SyndicatedRegistries for {registry_name}")
+    else:
+        arm_client.put_resource(resource_id=arm_resource_id, put_body=resource)
+
+
+def show_command(registry_name: str, as_arm_object: bool) -> object:
+    """Show registry discovery info or ARM object for the specified registry and region.
+
+    Args:
+        registry_name (str): Name of the AzureML registry.
+        as_arm_object (bool): If True, show as ARM object.
+
+    Returns:
+        dict: Discovery info or ARM resource object.
+    """
+    discovery = RegistryManagementClient(registry_name=registry_name).discovery()
+    if as_arm_object:
+        arm_resource_id = f"/subscriptions/{discovery.get('subscriptionId')}/resourceGroups/{discovery.get('resourceGroup')}/providers/Microsoft.MachineLearningServices/registries/{discovery.get('registryName')}"
+        return ArmClient().get_resource(resource_id=arm_resource_id)
+    return discovery
+
+
+def _add_common_args(p, arg_dicts=None):
+    if arg_dicts is None:
+        arg_dicts = []
+    for arg in arg_dicts:
+        p.add_argument(*arg["args"], **arg["kwargs"])
+
+
+def main() -> None:
+    """Azureml Registry Syndication CLI Extension.
+
+    Examples:
+      # Show the current manifest
+      registry-mgmt syndication manifest show --registry-name myreg
+
+      # Set manifest from a folder
+      registry-mgmt syndication manifest set --path ./manifest_folder
+
+      # Set manifest from a value
+      registry-mgmt syndication manifest set --value '{"Manifest": "val"}'
+
+      # Delete the current manifest
+      registry-mgmt syndication manifest delete --registry-name myreg
+
+      # Show the current target
+      registry-mgmt syndication target show --registry-name myreg
+
+      # Set target values
+      registry-mgmt syndication target set --registry-name myreg -v reg1Id -v reg2Id
+
+      # Show registry discovery info
+      registry-mgmt show --registry-name myreg
+
+      # Show registry as ARM object
+      registry-mgmt show --registry-name myreg --as-arm-object
+
+      # Dry run for any command
+      registry-mgmt syndication manifest set --registry-name myreg --path ./manifest_folder --dry-run
+    """
+    parser = argparse.ArgumentParser(prog="registry-mgmt", description="AzureML Registry Syndication CLI Extension")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    registry_name_arg = {
+        "args": ("-r", "--registry-name"),
+        "kwargs": {
+            "type": str,
+            "required": True,
+            "help": "Name of the AzureML registry."
+        }
+    }
+
+    dry_run_arg = {
+        "args": ("--dry-run",),
+        "kwargs": {
+            "action": "store_true",
+            "help": "Perform a dry run without making changes."
+        }
+    }
+
+    # syndication root command
+    synd_parser = subparsers.add_parser("syndication", help="Syndication operations")
+    synd_subparsers = synd_parser.add_subparsers(dest="synd_subcommand", required=True)
+
+    # syndication manifest
+    manifest_parser = synd_subparsers.add_parser("manifest", help="Manage syndication manifest.")
+    manifest_subparsers = manifest_parser.add_subparsers(dest="manifest_subcommand", required=True)
+
+    manifest_show_parser = manifest_subparsers.add_parser("show", help="Show the current manifest.")
+    _add_common_args(manifest_show_parser, [registry_name_arg, dry_run_arg])
+
+    manifest_set_parser = manifest_subparsers.add_parser("set", help="Set manifest values.")
+    _add_common_args(manifest_set_parser, [dry_run_arg])
+    group = manifest_set_parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("-v", "--value", type=str, help="Manifest value.")
+    group.add_argument("-p", "--path", type=str, help="Path to manifest root folder.")
+
+    manifest_delete_parser = manifest_subparsers.add_parser("delete", help="Delete the current manifest.")
+    _add_common_args(manifest_delete_parser, [registry_name_arg, dry_run_arg])
+
+    # syndication target
+    target_parser = synd_subparsers.add_parser("target", help="Manage syndication target.")
+    target_subparsers = target_parser.add_subparsers(dest="target_subcommand", required=True)
+
+    target_show_parser = target_subparsers.add_parser("show", help="Show the current target.")
+    _add_common_args(target_show_parser, [registry_name_arg, dry_run_arg])
+
+    target_set_parser = target_subparsers.add_parser("set", help="Set target values.")
+    _add_common_args(target_set_parser, [registry_name_arg, dry_run_arg])
+    target_set_parser.add_argument("-v", "--value", type=str, action="append", required=True, help="Target value (can be specified multiple times).")
+
+    # show root command
+    show_parser = subparsers.add_parser("show", help="Show syndication info.")
+    _add_common_args(show_parser, [registry_name_arg, dry_run_arg])
+    show_parser.add_argument("--as-arm-object", action="store_true", help="Show as ARM object.")
+
+    args = parser.parse_args()
+
+    # Command dispatch
+    if args.command == "syndication":
+        if args.synd_subcommand == "manifest":
+            if args.manifest_subcommand == "show":
+                print(syndication_manifest_show(args.registry_name))
+            elif args.manifest_subcommand == "set":
+                print(syndication_manifest_set(args.value, args.path, args.dry_run))
+            elif args.manifest_subcommand == "delete":
+                confirm = input(f"Proceed with manifest deletion for {args.registry_name}? [y/N]: ")
+                if confirm.lower() == "y":
+                    syndication_manifest_delete(args.registry_name, args.dry_run)
+                else:
+                    print("Manifest deletion cancelled.")
+        elif args.synd_subcommand == "target":
+            if args.target_subcommand == "show":
+                print(syndication_target_show(args.registry_name))
+            elif args.target_subcommand == "set":
+                syndication_target_set(args.registry_name, args.value, args.dry_run)
+    elif args.command == "show":
+        print(show_command(args.registry_name, args.as_arm_object))
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

azureml/registry/_rest_client/__init__.py

@@ -0,0 +1,4 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""This module is for rest clients."""

azureml/registry/_rest_client/arm_client.py

@@ -0,0 +1,133 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from azure.identity import DefaultAzureCredential
+from .base_rest_client import BaseRestClient
+import time
+from json.decoder import JSONDecodeError
+
+DEFAULT_API_VERSION = "2025-04-01"  # Default API version for Azure Resource Manager
+
+
+class ArmClient(BaseRestClient):
+    """Simple Azure Resource Manager (ARM) client leveraging BaseRestClient for GET, PATCH, PUT, and DELETE operations.
+
+    Handles authentication via Bearer token (pass as api_key) and supports standard ARM resource operations.
+    """
+
+    def __init__(self, api_key=None, max_retries=5, backoff_factor=1):
+        """
+        Initialize the ArmClient.
+
+        Args:
+            api_key (str, optional): API key or bearer token. If None, uses DefaultAzureCredential.
+            max_retries (int): Maximum number of retries for requests.
+            backoff_factor (int): Backoff factor for retries.
+        """
+        base_url = "https://management.azure.com"
+        self._credential = None
+        self._token_expires_on = None
+        if api_key is None:
+            # Use DefaultAzureCredential for authentication if no API key is provided
+            self._credential = DefaultAzureCredential()
+            token = self._credential.get_token("https://management.azure.com/.default")
+            api_key = token.token
+            self._token_expires_on = token.expires_on
+        super().__init__(base_url, api_key=api_key, max_retries=max_retries, backoff_factor=backoff_factor)
+
+    def _refresh_api_key_if_needed(self) -> None:
+        """Refresh the API key if using DefaultAzureCredential and the token is close to expiration."""
+        if self._credential is not None:
+            now = int(time.time())
+            # Refresh if less than 10 minutes (600 seconds) left
+            if not self._token_expires_on or self._token_expires_on - now < 600:
+                token = self._credential.get_token("https://management.azure.com/.default")
+                self.api_key = token.token
+                self.session.headers.update({'Authorization': f'Bearer {self.api_key}'})
+                self._token_expires_on = token.expires_on
+
+    def get_resource(self, resource_id, api_version=DEFAULT_API_VERSION, **kwargs) -> object:
+        """
+        Get an ARM resource by its resource ID.
+
+        Args:
+            resource_id (str): The ARM resource ID.
+            api_version (str): The API version to use.
+            **kwargs: Additional arguments for the request.
+
+        Returns:
+            object: The resource as a dict if JSON, or text/None otherwise.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}{resource_id}?api-version={api_version}"
+        response = self.get(url, **kwargs)
+        try:
+            return response.json()
+        except JSONDecodeError:
+            raise ValueError(f"Failed to decode JSON response from {url}: {response.text.strip()}")
+
+    def patch_resource(self, resource_id, patch_body, api_version=DEFAULT_API_VERSION, **kwargs) -> object:
+        """
+        Patch an ARM resource.
+
+        Args:
+            resource_id (str): The ARM resource ID.
+            patch_body (dict): The patch body to send.
+            api_version (str): The API version to use.
+            **kwargs: Additional arguments for the request.
+
+        Returns:
+            object: The resource as a dict if JSON, or text/None otherwise.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}{resource_id}?api-version={api_version}"
+        headers = kwargs.pop('headers', {})
+        headers['Content-Type'] = 'application/json'
+        response = self.patch(url, json=patch_body, headers=headers, **kwargs)
+        try:
+            return response.json()
+        except JSONDecodeError:
+            return response.text or None
+
+    def put_resource(self, resource_id, put_body, api_version=DEFAULT_API_VERSION, **kwargs) -> object:
+        """
+        Put (create or update) an ARM resource.
+
+        Args:
+            resource_id (str): The ARM resource ID.
+            put_body (dict): The body to send for the resource.
+            api_version (str): The API version to use.
+            **kwargs: Additional arguments for the request.
+
+        Returns:
+            object: The resource as a dict if JSON, or text/None otherwise.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}{resource_id}?api-version={api_version}"
+        headers = kwargs.pop('headers', {})
+        headers['Content-Type'] = 'application/json'
+        response = self.put(url, json=put_body, headers=headers, **kwargs)
+        try:
+            return response.json()
+        except JSONDecodeError:
+            return response.text or None
+
+    def delete_resource(self, resource_id, api_version=DEFAULT_API_VERSION, **kwargs) -> object:
+        """
+        Delete an ARM resource by its resource ID.
+
+        Args:
+            resource_id (str): The ARM resource ID.
+            api_version (str): The API version to use.
+            **kwargs: Additional arguments for the request.
+
+        Returns:
+            object: The resource as a dict if JSON, or text/None otherwise.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}{resource_id}?api-version={api_version}"
+        response = self.delete(url, **kwargs)
+        try:
+            return response.json()
+        except JSONDecodeError:
+            return response.text or None

azureml/registry/_rest_client/base_rest_client.py

@@ -0,0 +1,138 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import requests
+import time
+import logging
+
+
+class BaseRestClient:
+    """Base REST client for making HTTP requests with retry logic and optional API key authentication."""
+
+    def __init__(self, base_url: str, api_key: str = None, max_retries: int = 5, backoff_factor: int = 1) -> None:
+        """
+        Initialize the BaseRestClient.
+
+        Args:
+            base_url (str): The base URL for the REST API.
+            api_key (str): Bearer token for authentication.
+            max_retries (int): Maximum number of retries for failed requests.
+            backoff_factor (int): Backoff factor for retry delays.
+        """
+        self.base_url = base_url.rstrip('/')
+        self.session = requests.Session()
+        if api_key:
+            self.session.headers.update({'Authorization': f'Bearer {api_key}'})
+        self.max_retries = max_retries
+        self.backoff_factor = backoff_factor
+
+    def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response:
+        """
+        Make an HTTP request with retry logic for 5xx and 429 errors.
+
+        Args:
+            method (str): HTTP method (GET, POST, etc.).
+            url (str): Full URL for the request.
+            **kwargs: Additional arguments for requests.Session.request.
+
+        Returns:
+            requests.Response: The HTTP response object.
+        """
+        retries = 0
+        while True:
+            logging.info(f"Attempt {retries + 1}: {method} {url}")
+            response = self.session.request(method, url, **kwargs)
+            if response.status_code < 500 and response.status_code != 429:
+                logging.info(f"Success: {method} {url} (status {response.status_code})")
+                response.raise_for_status()
+                return response
+            # Handle 429 (Too Many Requests)
+            if response.status_code == 429:
+                retry_after = response.headers.get("Retry-After")
+                if retry_after:
+                    try:
+                        sleep_time = int(retry_after)
+                    except ValueError:
+                        sleep_time = self.backoff_factor * (2 ** retries)
+                else:
+                    sleep_time = self.backoff_factor * (2 ** retries)
+                logging.info(f"Received 429. Retrying after {sleep_time} seconds...")
+            else:
+                # For all other 4xx errors, raise with details
+                error_msg = (
+                    f"HTTP {response.status_code} error for {method} {url}: "
+                    f"{response.text.strip()}"
+                )
+                logging.error(f"Error: {error_msg}")
+                raise Exception(error_msg)
+            retries += 1
+            if retries > self.max_retries:
+                logging.error(f"Max retries exceeded for {method} {url}")
+                response.raise_for_status()
+            logging.info(f"Sleeping for {sleep_time} seconds before retry...")
+            time.sleep(sleep_time)
+
+    def get(self, url: str, **kwargs) -> requests.Response:
+        """
+        Make a GET request with retry logic.
+
+        Args:
+            url (str): The URL to request.
+            **kwargs: Additional arguments for requests.Session.request.
+
+        Returns:
+            requests.Response: The HTTP response object.
+        """
+        return self._request_with_retry('GET', url, **kwargs)
+
+    def post(self, url: str, **kwargs) -> requests.Response:
+        """
+        Make a POST request with retry logic.
+
+        Args:
+            url (str): The URL to request.
+            **kwargs: Additional arguments for requests.Session.request.
+
+        Returns:
+            requests.Response: The HTTP response object.
+        """
+        return self._request_with_retry('POST', url, **kwargs)
+
+    def put(self, url: str, **kwargs) -> requests.Response:
+        """
+        Make a PUT request with retry logic.
+
+        Args:
+            url (str): The URL to request.
+            **kwargs: Additional arguments for requests.Session.request.
+
+        Returns:
+            requests.Response: The HTTP response object.
+        """
+        return self._request_with_retry('PUT', url, **kwargs)
+
+    def patch(self, url: str, **kwargs) -> requests.Response:
+        """
+        Make a PATCH request with retry logic.
+
+        Args:
+            url (str): The URL to request.
+            **kwargs: Additional arguments for requests.Session.request.
+
+        Returns:
+            requests.Response: The HTTP response object.
+        """
+        return self._request_with_retry('PATCH', url, **kwargs)
+
+    def delete(self, url: str, **kwargs) -> requests.Response:
+        """
+        Make a DELETE request with retry logic.
+
+        Args:
+            url (str): The URL to request.
+            **kwargs: Additional arguments for requests.Session.request.
+
+        Returns:
+            requests.Response: The HTTP response object.
+        """
+        return self._request_with_retry('DELETE', url, **kwargs)

azureml/registry/_rest_client/registry_management_client.py

@@ -0,0 +1,114 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from azure.identity import DefaultAzureCredential
+from .base_rest_client import BaseRestClient
+import time
+
+
+class RegistryManagementClient(BaseRestClient):
+    """Python client for RegistrySyndicationManifestController (excluding S2S APIs).
+
+    Handles authentication, token refresh, and provides methods for manifest and registry management.
+    """
+
+    def __init__(self, registry_name: str, primary_region: str = "eastus2euap", api_key: str = None, max_retries: int = 5, backoff_factor: int = 1) -> None:
+        """
+        Initialize the RegistryManagementClient.
+
+        Args:
+            primary_region (str): The Azure region for the registry.
+            registry_name (str): The name of the AzureML registry.
+            api_key (str, optional): Bearer token for authentication. If None, uses DefaultAzureCredential.
+            max_retries (int): Maximum number of retries for failed requests.
+            backoff_factor (int): Backoff factor for retry delays.
+        """
+        base_url = f"https://{primary_region}.api.azureml.ms"
+        self._credential = None
+        self._token_expires_on = None
+        if api_key is None:
+            # Use DefaultAzureCredential for authentication if no API key is provided
+            self._credential = DefaultAzureCredential()
+            token = self._credential.get_token("https://management.azure.com/.default")
+            api_key = token.token
+            self._token_expires_on = token.expires_on
+        super().__init__(base_url, api_key=api_key, max_retries=max_retries, backoff_factor=backoff_factor)
+        self.registry_name = registry_name
+
+    def _refresh_api_key_if_needed(self) -> None:
+        """Refresh the API key if using DefaultAzureCredential and the token is close to expiration."""
+        # Only refresh if using DefaultAzureCredential
+        if self._credential is not None:
+            now = int(time.time())
+            # Refresh if less than 10 minutes (600 seconds) left
+            if not self._token_expires_on or self._token_expires_on - now < 600:
+                token = self._credential.get_token("https://management.azure.com/.default")
+                self.api_key = token.token
+                self.session.headers.update({'Authorization': f'Bearer {self.api_key}'})
+                self._token_expires_on = token.expires_on
+
+    def create_or_update_manifest(self, manifest_dto: dict) -> dict:
+        """
+        Create or update the syndication manifest for the registry.
+
+        Args:
+            manifest_dto (dict): The manifest data transfer object.
+
+        Returns:
+            dict: The response from the service.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}/registrymanagement/v1.0/registrySyndication/{self.registry_name}/createOrUpdateManifest"
+        response = self.post(url, json=manifest_dto)
+        return response.json()
+
+    def resync_assets_in_manifest(self, resync_assets_dto: dict) -> dict:
+        """
+        Resynchronize assets in the syndication manifest.
+
+        Args:
+            resync_assets_dto (dict): The DTO specifying which assets to resync.
+
+        Returns:
+            dict: The response from the service.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}/registrymanagement/v1.0/registrySyndication/{self.registry_name}/resyncAssetsInManifest"
+        response = self.post(url, json=resync_assets_dto)
+        return response.json()
+
+    def delete_manifest(self) -> dict:
+        """
+        Delete the syndication manifest for the registry.
+
+        Returns:
+            dict: The response from the service.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}/registrymanagement/v1.0/registrySyndication/{self.registry_name}/deleteManifest"
+        response = self.post(url)
+        return response.json()
+
+    def get_manifest(self) -> dict:
+        """
+        Get the syndication manifest for the registry.
+
+        Returns:
+            dict: The manifest data.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}/registrymanagement/v1.0/registrySyndication/{self.registry_name}/getManifest"
+        response = self.get(url)
+        return response.json()
+
+    def discovery(self) -> dict:
+        """
+        Get discovery information for the registry.
+
+        Returns:
+            dict: The discovery information for the registry.
+        """
+        self._refresh_api_key_if_needed()
+        url = f"{self.base_url}/registrymanagement/v1.0/registries/{self.registry_name}/discovery"
+        response = self.get(url)
+        return response.json()

azureml/registry/mgmt/__init__.py

@@ -0,0 +1,4 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""This module is for manifest operations."""

azureml/registry/mgmt/create_manifest.py

@@ -0,0 +1,235 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Syndication manifest generator for AzureML registries.
+
+This script parses YAML files describing source registries and assets, validates them, and generates a syndication
+manifest for AzureML registries.
+"""
+
+import os
+import glob
+import argparse
+import json
+from ruamel.yaml import YAML
+from .syndication_manifest import Asset, SourceRegistry, SyndicationManifest
+
+REQUIRED_SOURCE_FIELDS = ['name', 'tenant_id', 'is_ipp', 'assets']
+REQUIRED_REGISTRY_FIELDS = ['tenant_id', 'name']
+ALLOWED_ASSET_TYPES = {
+    'environments', 'models', 'deployment-templates', 'components', 'datasets'
+}
+
+ASSET_TYPE_MAP = {
+    'models': 'Models',
+    'environments': 'Environments',
+    'deployment-templates': 'DeploymentTemplates',
+    'components': 'Components',
+    'datasets': 'Datasets',
+}
+
+
+def validate_manifest_data(data: dict, file_path: str) -> None:
+    """Validate required root-level fields and allowed values for nested fields under 'assets'.
+
+    Args:
+        data (dict): Parsed YAML content.
+        file_path (str): Path to the YAML file (for error reporting).
+
+    Raises:
+        ValueError: If required fields are missing or invalid values are found.
+    """
+    missing = [field for field in REQUIRED_SOURCE_FIELDS if field not in data]
+    if missing:
+        raise ValueError(
+            f"Missing required field(s) in {file_path}: {', '.join(missing)}"
+        )
+    assets = data.get('assets', {})
+    if not isinstance(assets, dict):
+        raise ValueError(f"'assets' field in {file_path} must be a dictionary.")
+    for asset_type in assets:
+        if asset_type not in ALLOWED_ASSET_TYPES:
+            raise ValueError(
+                f"Invalid asset type '{asset_type}' in 'assets' of {file_path}. Allowed: "
+                f"{', '.join(ALLOWED_ASSET_TYPES)}"
+            )
+
+
+def parse_asset_objs(asset_type: str, asset_dict: dict) -> list:
+    """Parse asset objects from the asset dictionary for a given asset type.
+
+    Args:
+        asset_type (str): The asset type (e.g., 'models').
+        asset_dict (dict or str): Dictionary of asset names to version lists or dicts, or a wildcard string.
+
+    Returns:
+        list: List of Asset objects.
+    """
+    assets = []
+    # Handle wildcard string case: environments: "*"
+    if asset_dict == "*":
+        assets.append(Asset(name=".*", version=".*"))
+        return assets
+    if not asset_dict or not isinstance(asset_dict, dict):
+        return assets
+    for asset_name, asset_info in asset_dict.items():
+        if isinstance(asset_info, dict) and 'versions' in asset_info:
+            versions = asset_info['versions']
+        elif isinstance(asset_info, list):
+            versions = asset_info
+        else:
+            versions = [asset_info]
+        for version in versions:
+            assets.append(Asset(name=asset_name, version=version))
+    return assets
+
+
+def read_manifest_files(folder: str) -> dict:
+    """Recursively read all YAML files in the 'sources' subdirectory and validate required root-level fields.
+
+    Args:
+        folder (str): Root folder containing the 'sources' directory.
+
+    Returns:
+        dict: Mapping of file paths to loaded SourceRegistry instances.
+
+    Raises:
+        ValueError: If required fields are missing in any YAML file.
+    """
+    sources_path = os.path.join(folder, 'sources')
+    yaml_files = glob.glob(os.path.join(sources_path, '**', '*.yaml'), recursive=True)
+    contents = {}
+    yaml = YAML(typ='safe')
+    for file_path in yaml_files:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = yaml.load(f)
+        validate_manifest_data(data, file_path)
+        assets = {}
+        for yaml_key in ALLOWED_ASSET_TYPES:
+            asset_objs = parse_asset_objs(yaml_key, data.get('assets', {}).get(yaml_key))
+            if asset_objs:
+                assets[yaml_key] = asset_objs
+        source_registry = SourceRegistry(
+            registry_name=data['name'],
+            tenant_id=data['tenant_id'],
+            assets=assets
+        )
+        contents[file_path] = source_registry
+    return contents
+
+
+def _load_registry_yaml(folder: str) -> SyndicationManifest:
+    """Load 'registry.yaml' from the specified folder into a SyndicationManifest instance (without SourceRegistries).
+
+    Args:
+        folder (str): Folder containing the 'registry.yaml' file.
+
+    Returns:
+        SyndicationManifest: Instance populated from the YAML file (SourceRegistries is an empty list), with
+            _allow_wildcards set.
+
+    Raises:
+        FileNotFoundError: If 'registry.yaml' is not found in the folder.
+        ValueError: If required fields are missing.
+    """
+    registry_path = os.path.join(folder, 'registry.yaml')
+    if not os.path.isfile(registry_path):
+        raise FileNotFoundError(f"{registry_path} not found.")
+    yaml = YAML(typ='safe')
+    with open(registry_path, 'r', encoding='utf-8') as f:
+        data = yaml.load(f)
+    missing = [field for field in REQUIRED_REGISTRY_FIELDS if field not in data]
+    if missing:
+        raise ValueError(
+            f"Missing required field(s) in registry.yaml: {', '.join(missing)}"
+        )
+    # Read allow_wildcards flag, default to False if not present
+    allow_wildcards = data.get('allow_wildcards', False)
+    return SyndicationManifest(
+        registry_name=data['name'],
+        tenant_id=data['tenant_id'],
+        source_registries=[],
+        _allow_wildcards=allow_wildcards
+    )
+
+
+def generate_syndication_manifest(folder: str) -> SyndicationManifest:
+    """Load the destination registry and all source registries manifests from the specified folder.
+
+    This function loads the destination registry's configuration, all source registries, validates asset uniqueness
+    and wildcard usage, and returns a complete SyndicationManifest object.
+
+    Args:
+        folder (str): Root folder containing the manifest for the sources.
+
+    Returns:
+        SyndicationManifest: The complete syndication manifest with destination and source registries.
+
+    Raises:
+        ValueError: If asset names are not unique across all SourceRegistry instances, or if wildcards are not allowed
+            but used.
+    """
+    # Load the destination registry manifest and allow_wildcards flag
+    manifest = _load_registry_yaml(folder)
+    allow_wildcards = manifest._allow_wildcards
+    # Read all source registry manifests from the sources directory
+    source_registries_dict = read_manifest_files(folder)
+    source_registries = list(source_registries_dict.values())
+
+    # Validate uniqueness of asset names within the same asset type across all SourceRegistry instances
+    asset_types = [
+        ('Models', 'models'),
+        ('Environments', 'environments'),
+        ('DeploymentTemplates', 'deployment-templates'),
+        ('Components', 'components'),
+        ('Datasets', 'datasets'),
+    ]
+    for class_attr, yaml_key in asset_types:
+        registry_asset_map = dict()  # asset_name -> set of (registry_name, uri)
+        duplicates = dict()  # asset_name -> set of uris
+        for sr in source_registries:
+            asset_list = sr.assets.get(yaml_key, [])
+            for asset in asset_list:
+                uri = f"azureml://registries/{sr.registry_name}/{yaml_key}/name/{asset.name}/versions/{asset.version}"
+                # If allow_wildcards is False, disallow wildcard versions in asset URIs
+                if not allow_wildcards:
+                    if asset.version == ".*" or asset.version == "*":
+                        raise ValueError(
+                            f"Wildcard asset version not allowed (allow_wildcards is False): {uri}"
+                        )
+                # Extract asset name from URI for uniqueness validation
+                asset_name = asset.name
+                registry_name = sr.registry_name
+                if asset_name not in registry_asset_map:
+                    registry_asset_map[asset_name] = set()
+                registry_asset_map[asset_name].add((registry_name, uri))
+        # Check for duplicate asset names across registries for this asset type
+        for asset_name, reg_uris in registry_asset_map.items():
+            registries = {reg for reg, _ in reg_uris}
+            if len(registries) > 1:
+                uris = {uri for _, uri in reg_uris}
+                duplicates[asset_name] = uris
+        if duplicates:
+            error_lines = []
+            for asset_name, uris in sorted(duplicates.items()):
+                error_lines.append(
+                    f"Asset name '{asset_name}' for asset type '{yaml_key}' is duplicated across registries in URIs: "
+                    + ", ".join(sorted(uris))
+                )
+            raise ValueError("\n".join(error_lines))
+
+    # Populate SourceRegistries in the SyndicationManifest instance
+    manifest.source_registries = source_registries
+    return manifest
+
+
+if __name__ == "__main__":
+    # Parse command-line arguments for the manifest folder
+    parser = argparse.ArgumentParser(description="Syndication manifest structure.")
+    parser.add_argument(
+        '-f', '--folder', type=str, help='Registry syndication manifest folder', required=True
+    )
+    args = parser.parse_args()
+    # Generate the syndication manifest and print as formatted JSON
+    manifest = generate_syndication_manifest(args.folder)
+    print(json.dumps(manifest.to_dto(), indent=2))

azureml/registry/mgmt/syndication_manifest.py

@@ -0,0 +1,150 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Syndication manifest dataclasses and validation for AzureML registry asset syndication."""
+
+from dataclasses import dataclass, field
+from typing import List
+from uuid import UUID
+
+
+def _norm_key(k):
+    return k.replace('_', '').lower()
+
+
+def _get_key(dct, *keys, normalize_keys=False):
+    """Normalize and retrieve a value from a dict by trying multiple key casings.
+
+    If normalize_keys is True, normalize the dict keys before searching.
+    """
+    if normalize_keys:
+        dct_norm = {_norm_key(k): v for k, v in dct.items()}
+        for k in keys:
+            nk = _norm_key(k)
+            if nk in dct_norm:
+                return dct_norm[nk]
+        return None
+    else:
+        for k in keys:
+            if k in dct:
+                return dct[k]
+        return None
+
+
+@dataclass
+class Asset:
+    """Represents a single asset with a name and version."""
+
+    name: str
+    version: str
+
+    def to_dict(self) -> dict:
+        """Convert the Asset instance to a dictionary with serialized field names."""
+        return {"Name": self.name, "Version": self.version}
+
+    @staticmethod
+    def from_dict(d: dict, normalize_keys=False) -> "Asset":
+        """Create an Asset instance from a dictionary with serialized field names."""
+        return Asset(
+            name=_get_key(d, "Name", normalize_keys=normalize_keys),
+            version=_get_key(d, "Version", normalize_keys=normalize_keys) or ".*"
+        )
+
+
+@dataclass
+class SourceRegistry:
+    """Represents a source registry in the syndication manifest.
+
+    Attributes:
+        registry_name (str): The name of the source registry.
+        tenant_id (UUID): The Azure tenant ID for the source registry.
+        assets (dict): Dictionary mapping asset type to list of Asset objects.
+    """
+
+    registry_name: str
+    tenant_id: UUID
+    assets: dict  # asset_type (str) -> List[Asset]
+
+    def to_dict(self) -> dict:
+        """Convert the SourceRegistry instance to a dictionary with serialized field names."""
+        asset_dict = {}
+        for asset_type, asset_list in self.assets.items():
+            if asset_list:
+                asset_dict[asset_type] = [
+                    {"Name": a.name} if a.name == ".*" else {"Name": a.name, "Version": a.version}
+                    for a in asset_list
+                ]
+        return {
+            "RegistryName": self.registry_name,
+            "TenantId": str(self.tenant_id),
+            "Assets": asset_dict
+        }
+
+    @staticmethod
+    def from_dict(d: dict, normalize_keys=False) -> "SourceRegistry":
+        """Create a SourceRegistry instance from a dictionary with any key casing (snake, camel, Pascal)."""
+        assets = {}
+        assets_dict = _get_key(d, "Assets", "assets", normalize_keys=normalize_keys)
+        for asset_type, asset_list in assets_dict.items():
+            assets[asset_type] = [Asset.from_dict(a, normalize_keys=normalize_keys) for a in asset_list]
+        return SourceRegistry(
+            registry_name=_get_key(d, "RegistryName", "registry_name", normalize_keys=normalize_keys),
+            tenant_id=UUID(str(_get_key(d, "TenantId", "tenant_id", normalize_keys=normalize_keys))),
+            assets=assets
+        )
+
+
+@dataclass
+class SyndicationManifest:
+    """Represents the root syndication manifest for a destination registry.
+
+    Attributes:
+        registry_name (str): The name of the destination registry.
+        tenant_id (UUID): The Azure tenant ID for the destination registry.
+        source_registries (List[SourceRegistry]): All source registries.
+        _allow_wildcards (bool): Internal flag for wildcard version validation (not serialized).
+    """
+
+    registry_name: str
+    tenant_id: UUID
+    source_registries: List[SourceRegistry]
+    _allow_wildcards: bool = field(default=False, repr=False, compare=False)
+
+    def to_dict(self) -> dict:
+        """Convert the SyndicationManifest instance to a dictionary with serialized field names."""
+        return {
+            "RegistryName": self.registry_name,
+            "TenantId": str(self.tenant_id),
+            "SourceRegistries": [sr.to_dict() for sr in self.source_registries]
+        }
+
+    def to_dto(self) -> dict:
+        """Serialize the manifest as a value of {"Manifest": SyndicationManifest} for external consumers."""
+        return {"Manifest": self.to_dict()}
+
+    @staticmethod
+    def from_dto(dto: dict, normalize_keys=False) -> "SyndicationManifest":
+        """Deserialize a SyndicationManifest from a dictionary produced by to_dto, with validation. Handles any key casing."""
+        if not isinstance(dto, dict) or _get_key(dto, "Manifest", normalize_keys=normalize_keys) is None:
+            raise ValueError("Input must be a dict with a 'Manifest' key.")
+        manifest = _get_key(dto, "Manifest", normalize_keys=normalize_keys)
+        if not isinstance(manifest, dict):
+            raise ValueError("'Manifest' value must be a dict.")
+        for field_name in ("RegistryName", "TenantId", "SourceRegistries"):
+            if _get_key(manifest, field_name, normalize_keys=normalize_keys) is None:
+                raise ValueError(f"Missing required field '{field_name}' in Manifest.")
+        registry_name = _get_key(manifest, "RegistryName", "registry_name", normalize_keys=normalize_keys)
+        tenant_id = _get_key(manifest, "TenantId", "tenant_id", normalize_keys=normalize_keys)
+        try:
+            tenant_id = UUID(str(tenant_id))
+        except Exception:
+            raise ValueError("TenantId must be a valid UUID string.")
+        sr_list = _get_key(manifest, "SourceRegistries", "source_registries", normalize_keys=normalize_keys)
+        if not isinstance(sr_list, list):
+            raise ValueError("SourceRegistries must be a list.")
+        source_registries = [SourceRegistry.from_dict(sr, normalize_keys=normalize_keys) for sr in sr_list]
+        return SyndicationManifest(
+            registry_name=registry_name,
+            tenant_id=tenant_id,
+            source_registries=source_registries
+        )

{azureml_registry_tools-0.1.0a1.dist-info → azureml_registry_tools-0.1.0a2.dist-info}/METADATA

@@ -1,11 +1,13 @@
 Metadata-Version: 2.4
 Name: azureml-registry-tools
-Version: 0.1.0a1
+Version: 0.1.0a2
 Summary: AzureML Registry tools and CLI
 Author: Microsoft Corp
 License: https://aka.ms/azureml-sdk-license
-Requires-Python: >=3.9,<3.12
+Requires-Python: >=3.9,<3.14
 License-File: LICENSE.txt
+Requires-Dist: azure-identity<2.0
+Requires-Dist: ruamel-yaml<0.19,>=0.17.21
 Requires-Dist: azure-ai-ml<2.0
 Requires-Dist: azureml-assets<2.0
 Dynamic: author

azureml_registry_tools-0.1.0a2.dist-info/RECORD

@@ -0,0 +1,23 @@
+azureml/__init__.py,sha256=_gYz_vABVrp9EyOZEYn4Ya8XotJC2rZn-b9kFTEj1Dk,266
+azureml/registry/__init__.py,sha256=qjP86n1Yz6hdZb3az-wjfi7LNndsFjMQI6lXmKfXapM,266
+azureml/registry/_cli/__init__.py,sha256=dJ020ynrregpUaNGu8xVLLmX3cC9DAjaBMP1qnljLEE,208
+azureml/registry/_cli/registry_syndication_cli.py,sha256=87hxSu4iwFOdwfukSGqc0fjJEpSpQhDRlthiCGCDwQ8,10003
+azureml/registry/_cli/repo2registry_cli.py,sha256=4CDv4tYWLTjVgdXIcoKA9iu_gOv_Z_xn05wSANkdmdg,5378
+azureml/registry/_rest_client/__init__.py,sha256=Eh0vB8AVlwkZlHLO4DCGHyyfRYYgeTeVjiUAX_WujG0,219
+azureml/registry/_rest_client/arm_client.py,sha256=dgm9nDnmGI1NIZaopXCIXmViM8sSAipZETfuOWnQRa0,5638
+azureml/registry/_rest_client/base_rest_client.py,sha256=cXFYvYgSYSveEeY511VIlDpmXLOY9Jat_E-Zs1ehMyA,5164
+azureml/registry/_rest_client/registry_management_client.py,sha256=JGBohCVpL8XR4rNuScOKt7tNQnjw203pqlTSLKVjYgM,4833
+azureml/registry/mgmt/__init__.py,sha256=LMhqcEC8ItmmpKZljElGXH-6olHlT3SLl0dJU01OvuM,226
+azureml/registry/mgmt/create_manifest.py,sha256=N9wRmjAKO09A3utN_lCUsM_Ufpj7PL0SJz-XHPHWuyM,9528
+azureml/registry/mgmt/syndication_manifest.py,sha256=ma1_vUEHHtkRcgiFMQanlXZajvKodynylxfblTHuleE,6195
+azureml/registry/tools/__init__.py,sha256=IAuWWpGfZm__pAkBIxmpJz84QskpkxBr0yDk1TUSnkE,223
+azureml/registry/tools/config.py,sha256=tjPaoBsWtPXBL8Ww1hcJtsr2SuIjPKt79dR8iovcebg,3639
+azureml/registry/tools/create_or_update_assets.py,sha256=NexSY22bpN2eHPqZZ21eDIMerwq3SzVk_4fBV5_GTBY,14787
+azureml/registry/tools/registry_utils.py,sha256=zgYlCiOONtQJ4yZ9wg8tKVoE8dh6rrjB8hYBGhpV9-0,1403
+azureml/registry/tools/repo2registry_config.py,sha256=eXp_tU8Jyi30g8xGf7wbpLgKEPpieohBANKxMSLzq7s,4873
+azureml_registry_tools-0.1.0a2.dist-info/licenses/LICENSE.txt,sha256=n20rxwp7_NGrrShv9Qvcs90sjI1l3Pkt3m-5OPCWzgs,845
+azureml_registry_tools-0.1.0a2.dist-info/METADATA,sha256=SJpiD2SsYXqJi_DUSLk-mquFZgbnb54Sd_mrP6H-GBI,491
+azureml_registry_tools-0.1.0a2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+azureml_registry_tools-0.1.0a2.dist-info/entry_points.txt,sha256=iRUkAeQidMnO6RQzpLqMUBTcyYtNzAfSin9WnSdVGLw,147
+azureml_registry_tools-0.1.0a2.dist-info/top_level.txt,sha256=ZOeEa0TAXo6i5wOjwBoqfIGEuxOcKuscGgNSpizqREY,8
+azureml_registry_tools-0.1.0a2.dist-info/RECORD,,

{azureml_registry_tools-0.1.0a1.dist-info → azureml_registry_tools-0.1.0a2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{azureml_registry_tools-0.1.0a1.dist-info → azureml_registry_tools-0.1.0a2.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+registry-mgmt = azureml.registry._cli.registry_syndication_cli:main
 repo2registry = azureml.registry._cli.repo2registry_cli:main

azureml_registry_tools-0.1.0a1.dist-info/RECORD

@@ -1,15 +0,0 @@
-azureml/__init__.py,sha256=_gYz_vABVrp9EyOZEYn4Ya8XotJC2rZn-b9kFTEj1Dk,266
-azureml/registry/__init__.py,sha256=qjP86n1Yz6hdZb3az-wjfi7LNndsFjMQI6lXmKfXapM,266
-azureml/registry/_cli/__init__.py,sha256=dJ020ynrregpUaNGu8xVLLmX3cC9DAjaBMP1qnljLEE,208
-azureml/registry/_cli/repo2registry_cli.py,sha256=4CDv4tYWLTjVgdXIcoKA9iu_gOv_Z_xn05wSANkdmdg,5378
-azureml/registry/tools/__init__.py,sha256=IAuWWpGfZm__pAkBIxmpJz84QskpkxBr0yDk1TUSnkE,223
-azureml/registry/tools/config.py,sha256=tjPaoBsWtPXBL8Ww1hcJtsr2SuIjPKt79dR8iovcebg,3639
-azureml/registry/tools/create_or_update_assets.py,sha256=NexSY22bpN2eHPqZZ21eDIMerwq3SzVk_4fBV5_GTBY,14787
-azureml/registry/tools/registry_utils.py,sha256=zgYlCiOONtQJ4yZ9wg8tKVoE8dh6rrjB8hYBGhpV9-0,1403
-azureml/registry/tools/repo2registry_config.py,sha256=eXp_tU8Jyi30g8xGf7wbpLgKEPpieohBANKxMSLzq7s,4873
-azureml_registry_tools-0.1.0a1.dist-info/licenses/LICENSE.txt,sha256=n20rxwp7_NGrrShv9Qvcs90sjI1l3Pkt3m-5OPCWzgs,845
-azureml_registry_tools-0.1.0a1.dist-info/METADATA,sha256=o-QaSt2Mmc9w9okaay-Rv3tnWaHxNJszkP44iN9dl7M,415
-azureml_registry_tools-0.1.0a1.dist-info/WHEEL,sha256=L0N565qmK-3nM2eBoMNFszYJ_MTx03_tQ0CQu1bHLYo,91
-azureml_registry_tools-0.1.0a1.dist-info/entry_points.txt,sha256=tiXL5l4P1PsN9pEkj0SiSgzB_zi0MFz8FqMw1ixzqKo,79
-azureml_registry_tools-0.1.0a1.dist-info/top_level.txt,sha256=ZOeEa0TAXo6i5wOjwBoqfIGEuxOcKuscGgNSpizqREY,8
-azureml_registry_tools-0.1.0a1.dist-info/RECORD,,