ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl

ciocore/package_tree.py

@@ -1,50 +1,325 @@
-"""This module represents the list of software packages as a tree.
+"""
+A class to provide available packages as DAG structure. In reality however, the structure is just two levels deep: **hosts** and **plugins**. 
 
-It has methods and functions to traverse the tree to find
-packages and build paths etc.
+* DCCs such as **Maya** and **Cinema4D** are top-level host packages. 
+* Renderers and other plugins are children of those hosts.
 
-See tests in /tests/test_package_tree.py for additional usage info.
-"""
+Methods are provided to traverse the tree to find packages by name, version, platform and so on. If you are writing submission tools there's no need to create a Package tree directly. It is recommended to use the singleton module: [ciocore.data](/data/).
 
+The only functions you should need from this module are:
+
+* [supported_host_names()](/package_tree/#ciocore.package_tree.PackageTree.supported_host_names)
+* [supported_plugins()](/package_tree/#ciocore.package_tree.PackageTree.supported_plugins)
+"""
 
 import copy
 import json
 
-from ciocore.package_environment import PackageEnvironment
+WINDOWS = "windows"
+LINUX = "linux"
 
-WINDOWS="windows"
-LINUX="linux"
+#: The set of supported platforms, currently windows and linux.
+PLATFORMS = {WINDOWS, LINUX}
 
-PLATFORMS={WINDOWS, LINUX}
+class PackageTree(object):
+    def __init__(self, packages,  *host_products, **kwargs):
+        """Build the tree with a list of packages.
 
-def remove_unreachable(paths):
-    """Remove unreachable paths.
+        Args:
+            packages (list): List of packages direct from the [Conductor packages endpoint](https://dashboard.conductortech.com/api/v1/ee/packages).
 
-    Given some paths, remove those for which there are not additional paths present at each level up
-    the hierarchy to the host. It is possible to have orphaned paths if a package can be reached by
-    more than one path. For example, a shader library may be compatible with 2 versions of a
-    renderer, but only one of those renderers is compatible with this host. We need to remove the
-    entry that references a different host.
-    """
-    results = []
-    previous = ""
-    for path in sorted(paths):
-        parts = path.split("/")
-        valid_subpath = path == "%s/%s" % (previous, parts[-1])
-        if len(parts) == 1 or valid_subpath:
-            results.append(path)
-            previous = path
-    return results
+            *host_products: Filter the tree to contain only top-level host packages of products specified in this list and their plugins. If there are no host_products specified, and the product keyword is omitted, the tree contains all packages.
+
+        Keyword Args:
+            product (str): Build the tree from versions of a single product and its compatible plugins. Defaults to `None`, in which case the tree is built from all packages. It is an error to specify both host_products and product. If a nonexistent product is given, the PackageTree is empty. By specifying `product`, you can build the object based on a single plugin product.
+            platforms (set): Build the tree from versions for a specific platform. Defaults to the set `{"linux", "windows"}`.
+
+        Raises:
+            KeyError: An invalid platform was provided.
+            ValueError: Cannot choose both product and host_products.
+
+        Example:
+            >>> from ciocore import api_client, package_tree
+            # Request packages as a flat list from Conductor.
+            >>> packages = api_client.request_software_packages()
+            # Build tree of dependencies from packages list
+            >>> pt = package_tree.PackageTree(packages, "cinema4d", "maya-io")
+            >>> for path in pt.to_path_list():
+            >>>     print(path)
+            cinema4d 22.118.RB320081 linux
+            cinema4d 22.118.RB320081 linux/redshift-cinema4d 3.0.43 linux
+            cinema4d 22.118.RB320081 linux/redshift-cinema4d 3.0.45 linux
+            maya-io 2022.SP3 linux
+        """
+
+
+        platforms = kwargs.get("platforms",PLATFORMS)
+        product=kwargs.get("product")
+
+        unknown_platforms = set(platforms) - PLATFORMS
+        if unknown_platforms:
+            raise KeyError("Unrecognized platform {}".format(" ".join(unknown_platforms)))
+
+        if product and host_products:
+            raise ValueError("You cannot choose both product and host_products.")
+
+        packages = [_clean_package(p) for p in packages if p["platform"] in platforms]
+
+        root_ids = [] 
+        if product:
+             root_ids = [p["package_id"] for p in packages if p["product"] == product]
+        else:
+            for p in packages:
+                if not p["plugin_host_product"]:
+                    if p["product"] in host_products or not host_products:
+                        root_ids.append(p["package_id"])
+
+        self._tree = _build_tree(packages, {"children": [], "plugins": root_ids})
+
+    def supported_host_names(self):
+        """
+        All host names from the software tree.
+
+        These names can be used to populate a dropdown menu. Then a single selection from that menu
+        can be used to retrieve the complete package in order to generate an environment dictionary
+        and get package IDs.
+
+        Returns:
+            list(str): Fully qualified DCC hosts of the form: `product version platform`.
+
+        Example:
+            >>> from ciocore import api_client, package_tree
+            >>> packages = api_client.request_software_packages()
+            >>> pt = package_tree.PackageTree(packages, product="cinema4d")
+            >>> pt.supported_host_names()
+            cinema4d 21.209.RB305619 linux
+            cinema4d 22.116.RB316423 linux
+            cinema4d 22.118.RB320081 linux
+            cinema4d 23.110.RB330286 linux
+            cinema4d 24.111 linux
+            cinema4d 24.111 windows
+        """
+
+        paths = []
+        for pkg in self._tree["children"]:
+            paths.append(to_name(pkg))
+        return sorted(paths)
+
+    def supported_plugins(self, host):
+        """
+        Find the plugins that are children of the given DCC host.
+
+        The result does not contain platform information since we assume that plugins are compatible with the DCC host that was used to request them.
+
+        Args:
+            host (str): Name of the DCC host, typically one of the entries returned by [supported_host_names()](/package_tree/#ciocore.package_tree.PackageTree.supported_host_names).
+
+        Returns:
+            list(dict): Each entry contains a plugin product and a list of versions.
+
+        Example:
+            >>> from ciocore import api_client, package_tree 
+            >>> packages = api_client.request_software_packages() 
+            >>> pt = package_tree.PackageTree(packages, product="cinema4d") 
+            >>> name = pt.supported_host_names()[0]
+            >>> pt.supported_plugins(name)
+            [
+                {
+                    "plugin": "arnold-cinema4d",
+                    "versions": [
+                        "3.3.2.100",
+                        "3.3.3.0"
+                    ]
+                },
+                {
+                    "plugin": "redshift-cinema4d",
+                    "versions": [
+                        "2.6.54",
+                        "2.6.56",
+                        "3.0.21",
+                        "3.0.22",
+                    ],
+                },
+            ]
+        """
+
+        try:
+            subtree = self.find_by_name(host)
+            plugin_versions = _to_path_list(subtree)
+        except TypeError:
+            return []
+
+        if not plugin_versions:
+            return []
+
+        plugin_dict = {}
+        for plugin, version, _ in [pv.split(" ") for pv in plugin_versions]:
+            if plugin not in plugin_dict:
+                plugin_dict[plugin] = []
+            plugin_dict[plugin].append(version)
+
+        # convert to list so it can be sorted
+        plugins = []
+        for key in plugin_dict:
+            plugins.append({"plugin": key, "versions": sorted(plugin_dict[key])})
+
+        return sorted(plugins, key=lambda k: k["plugin"])
+
+    def find_by_name(self, name, limit=None):
+        """
+        Search the tree for a product with the given name.
+
+        Args:
+            name (str): The name constructed from the package using to_name(). It must be an exact match with product, version, and platform. For example: `maya 2018.0 windows`
+
+        Keyword Args:
+            limit (int): Limit the search depth. Defaults to `None`.
+
+        Returns:
+            object: The package that matches.
+
+        Example:
+            >>> from ciocore import api_client, package_tree
+            >>> packages = api_client.request_software_packages()
+            >>> pt = package_tree.PackageTree(packages, product="cinema4d")
+            >>> pt.find_by_name("redshift-cinema4d 3.0.64 linux")
+            {
+                'platform': 'linux',
+                'plugin_host_product': 'cinema4d',
+                'product': 'redshift-cinema4d',
+                'major_version': '3',
+                'release_version': '64',
+                'vendor': 'maxon',
+                'children': [],
+                ...
+            }
+        """
+
+        return _find_by_name(self._tree, name, limit, 0)
+
+    def find_by_path(self, path):
+        """
+        Find the package uniquely described by the given path.
+
+        The path is of the form returned by the to_path_list() method.
+
+        Args:
+            path (str): The path
+
+        Returns:
+            object: The package or None if no package exists with the given path.
+
+        Example:
+            >>> from ciocore import api_client, package_tree, package_environment
+            >>> packages = api_client.request_software_packages()
+            >>> pt = package_tree.PackageTree(packages, product="cinema4d")
+            >>> pt.find_by_path("cinema4d 24.111 linux/redshift-cinema4d 3.0.62 linux")
+            {
+                'platform': 'linux',
+                'plugin_host_product': 'cinema4d',
+                'product': 'redshift-cinema4d',
+                'major_version': '3',
+                'release_version': '62',
+                'vendor': 'maxon',
+                'children': [],
+                'plugin_host_version': "24",
+                ...
+            }
+        """
+        return _find_by_path(self._tree, path)
+
+
+    def to_path_list(self, name=None):
+        """
+        Get paths to all nodes.
+
+        This is useful for populating a chooser to choose packages fully qualified by path.
+        Houdini's tree widget, for example, takes the below format unchanged and generates the
+        appropriate UI.
+
+        Args:
+            name (str): Get paths below the tree represented by the name. Defaults to None (root node).
+
+        Returns:
+            list(str): Paths to all nodes in the tree.
+
+        Example:
+            >>> from ciocore import api_client, package_tree
+            >>> packages = api_client.request_software_packages()
+            >>> pt = package_tree.PackageTree(packages, product="cinema4d")
+            >>> pt.to_path_list()
+            cinema4d 22.118.RB320081 linux
+            cinema4d 22.118.RB320081 linux/redshift-cinema4d 3.0.43 linux
+            cinema4d 22.118.RB320081 linux/redshift-cinema4d 3.0.45 linux
+            cinema4d 22.118.RB320081 linux/redshift-cinema4d 3.0.22 linux
+            cinema4d 22.118.RB320081 linux/arnold-cinema4d 3.3.2.100 linux
+            ...
+
+            >>> pt.to_path_list(name="cinema4d 24.111 linux")
+            redshift-cinema4d 3.0.57 linux
+            redshift-cinema4d 3.0.62 linux
+            redshift-cinema4d 3.0.45 linux
+            redshift-cinema4d 3.0.64 linux
+        """
+        if name:
+            subtree = self.find_by_name(name)
+            return _to_path_list(subtree)
+        return _to_path_list(self._tree)
+
+    def platforms(self):
+        """
+        Get the platforms represented by packages in the tree.
+
+        Returns:
+            set: The set of platforms.
+        """
+
+        # No need to recurse. Plugins are assumed to be compatible with the host.
+        return set([host["platform"] for host in self._tree["children"]])
+
+    def json(self):
+        """
+        The whole tree of softwware as json.
+
+        Returns:
+            str: JSON.
+
+        """
+        return json.dumps(self._tree)
+
+    def __bool__(self):
+        return True if self._tree["children"] else False
+
+    def as_dict(self):
+        """
+        Returns:
+            dict: The underlying software dictionary.
+
+        """
+        return self._tree
 
 
 def to_name(pkg):
-    """Name like `houdini 16.5.323` or `maya 2016.SP3`.
+    """
+    Generate a name like `houdini 16.5.323 linux` or `maya 2016.SP3 linux`.
 
     This name is derived from the product and version fields in a package. Note: It is not
-    necessarily possible to go the other way and extract version fields from the name. It's purpose
-    is to enable path construction to uniquely identify a package. For example, houdini
-    16.0.736/arnold-houdini 2.0.2.2/al-shaders 1.0
-    """
+    necessarily possible to go the other way and extract version fields from the name.
+
+    Args:
+        pkg (object): An object with product, platform, and all version fields.
+
+    Returns:
+        str: The package name.
+
+    Examples:
+        >>> from ciocore import api_client, package_tree
+        >>> packages = api_client.request_software_packages()
+        >>> package_tree.to_name(packages[0])
+        redshift-maya 3.0.64 linux
+
+    """ 
+
     version_parts = [
         pkg["major_version"],
         pkg["minor_version"],
@@ -52,15 +327,15 @@ def to_name(pkg):
         pkg["build_version"],
     ]
     version_string = (".").join([p for p in version_parts if p])
-    platform = pkg.get("platform")
-    return " ".join(filter(None, [pkg["product"], version_string, platform]))
-
+    if pkg["platform"] not in PLATFORMS:
+        raise KeyError("Invalid platform: {}".format(pkg["platform"]))
+    return " ".join(filter(None, [pkg["product"], version_string, pkg["platform"]]))
 
 
 def _build_tree(packages, package):
     """Build a tree of dependent software plugins.
 
-    Add a children key, and For each ID in the `plugins` key, add the package it refers to to
+    Add a children key, and For each ID in the `plugins` key, add the package to which it refers to
     children. Recurse until no more plugins are left.
     """
 
@@ -73,41 +348,6 @@ def _build_tree(packages, package):
     return package
 
 
-def _is_product(pkg, **kwargs):
-    """Is this pkg the product described in kwargs.
-
-    Works in one of 2 ways. Match against either 1. display name, or 2. the raw keys (product,
-    major_version, minor_version, release_version, build_version, platform). The root node has no `product`
-    key because it is a collection of host packages.
-    """
-    if not pkg.get("product"):
-        return False
-
-    name = kwargs.get("name")
-    if name:
-        if name == to_name(pkg):
-            return True
-        return False
-
-
-    for key in kwargs:
-        value = kwargs[key]
-        pkg_value = pkg[key]
-        if value and value != pkg_value:
-            return False
-    return True
-
-def _find_by_keys(tree, **kw):
-    """Given product and version keys find the package."""
-    if not tree:
-        return None
-    if _is_product(tree, **kw):
-        return tree
-    for child_tree in tree["children"]:
-        result = _find_by_keys(child_tree, **kw)
-        if result:
-            return result
-    return None
 
 
 def _find_by_name(branch, name, limit=None, depth=0):
@@ -118,7 +358,8 @@ def _find_by_name(branch, name, limit=None, depth=0):
     """
     if not branch:
         return None
-    if _is_product(branch, name=name):
+
+    if  branch.get("product") and name == to_name(branch):
         return branch
     depth += 1
 
@@ -140,7 +381,8 @@ def _find_by_path(tree, path):
 
     if not path:
         return None
-        
+
+
     result = None
     for name in [p for p in path.split("/") if p]:
         tree = _find_by_name(tree, name, 1)
@@ -153,10 +395,12 @@ def _to_path_list(tree, **kw):
 
     This means starting at the level of the given tree, get all the paths to intermediate and leaf
     nodes. This is useful for populating a chooser to choose packages fully qualified by path.
+    Houdini's tree widget for example, takes the below format unchanged and generates the
+    appropriate UI. 
 
-    * 'houdini 16.0.736'
-    * 'houdini 16.0.736/arnold-houdini 2.0.1'
-    * 'houdini 16.0.736/arnold-houdini 2.0.1/al-shaders 1.0'
+    * 'houdini 16.0.736 linux'
+    * 'houdini 16.0.736 linux/arnold-houdini 2.0.1 linux'
+    * 'houdini 16.0.736 linux/arnold-houdini 2.0.1 linux/al-shaders 1.0 linux'
     """
     parent_name = kw.get("parent_name", "")
     paths = kw.get("paths", [])
@@ -168,21 +412,6 @@ def _to_path_list(tree, **kw):
     return paths
 
 
-def to_all_paths(path):
-    """Extract all ancestor paths from a path.
-
-    This can be useful if the user selects a plugin from a chooser, because we know we'll want its
-    host ancestors. Just split the string and work up the parts.
-    """
-    result = []
-    parts = path.split("/")
-    while parts:
-        result.append(("/").join(parts))
-        parts.pop()
-    result.reverse()
-    return result
-
-
 def _clean_package(package):
     """Remove some unwanted keys.
 
@@ -203,171 +432,3 @@ def _clean_package(package):
     pkg["children"] = []
     return pkg
 
-def _get_platform_filter(**kwargs):
-    """
-    Get the set of desired platforms from kwargs
-    If none given, assume all platforms
-    Raise on unrecognized input
-    """
-    platforms = set(kwargs.get("platforms", PLATFORMS))
-    unknown_platforms = platforms - PLATFORMS
-    if unknown_platforms:
-        raise KeyError("Unrecognized platform {}".format(" ".join(unknown_platforms)))
-    return platforms
-
-
-class PackageTree(object):
-    """Class to represent available packages as a tree.
-
-    Data structure is really a DAG because a tool may be compatible with more than one host product.
-    """
-
-    def __init__(self, packages, **kwargs):
-        """Initialize based on a product.
-
-        If product kwarg given then build the tree containing packages below versions of that product
-        only. e.g. "houdini" or "maya-io"
-
-        If platforms kwarg given, filter out non-matching platforms.
-        """
-        product = kwargs.get("product")
-
-        platforms = _get_platform_filter(**kwargs)
-
-        packages = [_clean_package(p) for p in packages if p["platform"] in platforms]
-
-        if product:
-            root_ids = [p["package_id"] for p in packages if p["product"] == product]
-        else:
-            root_ids = [p["package_id"] for p in packages if not p["plugin_host_product"]]
-
-        self.tree = _build_tree(packages, {"children": [], "plugins": root_ids})
-
-    def find_by_name(self, name, limit=None, depth=0):
-        """Search the tree for a product with the given name.
-
-        This name is the name originally constructed from the package using to_name.
-
-        It must be an exact match with product version platform, example: "maya 2018.0 windows".
-        """
-        return _find_by_name(self.tree, name, limit, depth)
-
-    def find_by_keys(self, **kw):
-        """Search the tree for a product with the given keys.
-
-        Whichever keys from the following are given, must match, product, major_version,
-        minor_version, release_version, build_version, platform
-        """
-        return _find_by_keys(self.tree, **kw)
-
-    def find_by_path(self, path):
-        """Find the package uniquely described by this path."""
-        return _find_by_path(self.tree, path)
-
-    def to_path_list(self, **kw):
-        """Get paths to all nodes.
-
-        Example:
-
-            'maya-io 2018.SP2 linux',
-            'maya-io 2018.SP2 linux/yeti 3.8 linux',
-            'maya-io 2018.SP2 linux/yeti 3.1.15 linux',
-            'maya-io 2018.SP4 linux',
-            'maya-io 2018.SP4 linux/yeti 2.2.2 linux',
-            'maya-io 2018.SP4 linux/arnold-maya 2.0.2.3 linux',
-            'maya-io 2018.SP4 linux/v-ray-maya 3.60.01.0 linux',
-
-        If the name keyword is given, get paths below the tree represented by that name.
-
-        Example: to_path_list(name='maya-io 2018.SP4') 'yeti 2.2.2', 'arnold-maya 2.0.2.3',
-        'v-ray-maya 3.60.01.0',
-
-        A TypeError will be thrown if the name is invalid.
-        """
-        name = kw.get("name")
-        if name:
-            subtree = self.find_by_name(name)
-            return _to_path_list(subtree)
-        return _to_path_list(self.tree)
-
-    def get_all_paths_to(self, **kw):
-        """All paths to the package described in kw.
-
-        Its possible there is more than one path to a given node. For now we just get all paths
-        through the tree and then select the ones whose leaf matches.
-        """
-        all_paths = _to_path_list(self.tree)
-        name = to_name(kw)
-        return [p for p in all_paths if p.endswith(name)]
-
-    def platforms(self):
-        """
-        Get the set of platforms represented by packages in the tree.
-        
-        Returns a set
-        """
-
-        # No need to recurse. Plugins are assumed to be compatible with the host.
-        return set([host["platform"] for host in self.tree["children"]])
-
-
-    def get_environment(self, paths):
-        """Get merged environment from paths."""
-        package_env = PackageEnvironment()
-        for path in paths:
-            package = _find_by_path(self.tree, path)
-            if package:
-                package_env.extend(package)
-        return package_env
-
-    def json(self):
-        """The whole tree as json."""
-        return json.dumps(self.tree)
-
-    # TODO write tests for this
-    def supported_host_names(self):
-        """Pluck host paths from the available software tree.
-
-        Dont get children
-
-        Get for all platforms, or platforms list if given.
-        """
-        pathlist = _to_path_list(self.tree)
-        return sorted([p for p in pathlist if "/" not in p])
-
-    def supported_plugins(self, host):
-        """Make sorted list of plugins.
-
-        Each entry is an object with a 'plugin' and a 'versions' key.
-
-        Example: plugins = [
-            {"plugin": "arnold", "versions": ["1","2","3"]}, 
-            {"plugin": "vray", "versions": ["1","2","3"]}
-        ]
-
-
-        We don't provide platform (3rd element) for plugins since the host software was used to
-        request the plugins and consumers can derive the platform from that.
-        """
-
-        try:
-            subtree = self.find_by_name(host)
-            plugin_versions = _to_path_list(subtree)
-        except TypeError:
-            return []
-
-        if not plugin_versions:
-            return []
-
-        plugin_dict = {}
-        for plugin, version, _ in [pv.split(" ") for pv in plugin_versions]:
-            if plugin not in plugin_dict:
-                plugin_dict[plugin] = []
-            plugin_dict[plugin].append(version)
-
-        # convert to list so it can be sorted
-        plugins = []
-        for key in plugin_dict:
-            plugins.append({"plugin": key, "versions": sorted(plugin_dict[key])})
-
-        return sorted(plugins, key=lambda k: k["plugin"])