openshift-python-wrapper 10.0.74__tar.gz → 10.0.76__tar.gz

{openshift_python_wrapper-10.0.74 → openshift_python_wrapper-10.0.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openshift-python-wrapper
-Version: 10.0.74
+Version: 10.0.76
 Summary: Wrapper around https://github.com/kubernetes-client/python
 Home-page: https://github.com/RedHatQE/openshift-python-wrapper
 License: Apache-2.0
@@ -22,7 +22,7 @@ Requires-Dist: colorlog (>=6.7.0,<7.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: kubernetes (>=30.0.0,<31.0.0)
 Requires-Dist: packaging (>=23.1,<24.0)
-Requires-Dist: pyhelper-utils (>=0.0.36,<0.0.37)
+Requires-Dist: pyhelper-utils (>=0.0.37,<0.0.38)
 Requires-Dist: pytest (>=8.0.0,<9.0.0)
 Requires-Dist: python-benedict (>=0.33.0,<0.34.0)
 Requires-Dist: python-simple-logger (>=1.0.6)

{openshift_python_wrapper-10.0.74 → openshift_python_wrapper-10.0.76}/class_generator/README.md

@@ -1,5 +1,14 @@
 # Resource class generator
 
+Utility to add a python module with class resources to openshift-python-wrapper based on `kind`
+
+Some resources have the same kind, in this case the script will create two files.
+For example `DNS` kind have two CRDs, one from `config.openshift.io` and one from `operator.openshift.io`
+The output will be:
+
+- dns_operator_openshift_io.py
+- dns_config_openshift_io.py
+
 ## Usage
 
 ### Installation

{openshift_python_wrapper-10.0.74 → openshift_python_wrapper-10.0.76}/class_generator/class_generator.py

@@ -5,13 +5,14 @@ import json
 import shlex
 import os
 import sys
+import requests
 from pathlib import Path
+from packaging.version import Version
 
 import textwrap
-from typing import Any, Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, List, Tuple
 import click
 import re
-import requests
 from concurrent.futures import Future, ThreadPoolExecutor, as_completed
 import cloup
 from cloup.constraints import If, IsSet, accept_none, require_one
@@ -31,106 +32,154 @@ FIELDS_STR: str = "FIELDS"
 LOGGER = get_logger(name="class_generator")
 TESTS_MANIFESTS_DIR: str = "class_generator/tests/manifests"
 SCHEMA_DIR: str = "class_generator/schema"
+SCHEMA_DEFINITION_FILE: str = os.path.join(SCHEMA_DIR, "_definitions.json")
 RESOURCES_MAPPING_FILE: str = os.path.join(SCHEMA_DIR, "__resources-mappings.json")
 
 
-def _is_kind_is_namespaced(_kind: str) -> Tuple[bool, str]:
-    return (
-        run_command(
-            command=shlex.split(f"bash -c 'oc api-resources --namespaced | grep -w {_kind} | wc -l'"),
-            check=False,
-        )[1].strip()
-        == "1"
-    ), _kind
-
+def _is_kind_and_namespaced(client: str, _key: str, _data: Dict[str, Any]) -> Dict[str, Any]:
+    x_kubernetes_group_version_kind = _data["x-kubernetes-group-version-kind"][0]
+    _kind = x_kubernetes_group_version_kind["kind"]
+    _group = x_kubernetes_group_version_kind.get("group")
+    _version = x_kubernetes_group_version_kind.get("version")
+    _group_and_version = f"{_group}/{_version}" if _group else _version
+
+    not_resource_dict = {"is_kind": False, "kind": _key}
+
+    # if explain command failed, this is not a resource
+    if not run_command(command=shlex.split(f"{client} explain {_kind}"), check=False, log_errors=False)[0]:
+        return not_resource_dict
+
+    # check if this as a valid version for the resource.
+    api_resources_base_cmd = f"bash -c '{client} api-resources"
+
+    if run_command(
+        command=shlex.split(f"{api_resources_base_cmd} | grep -w {_kind} | grep {_group_and_version}'"),
+        check=False,
+        log_errors=False,
+    )[0]:
+        # Check if the resource if namespaced.
+        _data["namespaced"] = (
+            run_command(
+                command=shlex.split(
+                    f"{api_resources_base_cmd} --namespaced | grep -w {_kind} | grep {_group_and_version} | wc -l'"
+                ),
+                check=False,
+                log_errors=False,
+            )[1].strip()
+            == "1"
+        )
+        return {"is_kind": True, "kind": _key, "data": _data}
 
-def _is_resource(_kind: str) -> Tuple[bool, str]:
-    return run_command(command=shlex.split(f"oc explain {_kind}"), check=False, log_errors=False)[0], _kind
+    return not_resource_dict
 
 
-def map_kind_to_namespaced():
+def map_kind_to_namespaced(client: str):
     not_kind_file: str = os.path.join(SCHEMA_DIR, "__not-kind.txt")
 
+    resources_mapping = read_resources_mapping_file()
+
     if os.path.isfile(not_kind_file):
         with open(not_kind_file) as fd:
             not_kind_list = fd.read().split("\n")
     else:
-        not_kind_list: List[str] = []
-
-    with open(os.path.join(f"{SCHEMA_DIR}/all.json")) as fd:
-        all_json_data = json.load(fd)
-
-    if os.path.isfile(RESOURCES_MAPPING_FILE):
-        resources_mapping = read_resources_mapping_file()
-
-    else:
-        resources_mapping: Dict[str, Dict[str, bool]] = {}
-
-    # `all.json` list all files that `openapi2jsonschema` generated which include duplication
-    kind_set: Set[str] = set()
-    for _def in all_json_data["oneOf"]:
-        _kind = _def["$ref"].rsplit(".", 1)[-1]
-        if _kind in not_kind_list:
-            continue
+        not_kind_list = []
 
-        kind_set.add(_kind)
+    with open(SCHEMA_DEFINITION_FILE) as fd:
+        _definitions_json_data = json.load(fd)
 
-    kind_list: List[str] = []
-    is_kind_futures: List[Future] = []
+    _kind_data_futures: List[Future] = []
     with ThreadPoolExecutor() as executor:
-        for _kind in kind_set:
-            if resources_mapping.get(_kind.lower()):  # resource_mappings store all keys in lowercase
+        for _key, _data in _definitions_json_data["definitions"].items():
+            _group_version_kind = _data.get("x-kubernetes-group-version-kind")
+            if not _group_version_kind:
                 continue
 
-            # Check if the kind we work on is a real kind
-            is_kind_futures.append(executor.submit(_is_resource, _kind=_kind))
+            if _key in not_kind_list:
+                continue
 
-    for _res in as_completed(is_kind_futures):
-        res = _res.result()
+            _kind_data_futures.append(executor.submit(_is_kind_and_namespaced, client=client, _key=_key, _data=_data))
 
-        if res[0]:
-            kind_list.append(res[1])
-        else:
-            not_kind_list.append(res[1])
+    _temp_resources_mappings: Dict[Any, Any] = {}
+    for res in as_completed(_kind_data_futures):
+        _res = res.result()
+        # _res["kind"] is group.version.kind, set only kind as key in the final dict
+        kind_key = _res["kind"].rsplit(".", 1)[-1].lower()
 
-    mapping_kind_futures: List[Future] = []
-    with ThreadPoolExecutor() as executor:
-        for _kind in kind_list:
-            mapping_kind_futures.append(executor.submit(_is_kind_is_namespaced, _kind=_kind))
+        if _res["is_kind"]:
+            _temp_resources_mappings.setdefault(kind_key, []).append(_res["data"])
+        else:
+            not_kind_list.append(_res["kind"])
 
-    for _res in as_completed(mapping_kind_futures):
-        res = _res.result()
-        resources_mapping[res[1].lower()] = {"namespaced": res[0]}
+    # Update the resources mapping dict with the one that we filled to avoid duplication in the lists
+    resources_mapping.update(_temp_resources_mappings)
 
     with open(RESOURCES_MAPPING_FILE, "w") as fd:
-        json.dump(resources_mapping, fd)
+        json.dump(resources_mapping, fd, indent=4)
 
     with open(not_kind_file, "w") as fd:
         fd.writelines("\n".join(not_kind_list))
 
 
 def read_resources_mapping_file() -> Dict[Any, Any]:
-    with open(RESOURCES_MAPPING_FILE) as fd:
-        return json.load(fd)
+    try:
+        with open(RESOURCES_MAPPING_FILE) as fd:
+            return json.load(fd)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def get_server_version(client: str):
+    rc, out, _ = run_command(command=shlex.split(f"{client} version -o json"), check=False)
+    if not rc:
+        LOGGER.error("Failed to get server version")
+        sys.exit(1)
+
+    json_out = json.loads(out)
+    return json_out["serverVersion"]["gitVersion"]
+
+
+def get_client_binary() -> str:
+    if os.system("which oc") == 0:
+        return "oc"
+
+    elif os.system("which kubectl") == 0:
+        return "kubectl"
+    else:
+        LOGGER.error("Failed to find oc or kubectl")
+        sys.exit(1)
+
+
+def check_minimum_cluster_version(client) -> None:
+    cluster_version = get_server_version(client=client)
+    minimum_server_version = "v1.30.0"
+    if Version(cluster_version) < Version(minimum_server_version):
+        LOGGER.error(
+            f"Server version {cluster_version} is not supported. Minimum supported version is {minimum_server_version}"
+        )
+        sys.exit(1)
 
 
 def update_kind_schema():
     openapi2jsonschema_str: str = "openapi2jsonschema"
+    client = get_client_binary()
+    check_minimum_cluster_version(client=client)
 
     if not run_command(command=shlex.split("which openapi2jsonschema"), check=False, log_errors=False)[0]:
         LOGGER.error(
-            f"{openapi2jsonschema_str}not found. Install it using `pipx install --python python3.9 openapi2jsonschema`"
+            f"{openapi2jsonschema_str} not found. Install it using `pipx install --python python3.9 openapi2jsonschema`"
         )
         sys.exit(1)
 
-    rc, token, _ = run_command(command=shlex.split("oc whoami -t"), check=False, log_errors=False)
+    rc, token, _ = run_command(command=shlex.split(f"{client} whoami -t"), check=False, log_errors=False)
     if not rc:
         LOGGER.error(
-            "Failed to get token.\nMake sure you are logged in to the cluster using user and password using `oc login`"
+            f"Failed to get token.\nMake sure you are logged in to the cluster using user and password using `{client} login`"
         )
         sys.exit(1)
 
-    api_url = run_command(command=shlex.split("oc whoami --show-server"), check=False, log_errors=False)[1].strip()
+    api_url = run_command(command=shlex.split(f"{client} whoami --show-server"), check=False, log_errors=False)[
+        1
+    ].strip()
     data = requests.get(f"{api_url}/openapi/v2", headers={"Authorization": f"Bearer {token.strip()}"}, verify=False)
 
     if not data.ok:
@@ -145,7 +194,7 @@ def update_kind_schema():
         LOGGER.error("Failed to generate schema.")
         sys.exit(1)
 
-    map_kind_to_namespaced()
+    map_kind_to_namespaced(client=client)
 
 
 def convert_camel_case_to_snake_case(string_: str) -> str:
@@ -266,7 +315,13 @@ def generate_resource_file_from_dict(
     dry_run: bool = False,
     output_file: str = "",
     add_tests: bool = False,
+    output_file_suffix: str = "",
+    output_dir: str = "",
 ) -> Tuple[str, str]:
+    base_dir = output_dir or "ocp_resources"
+    if not os.path.exists(base_dir):
+        os.makedirs(base_dir)
+
     rendered = render_jinja_template(
         template_dict=resource_dict,
         template_dir="class_generator/manifests",
@@ -274,15 +329,21 @@ def generate_resource_file_from_dict(
     )
 
     formatted_kind_str = convert_camel_case_to_snake_case(string_=resource_dict["kind"])
+    _file_suffix = f"{'_' + output_file_suffix if output_file_suffix else ''}"
+
     if add_tests:
         overwrite = True
-        _output_file = os.path.join(TESTS_MANIFESTS_DIR, resource_dict["kind"], f"{formatted_kind_str}_res.py")
+        tests_path = os.path.join(TESTS_MANIFESTS_DIR, resource_dict["kind"])
+        if not os.path.exists(tests_path):
+            os.makedirs(tests_path)
+
+        _output_file = os.path.join(tests_path, f"{formatted_kind_str}{_file_suffix}.py")
 
     elif output_file:
         _output_file = output_file
 
     else:
-        _output_file = os.path.join("ocp_resources", f"{formatted_kind_str}.py")
+        _output_file = os.path.join(base_dir, f"{formatted_kind_str}{_file_suffix}.py")
 
     orig_filename = _output_file
     if os.path.exists(_output_file):
@@ -346,12 +407,16 @@ def format_description(description: str) -> str:
 
 def prepare_property_dict(
     schema: Dict[str, Any],
-    requeired: List[str],
+    required: List[str],
     resource_dict: Dict[str, Any],
     dict_key: str,
 ) -> Dict[str, Any]:
+    keys_to_ignore = ["kind", "apiVersion", "status", SPEC_STR.lower()]
+    if dict_key != SPEC_STR.lower():
+        keys_to_ignore.append("metadata")
+
     for key, val in schema.items():
-        if key in {"metadata", "kind", "apiVersion", "status", SPEC_STR.lower()}:
+        if key in keys_to_ignore:
             continue
 
         val_schema = get_property_schema(property=val)
@@ -360,7 +425,7 @@ def prepare_property_dict(
         resource_dict[dict_key].append({
             "name-for-class-arg": python_name,
             "property-name": key,
-            "required": key in requeired,
+            "required": key in required,
             "description": format_description(description=val_schema["description"]),
             "type-for-docstring": type_dict["type-for-doc"],
             "type-for-class-arg": f"{python_name}: {type_dict['type-for-init']}",
@@ -370,80 +435,77 @@ def prepare_property_dict(
 
 
 def parse_explain(
-    kind_schema_file: str,
-    namespaced: Optional[bool] = None,
-) -> Dict[str, Any]:
-    with open(kind_schema_file) as fd:
-        kind_schema = json.load(fd)
-
-    resource_dict: Dict[str, Any] = {
-        "base_class": "NamespacedResource" if namespaced else "Resource",
-        "description": kind_schema["description"],
-        "fields": [],
-        "spec": [],
-    }
-
-    schema_properties: Dict[str, Any] = kind_schema["properties"]
-    fields_requeired = kind_schema.get("required", [])
-    resource_dict.update(kind_schema["x-kubernetes-group-version-kind"][0])
+    kind: str,
+) -> List[Dict[str, Any]]:
+    _schema_definition = read_resources_mapping_file()
+    _resources: List[Dict[str, Any]] = []
+
+    _kinds_schema = _schema_definition[kind.lower()]
+    for _kind_schema in _kinds_schema:
+        namespaced = _kind_schema["namespaced"]
+        resource_dict: Dict[str, Any] = {
+            "base_class": "NamespacedResource" if namespaced else "Resource",
+            "description": _kind_schema["description"],
+            "fields": [],
+            "spec": [],
+        }
+
+        schema_properties: Dict[str, Any] = _kind_schema["properties"]
+        fields_required = _kind_schema.get("required", [])
+        resource_dict.update(_kind_schema["x-kubernetes-group-version-kind"][0])
+
+        if spec_schema := schema_properties.get("spec", {}):
+            spec_schema = get_property_schema(property=spec_schema)
+            spec_required = spec_schema.get("required", [])
+            resource_dict = prepare_property_dict(
+                schema=spec_schema.get("properties", {}),
+                required=spec_required,
+                resource_dict=resource_dict,
+                dict_key="spec",
+            )
 
-    if spec_schema := schema_properties.get("spec", {}):
-        spec_schema = get_property_schema(property=spec_schema)
-        spec_requeired = spec_schema.get("required", [])
         resource_dict = prepare_property_dict(
-            schema=spec_schema.get("properties", {}),
-            requeired=spec_requeired,
+            schema=schema_properties,
+            required=fields_required,
             resource_dict=resource_dict,
-            dict_key="spec",
+            dict_key="fields",
         )
 
-    resource_dict = prepare_property_dict(
-        schema=schema_properties,
-        requeired=fields_requeired,
-        resource_dict=resource_dict,
-        dict_key="fields",
-    )
-
-    api_group_real_name = resource_dict.get("group")
-    # If API Group is not present in resource, try to get it from VERSION
-    if not api_group_real_name:
-        version_splited = resource_dict["version"].split("/")
-        if len(version_splited) == 2:
-            api_group_real_name = version_splited[0]
-
-    if api_group_real_name:
-        api_group_for_resource_api_group = api_group_real_name.upper().replace(".", "_")
-        resource_dict["group"] = api_group_for_resource_api_group
-        missing_api_group_in_resource: bool = not hasattr(Resource.ApiGroup, api_group_for_resource_api_group)
-
-        if missing_api_group_in_resource:
-            LOGGER.warning(
-                f"Missing API Group in Resource\n"
-                f"Please add `Resource.ApiGroup.{api_group_for_resource_api_group} = {api_group_real_name}` "
-                "manually into ocp_resources/resource.py under Resource class > ApiGroup class."
-            )
+        api_group_real_name = resource_dict.get("group")
+        # If API Group is not present in resource, try to get it from VERSION
+        if not api_group_real_name:
+            version_splited = resource_dict["version"].split("/")
+            if len(version_splited) == 2:
+                api_group_real_name = version_splited[0]
+
+        if api_group_real_name:
+            api_group_for_resource_api_group = api_group_real_name.upper().replace(".", "_")
+            resource_dict["group"] = api_group_for_resource_api_group
+            missing_api_group_in_resource: bool = not hasattr(Resource.ApiGroup, api_group_for_resource_api_group)
+
+            if missing_api_group_in_resource:
+                LOGGER.warning(
+                    f"Missing API Group in Resource\n"
+                    f"Please add `Resource.ApiGroup.{api_group_for_resource_api_group} = {api_group_real_name}` "
+                    "manually into ocp_resources/resource.py under Resource class > ApiGroup class."
+                )
 
-    else:
-        api_version_for_resource_api_version = resource_dict["version"].upper()
-        missing_api_version_in_resource: bool = not hasattr(Resource.ApiVersion, api_version_for_resource_api_version)
-
-        if missing_api_version_in_resource:
-            LOGGER.warning(
-                f"Missing API Version in Resource\n"
-                f"Please add `Resource.ApiVersion.{api_version_for_resource_api_version} = {resource_dict['version']}` "
-                "manually into ocp_resources/resource.py under Resource class > ApiGroup class."
+        else:
+            api_version_for_resource_api_version = resource_dict["version"].upper()
+            missing_api_version_in_resource: bool = not hasattr(
+                Resource.ApiVersion, api_version_for_resource_api_version
             )
 
-    return resource_dict
-
+            if missing_api_version_in_resource:
+                LOGGER.warning(
+                    f"Missing API Version in Resource\n"
+                    f"Please add `Resource.ApiVersion.{api_version_for_resource_api_version} = {resource_dict['version']}` "
+                    "manually into ocp_resources/resource.py under Resource class > ApiGroup class."
+                )
 
-def get_kind_schema_file(kind: str) -> str:
-    kind_file = os.path.join(SCHEMA_DIR, f"{kind.lower()}.json")
-    if os.path.isfile(kind_file):
-        return kind_file
+        _resources.append(resource_dict)
 
-    LOGGER.error(f"{kind} schema not found, please run with `--update-schema`")
-    sys.exit(1)
+    return _resources
 
 
 def class_generator(
@@ -451,11 +513,13 @@ def class_generator(
     overwrite: bool = False,
     dry_run: bool = False,
     output_file: str = "",
+    output_dir: str = "",
     add_tests: bool = False,
-) -> str:
+) -> List[str]:
     """
     Generates a class for a given Kind.
     """
+    LOGGER.info(f"Generating class for {kind}")
     kind = kind.lower()
     kind_and_namespaced_mappings = read_resources_mapping_file().get(kind)
 
@@ -463,33 +527,37 @@ def class_generator(
         LOGGER.error(f"{kind} not found in {RESOURCES_MAPPING_FILE}, Please run with --update-schema")
         sys.exit(1)
 
-    resource_dict = parse_explain(
-        kind_schema_file=get_kind_schema_file(kind=kind),
-        namespaced=kind_and_namespaced_mappings["namespaced"],
-    )
-    if not resource_dict:
-        return ""
-
-    orig_filename, generated_py_file = generate_resource_file_from_dict(
-        resource_dict=resource_dict,
-        overwrite=overwrite,
-        dry_run=dry_run,
-        output_file=output_file,
-        add_tests=add_tests,
-    )
+    resources = parse_explain(kind=kind)
 
-    if not dry_run:
-        run_command(
-            command=shlex.split(f"pre-commit run --files {generated_py_file}"),
-            verify_stderr=False,
-            check=False,
+    use_output_file_suffix: bool = len(resources) > 1
+    generated_files: List[str] = []
+    for resource_dict in resources:
+        output_file_suffix = resource_dict["group"].lower() if use_output_file_suffix else ""
+
+        orig_filename, generated_py_file = generate_resource_file_from_dict(
+            resource_dict=resource_dict,
+            overwrite=overwrite,
+            dry_run=dry_run,
+            output_file=output_file,
+            add_tests=add_tests,
+            output_file_suffix=output_file_suffix,
+            output_dir=output_dir,
         )
 
-        if orig_filename != generated_py_file and filecmp.cmp(orig_filename, generated_py_file):
-            LOGGER.warning(f"File {orig_filename} was not updated, deleting {generated_py_file}")
-            Path.unlink(Path(generated_py_file))
+        if not dry_run:
+            run_command(
+                command=shlex.split(f"pre-commit run --files {generated_py_file}"),
+                verify_stderr=False,
+                check=False,
+            )
+
+            if orig_filename != generated_py_file and filecmp.cmp(orig_filename, generated_py_file):
+                LOGGER.warning(f"File {orig_filename} was not updated, deleting {generated_py_file}")
+                Path.unlink(Path(generated_py_file))
+
+        generated_files.append(generated_py_file)
 
-    return generated_py_file
+    return generated_files
 
 
 def write_and_format_rendered(filepath: str, data: str) -> None:

{openshift_python_wrapper-10.0.74 → openshift_python_wrapper-10.0.76}/class_generator/manifests/class_generator_template.j2

@@ -38,7 +38,6 @@ class {{ kind }}({{ base_class }}):
             {% endfor %}
         """
         {% endif %}
-
         super().__init__(**kwargs)
 
         {% for arg in fields + spec %}
@@ -61,29 +60,34 @@ class {{ kind }}({{ base_class }}):
             _spec = self.res["spec"]
           {% endif %}
 
+          {# Add required args under `self.res` #}
           {% for arg in fields %}
           {% if arg["required"] == True %}
             self.res["{{ arg["property-name"] }}"] = self.{{ arg["name-for-class-arg"] }}
           {% endif %}
           {% endfor %}
 
+          {# Add required args under `_spec` #}
           {% for arg in spec %}
           {% if arg["required"] == True %}
             _spec["{{ arg["property-name"] }}"] = self.{{ arg["name-for-class-arg"] }}
           {% endif %}
           {% endfor %}
 
+          {# Add optional args under `self.res` #}
           {% for arg in fields %}
           {% if arg["required"] == False %}
-            if self.{{ arg["name-for-class-arg"] }}{% if arg["type"] == "bool" %} is not None{% endif %}:
-              self.res["{{ arg["property-name"] }}"] = self.{{ arg["name-for-class-arg"] }}
+            if self.{{ arg["name-for-class-arg"] }}{% if arg["type-for-docstring"] == "bool" %} is not None{% endif %}:
+                self.res["{{ arg["property-name"] }}"] = self.{{ arg["name-for-class-arg"] }}
 
           {% endif %}
           {% endfor %}
+
+          {# Add optional args under `_spec` #}
           {% for arg in spec %}
           {% if arg["required"] == False %}
-            if self.{{ arg["name-for-class-arg"] }}{% if arg["type"] == "bool" %} is not None{% endif %}:
-              _spec["{{ arg["property-name"] }}"] = self.{{ arg["name-for-class-arg"] }}
+            if self.{{ arg["name-for-class-arg"] }}{% if arg["type-for-docstring"] == "bool" %} is not None{% endif %}:
+                _spec["{{ arg["property-name"] }}"] = self.{{ arg["name-for-class-arg"] }}
 
           {% endif %}
           {% endfor %}