tutor 23.0.0.dev3201__tar.gz → 23.0.0.dev3205__tar.gz

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tutor
-Version: 23.0.0.dev3201
+Version: 23.0.0.dev3205
 Summary: The Docker-based Open edX distribution designed for peace of mind
 Project-URL: Homepage, https://docs.tutor.edly.io/
 Project-URL: Documentation, https://docs.tutor.edly.io/
@@ -33,6 +33,7 @@ Requires-Dist: mypy
 Requires-Dist: packaging
 Requires-Dist: pycryptodome>=3.17.0
 Requires-Dist: pyyaml>=6.0
+Requires-Dist: ruamel-yaml>=0.18.11
 Requires-Dist: typing-extensions>=4.4.0
 Provides-Extra: dev
 Requires-Dist: altgraph==0.17.5; extra == 'dev'
@@ -92,6 +93,8 @@ Requires-Dist: requests==2.32.5; extra == 'dev'
 Requires-Dist: rfc3986==2.0.0; extra == 'dev'
 Requires-Dist: rich==15.0.0; extra == 'dev'
 Requires-Dist: rsa==4.9.1; extra == 'dev'
+Requires-Dist: ruamel-yaml-clib==0.2.15; extra == 'dev'
+Requires-Dist: ruamel-yaml==0.19.1; extra == 'dev'
 Requires-Dist: ruff==0.15.14; extra == 'dev'
 Requires-Dist: scriv==1.7.0; extra == 'dev'
 Requires-Dist: six==1.17.0; extra == 'dev'

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/requirements/base.in

@@ -6,6 +6,7 @@ mypy
 packaging
 pycryptodome>=3.17.0
 pyyaml>=6.0
+ruamel.yaml>=0.18.11
 typing-extensions>=4.4.0
 importlib-metadata>=7.0.1
 importlib-resources>=6.1.1

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/requirements/dev.txt

@@ -190,6 +190,12 @@ rsa==4.9.1
     # via
     #   -r requirements/base.txt
     #   google-auth
+ruamel-yaml==0.19.1
+    # via -r requirements/base.txt
+ruamel-yaml-clib==0.2.15
+    # via
+    #   -r requirements/base.txt
+    #   ruamel-yaml
 ruff==0.15.14
     # via -r requirements/dev.in
 scriv==1.7.0

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/commands/compose.py

@@ -222,13 +222,14 @@ def interactive_configuration(
     run_for_prod: t.Optional[bool] = None,
 ) -> None:
     config = tutor_config.load_minimal(context.obj.root)
+    answered: list[str] = []
     if interactive:
         click.echo(fmt.title("Interactive platform configuration"))
-        interactive_config.ask_questions(
+        answered = interactive_config.ask_questions(
             config,
             run_for_prod=run_for_prod,
         )
-    tutor_config.save_config_file(context.obj.root, config)
+    tutor_config.save_config_file(context.obj.root, config, explicit=answered)
     config = tutor_config.load_full(context.obj.root)
     tutor_env.save(context.obj.root, config)
 

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/commands/config.py

@@ -170,8 +170,9 @@ def save(
             if run_clean:
                 env.delete_env_dir(context.root)
 
+    answered: list[str] = []
     if interactive:
-        interactive_config.ask_questions(config)
+        answered = interactive_config.ask_questions(config)
     elif clean_env:
         env.delete_env_dir(context.root)
     if set_vars:
@@ -207,7 +208,19 @@ def save(
     for key in unset_vars:
         config.pop(key, None)
     if not env_only:
-        tutor_config.save_config_file(context.root, config)
+        # Entries that the user named on the command line are saved as literal
+        # values, even when config.yml holds a Jinja expression that renders to
+        # the same result. So are the interactive answers that differ from the
+        # current value, but not the accepted defaults: accepting a default is
+        # not a request to replace an expression by its result.
+        explicit = (
+            answered
+            + [key for key, _value in set_vars]
+            + [key for key, _value in append_vars]
+            + [key for key, _value in remove_vars]
+            + list(unset_vars)
+        )
+        tutor_config.save_config_file(context.root, config, explicit=explicit)
 
     # Reload configuration, without version checking
     config = tutor_config.load_full(context.root)

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/commands/k8s.py

@@ -241,10 +241,11 @@ def launch(context: click.Context, non_interactive: bool) -> None:
         )
 
     config = tutor_config.load_minimal(context.obj.root)
+    answered: list[str] = []
     if not non_interactive:
         click.echo(fmt.title("Interactive platform configuration"))
-        interactive_config.ask_questions(config, run_for_prod=True)
-    tutor_config.save_config_file(context.obj.root, config)
+        answered = interactive_config.ask_questions(config, run_for_prod=True)
+    tutor_config.save_config_file(context.obj.root, config, explicit=answered)
     config = tutor_config.load_full(context.obj.root)
     tutor_env.save(context.obj.root, config)
 

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/commands/plugins.py

@@ -143,7 +143,9 @@ def enable(context: click.Context, plugin_names: list[str]) -> None:
         plugins.load(plugin)
         fmt.echo_info(f"Plugin {plugin} enabled")
     tutor_config.save_enabled_plugins(config)
-    tutor_config.save_config_file(context.obj.root, config)
+    tutor_config.save_config_file(
+        context.obj.root, config, explicit=[tutor_config.PLUGINS_CONFIG_KEY]
+    )
     context.invoke(config_save_command, env_only=True)
 
 
@@ -166,7 +168,9 @@ def disable(context: click.Context, plugin_names: list[str]) -> None:
             disabled.append(plugin)
             fmt.echo_info(f"Plugin {plugin} disabled")
     if disabled:
-        tutor_config.save_config_file(context.obj.root, config)
+        tutor_config.save_config_file(
+            context.obj.root, config, explicit=[tutor_config.PLUGINS_CONFIG_KEY]
+        )
         context.invoke(config_save_command, env_only=True)
 
 
@@ -458,7 +462,9 @@ def index_add(context: Context, url: str) -> None:
     """
     config = tutor_config.load_minimal(context.root)
     if indexes.add(url, config):
-        tutor_config.save_config_file(context.root, config)
+        tutor_config.save_config_file(
+            context.root, config, explicit=[indexes.PLUGIN_INDEXES_KEY]
+        )
         update_indexes(config)
     else:
         fmt.echo_alert("Plugin index was already added")
@@ -473,7 +479,9 @@ def index_remove(context: Context, url: str) -> None:
     """
     config = tutor_config.load_minimal(context.root)
     if indexes.remove(url, config):
-        tutor_config.save_config_file(context.root, config)
+        tutor_config.save_config_file(
+            context.root, config, explicit=[indexes.PLUGIN_INDEXES_KEY]
+        )
         update_indexes(config)
     else:
         fmt.echo_alert("Plugin index not present")

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/config.py

@@ -278,14 +278,107 @@ def convert_json2yml(root: str) -> None:
     )
 
 
-def save_config_file(root: str, config: Config) -> None:
+def save_config_file(
+    root: str, config: Config, explicit: t.Optional[t.Iterable[str]] = None
+) -> None:
+    """
+    Write the configuration to config.yml.
+
+    The file is not re-generated from scratch: instead, the document that is
+    already on disk is modified in place, such that comments, key order and
+    hand-written Jinja expressions are all preserved. Only the entries that
+    actually changed are written, and the file is left untouched when there is
+    nothing to change.
+
+    `explicit` lists the entries that the user set explicitly, for instance with
+    `tutor config save --set KEY=VALUE`. Those are always written as literal
+    values, even when the file happens to contain a Jinja expression that
+    renders to the same result.
+    """
     path = config_path(root)
+    document, layout = load_config_document(path)
+    if not update_config_document(document, config, set(explicit or [])):
+        # Nothing changed: leave the file alone, mtime included.
+        return
     utils.ensure_file_directory_exists(path)
     with open(path, "w", encoding="utf-8") as of:
-        serialize.dump(config, of)
+        serialize.dump_round_trip(document, of, layout)
     fmt.echo_info(f"Configuration saved to {path}")
 
 
+def load_config_document(path: str) -> tuple[t.Any, serialize.YamlLayout]:
+    """
+    Load config.yml as an editable document that remembers its own formatting.
+
+    An empty document is returned when the file does not exist yet, or when it
+    does not hold a mapping.
+    """
+    if not os.path.exists(path):
+        return serialize.empty_document(), serialize.YamlLayout()
+    with open(path, encoding="utf-8") as f:
+        document, layout = serialize.load_round_trip(f.read())
+    if not isinstance(document, dict):
+        return serialize.empty_document(), layout
+    return document, layout
+
+
+def update_config_document(document: t.Any, config: Config, explicit: set[str]) -> bool:
+    """
+    Apply `config` to an existing config.yml document, in place.
+
+    Return True if the document was modified, and thus needs to be written back
+    to disk.
+    """
+    changed = False
+    for key, value in config.items():
+        if key not in document:
+            # New entry: append it at the end of the document.
+            document[key] = value
+            changed = True
+            continue
+        if document[key] == value:
+            continue
+        if key not in explicit and is_stable_expression(config, document[key]):
+            # The file holds a Jinja expression that is re-evaluated on every
+            # load, so the value we have in memory is derived data: saving it
+            # would replace the expression by one of its results. Leave it be.
+            continue
+        document[key] = value
+        changed = True
+    for key in [key for key in document if key not in config]:
+        del document[key]
+        changed = True
+    return changed
+
+
+def is_stable_expression(config: Config, raw_value: t.Any) -> bool:
+    """
+    Check whether `raw_value`, as read from config.yml, is a Jinja expression
+    that always renders to the same result.
+
+    This is what allows a hand-written entry such as:
+
+        CMS_HOST: "studio.{{ LMS_HOST }}"
+
+    to survive a save, instead of being flattened to its result. Expressions
+    that render to something different every time, such as
+    "{{ 24|random_string }}", are not stable: those are frozen to a literal,
+    which is how generated secrets have always behaved.
+
+    Rendering happens against the final configuration, so an expression that
+    points at another entry keeps working even when that other entry just
+    changed.
+    """
+    try:
+        rendered = env.render_unknown(config, raw_value)
+        if rendered == raw_value:
+            # Not an expression at all, just a plain value.
+            return False
+        return bool(rendered == env.render_unknown(config, raw_value))
+    except exceptions.TutorError:
+        return False
+
+
 def config_path(root: str) -> str:
     return os.path.join(root, CONFIG_FILENAME)
 

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/hooks/catalog.py

@@ -287,6 +287,10 @@ class Filters:
     DOCKER_BUILD_COMMAND: Filter[list[str], []] = Filter()
 
     #: List of patches that should be inserted in a given location of the templates.
+    #: Each ``{{ patch("name") }}`` statement in the templates is replaced by the
+    #: rendered content of all the patches with the same name. See :ref:`template_patches`
+    #: for a detailed explanation and the :ref:`template patch catalog <patches>` for
+    #: the list of all available patches.
     #:
     #: :parameter list[tuple[str, str]] patches: pairs of (name, content) tuples. Use this
     #:   filter to modify the Tutor templates.
@@ -325,7 +329,7 @@ class Filters:
     #: - ``list_if``: In a list of ``(value, condition)`` tuples, return the list of ``value`` for which the ``condition`` is true.
     #: - ``long_to_base64``: Base-64 encode a long integer.
     #: - ``iter_values_named``: Yield the values of the configuration settings that match a certain pattern. Example: ``{% for value in iter_values_named(prefix="KEY", suffix="SUFFIX")%}...{% endfor %}``. By default, only non-empty values are yielded. To iterate also on empty values, pass the ``allow_empty=True`` argument.
-    #: - ``patch``: See :ref:`patches <v0_plugin_patches>`.
+    #: - ``patch``: See :ref:`template_patches`.
     #: - ``random_string``: Return a random string of the given length composed of ASCII letters and digits. Example: ``{{ 8|random_string }}``.
     #: - ``reverse_host``: Reverse a domain name (see `reference <https://en.wikipedia.org/wiki/Reverse_domain_name_notation>`__). Example: ``{{ "demo.myopenedx.com"|reverse_host }}`` is equal to "com.myopenedx.demo".
     #: - ``rsa_import_key``: Import a PEM-formatted RSA key and return the corresponding object.

{tutor-23.0.0.dev3201 → tutor-23.0.0.dev3205}/tutor/interactive.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import List, Optional
 
 import click
@@ -7,7 +8,7 @@ from . import env, exceptions, fmt, hooks
 from .types import Config, get_typed
 
 
-def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> None:
+def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> List[str]:
     """
     Interactively ask questions to collect configuration values from the user.
 
@@ -16,8 +17,10 @@ def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> None:
         run_for_prod: Whether platform should be configured for production.
             If None, then ask the user.
     Returns:
-        None
+        The names of the entries whose value was changed by the answers. Entries
+        whose default value was accepted are not part of this list.
     """
+    before = deepcopy(config)
     defaults = tutor_config.get_defaults()
     if run_for_prod is None:
         run_for_prod = config.get("LMS_HOST") not in [
@@ -154,6 +157,12 @@ def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> None:
 
     hooks.Actions.CONFIG_INTERACTIVE.do(config)
 
+    return [
+        key
+        for key in set(before) | set(config)
+        if key not in before or key not in config or before[key] != config[key]
+    ]
+
 
 def ask(question: str, key: str, config: Config, defaults: Config) -> None:
     default = get_typed(defaults, key, str)

tutor-23.0.0.dev3205/tutor/serialize.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import re
+import typing as t
+from _io import TextIOWrapper
+
+import yaml
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.resolver import VersionedResolver
+from ruamel.yaml.util import load_yaml_guess_indent
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+
+def load(stream: t.Union[str, t.IO[str]]) -> t.Any:
+    return yaml.load(stream, Loader=yaml.SafeLoader)
+
+
+def load_all(stream: str) -> t.Iterator[t.Any]:
+    return yaml.load_all(stream, Loader=yaml.SafeLoader)
+
+
+def dump_all(documents: t.Sequence[t.Any], fileobj: TextIOWrapper) -> None:
+    yaml.safe_dump_all(
+        documents, stream=fileobj, default_flow_style=False, allow_unicode=True
+    )
+
+
+def dump(content: t.Any, fileobj: TextIOWrapper) -> None:
+    yaml.dump(content, stream=fileobj, default_flow_style=False, allow_unicode=True)
+
+
+def dumps(content: t.Any) -> str:
+    result = yaml.dump(content, default_flow_style=False, allow_unicode=True)
+    assert isinstance(result, str)
+    return result
+
+
+class YamlLayout(t.NamedTuple):
+    """
+    Layout of a yaml document that the round-trip loader does not carry inside
+    the document itself, and that must therefore be handed back to the dumper.
+    """
+
+    sequence_indent: int = 2
+    sequence_dash_offset: int = 0
+    explicit_start: bool = False
+
+
+class Yaml11Resolver(VersionedResolver):
+    """
+    Resolve plain scalars with the YAML 1.1 rules, which are the ones used by
+    PyYAML, instead of the YAML 1.2 rules that ruamel.yaml applies by default.
+
+    Files that are written by the round-trip dumper are read back with PyYAML, so
+    both must agree on the type of every value. Under YAML 1.2, `on`, `yes` and
+    `0777` are a string, a string and the integer 777, while PyYAML loads them as
+    True, True and 511. With this resolver, the round-trip loader returns the same
+    values as PyYAML, and the dumper quotes any string that PyYAML would load as
+    something else.
+
+    Note that ruamel.yaml would also switch to YAML 1.1 if we set `YAML.version`,
+    but then every document would be dumped with a "%YAML 1.1" directive.
+    """
+
+    @property
+    def processing_version(self) -> t.Any:
+        return (1, 1)
+
+
+def round_trip_yaml(layout: YamlLayout) -> YAML:
+    """
+    YAML parser/dumper that preserves comments, key order and quoting style.
+
+    This is considerably slower than the SafeLoader used everywhere else, so it
+    should only be used for the few files that users are expected to edit by
+    hand, such as config.yml.
+    """
+    yaml_rt = YAML(typ="rt")
+    yaml_rt.Resolver = Yaml11Resolver
+    # PyYAML keeps the last of duplicate keys, while ruamel.yaml refuses to load
+    # them. Accept them too: ruamel.yaml keeps the first value, but the save then
+    # sees that it differs from the one we loaded with PyYAML, and writes the
+    # latter back, once.
+    yaml_rt.allow_duplicate_keys = True
+    yaml_rt.preserve_quotes = True
+    yaml_rt.default_flow_style = False
+    yaml_rt.allow_unicode = True
+    yaml_rt.explicit_start = layout.explicit_start
+    yaml_rt.indent(
+        mapping=2,
+        sequence=layout.sequence_indent,
+        offset=layout.sequence_dash_offset,
+    )
+    # Never re-wrap long values, such as RSA private keys, onto several lines.
+    yaml_rt.width = 2**16
+    return yaml_rt
+
+
+def load_round_trip(text: str) -> tuple[t.Any, YamlLayout]:
+    """
+    Load a yaml document, preserving comments, key order and quoting style.
+
+    Return the document, along with the layout that must be passed back to
+    `dump_round_trip` for the document to be written out as the user wrote it.
+    The layout does not affect parsing, so the default one is good enough to load
+    the document with.
+    """
+    document, sequence_indent, dash_offset = load_yaml_guess_indent(
+        text, yaml=round_trip_yaml(YamlLayout())
+    )
+    layout = YamlLayout(
+        sequence_indent=2 if sequence_indent is None else int(sequence_indent),
+        sequence_dash_offset=0 if dash_offset is None else int(dash_offset),
+        explicit_start=has_explicit_start(text),
+    )
+    return document, layout
+
+
+def has_explicit_start(text: str) -> bool:
+    """
+    Check whether a yaml document opens with an explicit "---" marker.
+    """
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        return stripped == "---"
+    return False
+
+
+def dump_round_trip(
+    content: t.Any, fileobj: t.IO[str], layout: YamlLayout = YamlLayout()
+) -> None:
+    """
+    Dump a document loaded with `load_round_trip`, preserving its formatting.
+    """
+    round_trip_yaml(layout).dump(content, fileobj)
+
+
+def empty_document() -> CommentedMap:
+    """
+    An empty document suitable for `dump_round_trip`.
+    """
+    return CommentedMap()
+
+
+def str_format(content: t.Any) -> str:
+    """
+    Convert a value to str.
+
+    This is almost like json, but more convenient for printing to the standard output.
+    """
+    if content is True:
+        return "true"
+    if content is False:
+        return "false"
+    if content is None:
+        return "null"
+    return str(content)
+
+
+def parse(v: t.Union[str, t.IO[str]]) -> t.Any:
+    """
+    Parse a yaml-formatted string.
+    """
+    try:
+        return load(v)
+    except (ParserError, ScannerError):
+        pass
+    return v
+
+
+def parse_key_value(text: str) -> t.Optional[tuple[str, t.Any]]:
+    """
+    Parse <KEY>=<YAML VALUE> command line arguments.
+
+    Return None if text could not be parsed.
+    """
+    match = re.match(r"(?P<key>[a-zA-Z0-9_-]+)=(?P<value>(.|\n|\r)*)", text)
+    if not match:
+        return None
+    key = match.groupdict()["key"]
+    value = match.groupdict()["value"]
+    if not value:
+        # Empty strings are interpreted as null values, which is incorrect.
+        value = "''"
+    elif "\n" not in value and value.startswith("#"):
+        # Single-line string that starts with a pound # key
+        # We need to escape the string, otherwise pound will be interpreted as a comment.
+        value = f'"{value}"'
+    return key, parse(value)

tutor-23.0.0.dev3201/tutor/serialize.py

@@ -1,80 +0,0 @@
-from __future__ import annotations
-
-import re
-import typing as t
-from _io import TextIOWrapper
-
-import yaml
-from yaml.parser import ParserError
-from yaml.scanner import ScannerError
-
-
-def load(stream: t.Union[str, t.IO[str]]) -> t.Any:
-    return yaml.load(stream, Loader=yaml.SafeLoader)
-
-
-def load_all(stream: str) -> t.Iterator[t.Any]:
-    return yaml.load_all(stream, Loader=yaml.SafeLoader)
-
-
-def dump_all(documents: t.Sequence[t.Any], fileobj: TextIOWrapper) -> None:
-    yaml.safe_dump_all(
-        documents, stream=fileobj, default_flow_style=False, allow_unicode=True
-    )
-
-
-def dump(content: t.Any, fileobj: TextIOWrapper) -> None:
-    yaml.dump(content, stream=fileobj, default_flow_style=False, allow_unicode=True)
-
-
-def dumps(content: t.Any) -> str:
-    result = yaml.dump(content, default_flow_style=False, allow_unicode=True)
-    assert isinstance(result, str)
-    return result
-
-
-def str_format(content: t.Any) -> str:
-    """
-    Convert a value to str.
-
-    This is almost like json, but more convenient for printing to the standard output.
-    """
-    if content is True:
-        return "true"
-    if content is False:
-        return "false"
-    if content is None:
-        return "null"
-    return str(content)
-
-
-def parse(v: t.Union[str, t.IO[str]]) -> t.Any:
-    """
-    Parse a yaml-formatted string.
-    """
-    try:
-        return load(v)
-    except (ParserError, ScannerError):
-        pass
-    return v
-
-
-def parse_key_value(text: str) -> t.Optional[tuple[str, t.Any]]:
-    """
-    Parse <KEY>=<YAML VALUE> command line arguments.
-
-    Return None if text could not be parsed.
-    """
-    match = re.match(r"(?P<key>[a-zA-Z0-9_-]+)=(?P<value>(.|\n|\r)*)", text)
-    if not match:
-        return None
-    key = match.groupdict()["key"]
-    value = match.groupdict()["value"]
-    if not value:
-        # Empty strings are interpreted as null values, which is incorrect.
-        value = "''"
-    elif "\n" not in value and value.startswith("#"):
-        # Single-line string that starts with a pound # key
-        # We need to escape the string, otherwise pound will be interpreted as a comment.
-        value = f'"{value}"'
-    return key, parse(value)