python-hcl2 7.2.0__tar.gz → 7.3.0__tar.gz

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/CHANGELOG.md

@@ -7,7 +7,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## \[Unreleased\]
 
-- Nothing Yet
+- Nothing yet.
+
+## \[7.3.0\] - 2025-07-23
+
+### Fixed
+
+- Issue parsing interpolations and escaped interpolations in a single string. ([#239](https://github.com/amplify-education/python-hcl2/pull/239))
+
+## \[7.2.1\] - 2025-05-16
+
+### Fixed
+
+- More robust escaping for special characters. Thanks, @eranor ([#224](https://github.com/amplify-education/python-hcl2/pull/224))
+- Issue parsing interpolation string as an object key ([#232](https://github.com/amplify-education/python-hcl2/pull/232))
 
 ## \[7.2.0\] - 2025-04-24
 

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-hcl2
-Version: 7.2.0
+Version: 7.3.0
 Summary: A parser for HCL2
 Author-email: Amplify Education <github@amplify.com>
 License: MIT

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/hcl2/hcl2.lark

@@ -1,7 +1,7 @@
 start : body
 body : (new_line_or_comment? (attribute | block))* new_line_or_comment?
 attribute : identifier EQ expression
-block : identifier (identifier | STRING_LIT | string_with_interpolation)* new_line_or_comment? "{" body "}"
+block : identifier (identifier | string)* new_line_or_comment? "{" body "}"
 new_line_or_comment: ( NL_OR_COMMENT )+
 NL_OR_COMMENT: /\n[ \t]*/ | /#.*\n/ | /\/\/.*\n/ | /\/\*(.|\n)*?(\*\/)/
 
@@ -44,8 +44,7 @@ COLON : ":"
 expr_term : LPAR new_line_or_comment? expression new_line_or_comment? RPAR
             | float_lit
             | int_lit
-            | STRING_LIT
-            | string_with_interpolation
+            | string
             | tuple
             | object
             | function_call
@@ -60,11 +59,13 @@ expr_term : LPAR new_line_or_comment? expression new_line_or_comment? RPAR
             | for_tuple_expr
             | for_object_expr
 
-STRING_LIT : "\"" STRING_CHARS? "\""
-STRING_CHARS : /(?:(?!\${)([^"\\]|\\.|\$\$))+/ // any character except '"', including escaped $$
-string_with_interpolation: "\"" (STRING_CHARS)* interpolation_maybe_nested (STRING_CHARS | interpolation_maybe_nested)* "\""
-interpolation_maybe_nested: "${" expression "}"
-
+string: "\"" string_part* "\""
+string_part: STRING_CHARS
+           | ESCAPED_INTERPOLATION
+           | interpolation
+interpolation: "${" expression "}"
+ESCAPED_INTERPOLATION.2: /\$\$\{[^}]*\}/
+STRING_CHARS.1: /(?:(?!\$\$\{)(?!\$\{)[^"\\]|\\.|(?:\$(?!\$?\{)))+/
 
 int_lit : NEGATIVE_DECIMAL? DECIMAL+ | NEGATIVE_DECIMAL+
 !float_lit: (NEGATIVE_DECIMAL? DECIMAL+ | NEGATIVE_DECIMAL+) "." DECIMAL+ (EXP_MARK)?
@@ -77,7 +78,7 @@ EQ : /[ \t]*=(?!=|>)/
 tuple : "[" (new_line_or_comment* expression new_line_or_comment* ",")* (new_line_or_comment* expression)? new_line_or_comment* "]"
 object : "{" new_line_or_comment? (new_line_or_comment* (object_elem | (object_elem COMMA)) new_line_or_comment*)* "}"
 object_elem : object_elem_key ( EQ | COLON ) expression
-object_elem_key : float_lit | int_lit | identifier | STRING_LIT | object_elem_key_dot_accessor | object_elem_key_expression
+object_elem_key : float_lit | int_lit | identifier | string | object_elem_key_dot_accessor | object_elem_key_expression
 object_elem_key_expression : LPAR expression RPAR
 object_elem_key_dot_accessor : identifier (DOT identifier)+
 

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/hcl2/reconstructor.py

@@ -1,7 +1,6 @@
 """A reconstructor for HCL2 implemented using Lark's experimental reconstruction functionality"""
 
 import re
-import json
 from typing import List, Dict, Callable, Optional, Union, Any, Tuple
 
 from lark import Lark, Tree
@@ -10,6 +9,7 @@ from lark.lexer import Token, PatternStr, TerminalDef
 from lark.reconstruct import Reconstructor
 from lark.tree_matcher import is_discarded_terminal
 from lark.visitors import Transformer_InPlace
+from regex import regex
 
 from hcl2.const import START_LINE_KEY, END_LINE_KEY
 from hcl2.parser import reconstruction_parser
@@ -137,7 +137,7 @@ class HCLReconstructor(Reconstructor):
         )
 
     # pylint: disable=too-many-branches, too-many-return-statements
-    def _should_add_space(self, rule, current_terminal):
+    def _should_add_space(self, rule, current_terminal, is_block_label: bool = False):
         """
         This method documents the situations in which we add space around
         certain tokens while reconstructing the generated HCL.
@@ -155,6 +155,7 @@ class HCLReconstructor(Reconstructor):
 
         This should be sufficient to make a spacing decision.
         """
+
         # we don't need to add multiple spaces
         if self._last_char_space:
             return False
@@ -166,6 +167,14 @@ class HCLReconstructor(Reconstructor):
         if self._is_equals_sign(current_terminal):
             return True
 
+        if is_block_label and isinstance(rule, Token) and rule.value == "string":
+            if (
+                current_terminal == self._last_terminal == Terminal("DBLQUOTE")
+                or current_terminal == Terminal("DBLQUOTE")
+                and self._last_terminal == Terminal("NAME")
+            ):
+                return True
+
         # if we're in a ternary or binary operator, add space around the operator
         if (
             isinstance(rule, Token)
@@ -235,7 +244,7 @@ class HCLReconstructor(Reconstructor):
                 return True
 
             # always add space between string literals
-            if current_terminal == Terminal("STRING_LIT"):
+            if current_terminal == Terminal("STRING_CHARS"):
                 return True
 
         # if we just opened a block, add a space, unless the block is empty
@@ -257,7 +266,7 @@ class HCLReconstructor(Reconstructor):
             # preceded by a space if they're following a comma in a tuple or
             # function arg
             if current_terminal in [
-                Terminal("STRING_LIT"),
+                Terminal("DBLQUOTE"),
                 Terminal("DECIMAL"),
                 Terminal("NAME"),
                 Terminal("NEGATIVE_DECIMAL"),
@@ -267,13 +276,15 @@ class HCLReconstructor(Reconstructor):
         # the catch-all case, we're not sure, so don't add a space
         return False
 
-    def _reconstruct(self, tree):
+    def _reconstruct(self, tree, is_block_label=False):
         unreduced_tree = self.match_tree(tree, tree.data)
         res = self.write_tokens.transform(unreduced_tree)
         for item in res:
             # any time we encounter a child tree, we recurse
             if isinstance(item, Tree):
-                yield from self._reconstruct(item)
+                yield from self._reconstruct(
+                    item, (unreduced_tree.data == "block" and item.data != "body")
+                )
 
             # every leaf should be a tuple, which contains information about
             # which terminal the leaf represents
@@ -309,7 +320,7 @@ class HCLReconstructor(Reconstructor):
                     self._deferred_item = None
 
                 # potentially add a space before the next token
-                if self._should_add_space(rule, terminal):
+                if self._should_add_space(rule, terminal, is_block_label):
                     yield " "
                     self._last_char_space = True
 
@@ -353,11 +364,21 @@ class HCLReverseTransformer:
 
     @staticmethod
     def _escape_interpolated_str(interp_s: str) -> str:
-        # begin by doing basic JSON string escaping, to add backslashes
-        interp_s = json.dumps(interp_s)
-
+        if interp_s.strip().startswith("<<-") or interp_s.strip().startswith("<<"):
+            # For heredoc strings, preserve their format exactly
+            return reverse_quotes_within_interpolation(interp_s)
+        # Escape backslashes first (very important to do this first)
+        escaped = interp_s.replace("\\", "\\\\")
+        # Escape quotes
+        escaped = escaped.replace('"', '\\"')
+        # Escape control characters
+        escaped = escaped.replace("\n", "\\n")
+        escaped = escaped.replace("\r", "\\r")
+        escaped = escaped.replace("\t", "\\t")
+        escaped = escaped.replace("\b", "\\b")
+        escaped = escaped.replace("\f", "\\f")
         # find each interpolation within the string and remove the backslashes
-        interp_s = reverse_quotes_within_interpolation(interp_s)
+        interp_s = reverse_quotes_within_interpolation(f"{escaped}")
         return interp_s
 
     @staticmethod
@@ -410,6 +431,44 @@ class HCLReverseTransformer:
             [Token("NL_OR_COMMENT", f"\n{'  ' * level}") for _ in range(count)],
         )
 
+    def _build_string_rule(self, string: str, level: int = 0) -> Tree:
+        # grammar in hcl2.lark defines that a string is built of any number of string parts,
+        #   each string part can be either interpolation expression, escaped interpolation string
+        #   or regular string
+        # this method build hcl2 string rule based on arbitrary string,
+        #   splitting such string into individual parts and building a lark tree out of them
+        #
+        result = []
+
+        pattern = regex.compile(r"(\${1,2}\{(?:[^{}]|(?R))*\})")
+        parts = [part for part in pattern.split(string) if part != ""]
+        # e.g. 'aaa$${bbb}ccc${"ddd-${eee}"}' -> ['aaa', '$${bbb}', 'ccc', '${"ddd-${eee}"}']
+        # 'aa-${"bb-${"cc-${"dd-${5 + 5}"}"}"}' -> ['aa-', '${"bb-${"cc-${"dd-${5 + 5}"}"}"}']
+
+        for part in parts:
+            if part.startswith("$${") and part.endswith("}"):
+                result.append(Token("ESCAPED_INTERPOLATION", part))
+
+            # unwrap interpolation expression and recurse into it
+            elif part.startswith("${") and part.endswith("}"):
+                part = part[2:-1]
+                if part.startswith('"') and part.endswith('"'):
+                    part = part[1:-1]
+                    part = self._transform_value_to_expr_term(part, level)
+                else:
+                    part = Tree(
+                        Token("RULE", "expr_term"),
+                        [Tree(Token("RULE", "identifier"), [Token("NAME", part)])],
+                    )
+
+                result.append(Tree(Token("RULE", "interpolation"), [part]))
+
+            else:
+                result.append(Token("STRING_CHARS", part))
+
+        result = [Tree(Token("RULE", "string_part"), [element]) for element in result]
+        return Tree(Token("RULE", "string"), result)
+
     def _is_block(self, value: Any) -> bool:
         if isinstance(value, dict):
             block_body = value
@@ -475,8 +534,8 @@ class HCLReverseTransformer:
                     block_labels, block_body_dict = self._calculate_block_labels(
                         block_v
                     )
-                    block_label_tokens = [
-                        Token("STRING_LIT", f'"{block_label}"')
+                    block_label_trees = [
+                        self._build_string_rule(block_label, level)
                         for block_label in block_labels
                     ]
                     block_body = self._transform_dict_to_body(
@@ -486,7 +545,7 @@ class HCLReverseTransformer:
                     # create our actual block to add to our own body
                     block = Tree(
                         Token("RULE", "block"),
-                        [identifier_name] + block_label_tokens + [block_body],
+                        [identifier_name] + block_label_trees + [block_body],
                     )
                     children.append(block)
                     # add empty line after block
@@ -665,10 +724,10 @@ class HCLReverseTransformer:
                 parsed_value = attribute.children[2]
                 return parsed_value
 
-            # otherwise it's just a string.
+            # otherwise it's a string
             return Tree(
                 Token("RULE", "expr_term"),
-                [Token("STRING_LIT", self._escape_interpolated_str(value))],
+                [self._build_string_rule(self._escape_interpolated_str(value), level)],
             )
 
         # otherwise, we don't know the type

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/hcl2/transformer.py

@@ -102,10 +102,12 @@ class DictTransformer(Transformer):
         # This returns a dict with a single key/value pair to make it easier to merge these
         # into a bigger dict that is returned by the "object" function
 
-        key = self.strip_quotes(str(args[0].children[0]))
-        value = args[2]
+        key = str(args[0].children[0])
+        if not re.match(r".*?(\${).*}.*", key):
+            # do not strip quotes of a interpolation string
+            key = self.strip_quotes(key)
 
-        value = self.to_string_dollar(value)
+        value = self.to_string_dollar(args[2])
         return {key: value}
 
     def object_elem_key_dot_accessor(self, args: List) -> str:
@@ -245,7 +247,8 @@ class DictTransformer(Transformer):
             raise RuntimeError(f"Invalid Heredoc token: {args[0]}")
 
         trim_chars = "\n\t "
-        return f'"{match.group(2).rstrip(trim_chars)}"'
+        result = match.group(2).rstrip(trim_chars)
+        return f'"{result}"'
 
     def heredoc_template_trim(self, args: List) -> str:
         # See https://github.com/hashicorp/hcl2/blob/master/hcl/hclsyntax/spec.md#template-expressions
@@ -295,12 +298,17 @@ class DictTransformer(Transformer):
         # e.g. f"{2 + 2} {{2 + 2}}" == "4 {2 + 2}"
         return f"{{{for_expr}}}"
 
-    def string_with_interpolation(self, args: List) -> str:
-        return '"' + ("".join(args)) + '"'
+    def string(self, args: List) -> str:
+        return '"' + "".join(args) + '"'
+
+    def string_part(self, args: List) -> str:
+        value = self.to_tf_inline(args[0])
+        if value.startswith('"') and value.endswith('"'):
+            value = value[1:-1]
+        return value
 
-    def interpolation_maybe_nested(self, args: List) -> str:
-        # return "".join(args)
-        return "${" + ("".join(args)) + "}"
+    def interpolation(self, args: List) -> str:
+        return '"${' + str(args[0]) + '}"'
 
     def strip_new_line_tokens(self, args: List) -> List:
         """

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/hcl2/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '7.2.0'
-__version_tuple__ = version_tuple = (7, 2, 0)
+__version__ = version = '7.3.0'
+__version_tuple__ = version_tuple = (7, 3, 0)

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/python_hcl2.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-hcl2
-Version: 7.2.0
+Version: 7.3.0
 Summary: A parser for HCL2
 Author-email: Amplify Education <github@amplify.com>
 License: MIT

{python_hcl2-7.2.0 → python_hcl2-7.3.0}/requirements.txt

@@ -1,3 +1,4 @@
 # Place dependencies in this file, following the distutils format:
 # http://docs.python.org/2/distutils/setupscript.html#relationships-between-distributions-and-packages
 lark>=1.1.5,<2.0
+regex>=2024.4.16