pineforge-codegen 1.8.0__py3-none-any.whl → 1.9.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,8 +6,9 @@ from .lexer import Lexer
6
6
  from .parser import Parser
7
7
  from .analyzer import Analyzer
8
8
  from .codegen import CodeGen
9
+ from .cpp_allowlist import REASONS, check_generated_cpp
9
10
  from .diagnostic_codes import diagnostics_catalog, render_diagnostic
10
- from .errors import CompileError, Level, Phase
11
+ from .errors import CompileError, Diagnostic, Level, Phase, SourceLocation
11
12
  from .external_requests import lower_no_data_requests
12
13
  from .builtin_keywords import bind_builtin_keywords
13
14
  from .finite_ta_length import expand_finite_choice_extrema_lengths
@@ -99,6 +100,18 @@ def _generate(pine_source: str, check_support: bool, filename: str,
99
100
  continue
100
101
  needed = gen.session_functions_needing_clones
101
102
  if not needed:
103
+ findings = check_generated_cpp(cpp)
104
+ if findings:
105
+ raise CompileError([
106
+ Diagnostic(
107
+ level=Level.ERROR,
108
+ phase=Phase.CODEGEN,
109
+ location=SourceLocation(file=filename, line=1, col=1, end_col=1),
110
+ message=REASONS[finding.rule],
111
+ hint="Report this generated C++ allowlist failure to PineForge.",
112
+ )
113
+ for finding in findings[:20]
114
+ ])
102
115
  return gen, ctx, cpp, support_diagnostics, sites
103
116
  # Only uncloned functions ask, so the set grows each time and the
104
117
  # loop ends by the time every function that reads a flag at an
@@ -71,6 +71,7 @@ from ..ast_nodes import (
71
71
  Ternary, TupleAssign, TupleLiteral, UnaryOp, VarDecl,
72
72
  )
73
73
  from ..method_binding import bind_function_defaults
74
+ from ..pine_spelling import sub_outside_literals
74
75
  from ..external_requests import FEED_LOWERING, LOWERING_ANNOTATION
75
76
  from ..security_contexts import CONTEXT_ANNOTATION, DEAD_ANNOTATION
76
77
  from ..symbols import PineType
@@ -1524,7 +1525,7 @@ class CallHandlers:
1524
1525
  result = arg
1525
1526
  for param, value in sorted(
1526
1527
  pmap.items(), key=lambda item: len(item[0]), reverse=True):
1527
- result = re.sub(rf'\b{re.escape(param)}\b', value, result)
1528
+ result = sub_outside_literals(rf'\b{re.escape(param)}\b', value, result)
1528
1529
  return result
1529
1530
 
1530
1531
  def _expand_locals(arg: str) -> str:
@@ -1537,7 +1538,7 @@ class CallHandlers:
1537
1538
  if name in local_defs:
1538
1539
  return "(" + local_defs[name] + ")"
1539
1540
  return name
1540
- expanded = re.sub(r"[A-Za-z_][A-Za-z_0-9]*", _rep, arg)
1541
+ expanded = sub_outside_literals(r"[A-Za-z_][A-Za-z_0-9]*", _rep, arg)
1541
1542
  if expanded == arg:
1542
1543
  break
1543
1544
  arg = expanded
@@ -0,0 +1,101 @@
1
+ """ASCII spelling of the completed C++ translation unit.
2
+
3
+ Keep the compiler's internal names and string values intact until emission is
4
+ complete: generated history, callable and library names can contain an authored
5
+ name, and settings metadata reads the emitter's original string spelling.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Iterable
11
+ import re
12
+
13
+ from .helpers import BUILTIN_ACCESSOR_NAMES, CPP_RESERVED, is_emitter_temporary
14
+
15
+
16
+ # These are the tokens emitted by codegen, including ordinary character
17
+ # literals. Comments and quoted text must be consumed before identifiers:
18
+ # a name inside either is data, even when it spells a bound identifier.
19
+ # Consume preprocessing numbers before character literals so a digit separator
20
+ # cannot pair with a later apostrophe and hide the intervening code.
21
+ _TOKENS = re.compile(
22
+ r"(?P<comment>//[^\r\n]*|/\*.*?(?:\*/|\Z))"
23
+ r"|(?P<number>(?:[0-9]|\.[0-9])(?:[eEpP][+-]|[\w.]|'[\w])*)"
24
+ r"""|(?P<prefixed>(?:u8|[LuU])?R(?=")|(?:u8|[LuU])(?=["']))"""
25
+ r"""|(?P<quoted>"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')"""
26
+ r"|(?P<identifier>[^\W\d]\w*)",
27
+ re.DOTALL,
28
+ )
29
+
30
+
31
+ def _escape_utf8(text: str) -> str:
32
+ """Spell non-ASCII UTF-8 bytes as fixed-width C++ octal escapes.
33
+
34
+ Three digits terminate each escape even before an authored digit. ASCII
35
+ escapes already emitted for quotes, backslashes, whitespace and NUL remain
36
+ byte-identical. In comments the same spelling makes the text visible
37
+ without introducing a comment delimiter or a line splice.
38
+ """
39
+ return "".join(
40
+ ch if ch.isascii() else "".join(f"\\{byte:03o}" for byte in ch.encode("utf-8"))
41
+ for ch in text
42
+ )
43
+
44
+
45
+ def ascii_cpp(cpp: str, *, reserved: Iterable[str] = ()) -> str:
46
+ """Encode non-ASCII tokens without changing any existing ASCII token.
47
+
48
+ UTF-8 hex is injective. Reserve the complete translation unit's identifiers,
49
+ all authored names (including unused ones), and engine/emitter names before
50
+ allocating encodings in sorted order. A suffix avoids an ASCII author
51
+ deliberately spelling an encoding; allocated names are reserved too.
52
+
53
+ Prefixed literals and non-ASCII character literals have no byte-preserving
54
+ rewrite here. Those forms, or any non-ASCII code left after substitution,
55
+ violate emitter invariants and raise AssertionError.
56
+ """
57
+ if cpp.isascii():
58
+ return cpp
59
+
60
+ identifiers = set()
61
+ for match in _TOKENS.finditer(cpp):
62
+ text = match.group()
63
+ if match.lastgroup == "prefixed":
64
+ line = cpp.count("\n", 0, match.start()) + 1
65
+ raise AssertionError(f"unexpected prefixed C++ literal at line {line}")
66
+ if (match.lastgroup == "quoted" and text.startswith("'")
67
+ and not text.isascii()):
68
+ line = cpp.count("\n", 0, match.start()) + 1
69
+ raise AssertionError(f"non-ASCII C++ character literal at line {line}")
70
+ if match.lastgroup == "identifier":
71
+ identifiers.add(text)
72
+ occupied = identifiers | set(reserved) | CPP_RESERVED | BUILTIN_ACCESSOR_NAMES
73
+ names: dict[str, str] = {}
74
+ for name in sorted(identifiers):
75
+ if name.isascii():
76
+ continue
77
+ base = f"pf_utf8_{name.encode('utf-8').hex()}"
78
+ candidate = base
79
+ suffix = 2
80
+ while candidate in occupied or is_emitter_temporary(candidate):
81
+ candidate = f"{base}_{suffix}"
82
+ suffix += 1
83
+ names[name] = candidate
84
+ occupied.add(candidate)
85
+
86
+ def replace(match: re.Match[str]) -> str:
87
+ text = match.group()
88
+ if match.lastgroup == "identifier":
89
+ return names.get(text, text)
90
+ if match.lastgroup == "number":
91
+ return text
92
+ return text if text.isascii() else _escape_utf8(text)
93
+
94
+ result = _TOKENS.sub(replace, cpp)
95
+ if not result.isascii():
96
+ first = next(index for index, ch in enumerate(result) if not ch.isascii())
97
+ line = result.count("\n", 0, first) + 1
98
+ raise AssertionError(
99
+ f"non-ASCII C++ code point U+{ord(result[first]):04X} at line {line}"
100
+ )
101
+ return result
@@ -37,7 +37,7 @@ from ..session_reads import emitted_session_reads
37
37
  from ..block_locals import block_declarations, decl_key
38
38
  from ..pine_spelling import (
39
39
  blank_string_literals, input_call_spans, pine_string_literal,
40
- spell_input_call, sub_identifiers,
40
+ spell_input_call, sub_identifiers, sub_outside_literals,
41
41
  )
42
42
 
43
43
 
@@ -299,6 +299,7 @@ TA_TUPLE_RESULT_TYPES = {
299
299
 
300
300
  # CPP_RESERVED + the NamingHelper mixin are pulled in from helpers.py so the
301
301
  # small naming/walk utilities can be shared with future visitor mixins.
302
+ from .ascii import ascii_cpp
302
303
  from .collection_history import CollectionHistoryEmitter
303
304
  from .helpers import (
304
305
  CPP_RESERVED, INLINE_HISTORY_KINDS, SESSION_FLAG_MEMBERS, NamingHelper,
@@ -787,6 +788,8 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
787
788
  self._security_arg_call_values: dict[tuple, str] = {}
788
789
  self._random_call_counter = 0
789
790
  self._for_counter = 0
791
+ # Retain each loop node alongside its plan; ids are lookup keys only.
792
+ self._closed_trade_folds: dict[int, tuple[ForStmt, object]] = {}
790
793
  self._tuple_assign_counter = 0
791
794
  # Synthetic history buffers used by inline call-history and by scalar
792
795
  # expressions passed to UDF series parameters. They are pre-registered
@@ -4635,7 +4638,7 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
4635
4638
  for type_name in self._udt_defs:
4636
4639
  self._udt_record_cpp_names[type_name] = (
4637
4640
  self._allocate_generated_cpp_name(
4638
- f"_PFUdtRecord_{type_name}", global_used
4641
+ self._udt_record_cpp_type(type_name), global_used
4639
4642
  )
4640
4643
  )
4641
4644
 
@@ -4964,7 +4967,7 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
4964
4967
  raise first from None
4965
4968
  if self._deferred_settings_errors:
4966
4969
  raise self._deferred_settings_errors[0]
4967
- return cpp
4970
+ return ascii_cpp(cpp, reserved=self._safe_name_occupied)
4968
4971
  finally:
4969
4972
  self._deferred_settings_errors = None
4970
4973
 
@@ -4984,6 +4987,44 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
4984
4987
  return held[0]
4985
4988
  return None
4986
4989
 
4990
+ def _prepare_closed_trade_folds(self) -> None:
4991
+ """Keep planned loops whose bindings use ordinary scalar storage."""
4992
+ from .closed_trade_fold import plan_closed_trade_folds
4993
+
4994
+ plans = plan_closed_trade_folds(self.ctx.ast, self._safe_name)
4995
+ self._closed_trade_folds = {}
4996
+ if not plans:
4997
+ return
4998
+
4999
+ loops: dict[int, ForStmt] = {}
5000
+ for statement in self.ctx.ast.body:
5001
+ if self._budget is not None:
5002
+ self._budget.check(statement.loc, Phase.CODEGEN)
5003
+ if isinstance(statement, IfStmt):
5004
+ for child in statement.body:
5005
+ if isinstance(child, ForStmt):
5006
+ loops[id(child)] = child
5007
+
5008
+ special_storage = set(self.ctx.series_vars) | self._var_names
5009
+ excluded_accumulators = (
5010
+ special_storage | set(self._known_vars) | self._nonfinite_int_names()
5011
+ )
5012
+ excluded_locals = special_storage | self._global_member_vars
5013
+ for plan in plans:
5014
+ node = loops.get(plan.for_node_id)
5015
+ if node is None:
5016
+ continue
5017
+ if self._budget is not None:
5018
+ self._budget.check(node.loc, Phase.CODEGEN)
5019
+ if any(
5020
+ name in excluded_accumulators or name not in self._global_member_vars
5021
+ for name, _safe in plan.accumulators
5022
+ ):
5023
+ continue
5024
+ if any(name in excluded_locals for name, _accessor, _cpp in plan.locals):
5025
+ continue
5026
+ self._closed_trade_folds[plan.for_node_id] = (node, plan)
5027
+
4987
5028
  def _generate_cpp(self) -> str:
4988
5029
  # Every input is keyed by its title: refuse a non-constant one first,
4989
5030
  # then flag inputs one override key would reach together.
@@ -5006,6 +5047,7 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
5006
5047
  self._prepare_array_journal()
5007
5048
  self._prepare_history_sharing()
5008
5049
  graph_journal.prepare(self)
5050
+ self._prepare_closed_trade_folds()
5009
5051
  self._security_ohlc_hist_fields_by_sec: dict[int, set[str]] = {}
5010
5052
  # A source input's selected series read at a history offset in a
5011
5053
  # payload: (key, default) -> (the input call, its history field).
@@ -5869,6 +5911,15 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
5869
5911
  f"{self._collection_history_capacity(variable)};"
5870
5912
  )
5871
5913
 
5914
+ # Closed-trade prefixes belong to the declaration-derived checkpoint.
5915
+ for _node, plan in self._closed_trade_folds.values():
5916
+ lines.append(f" int {plan.stem}settled = 0;")
5917
+ lines.append(f" int {plan.stem}seen = 0;")
5918
+ for index, (_name, safe) in enumerate(plan.accumulators):
5919
+ lines.append(
5920
+ f" decltype(GeneratedStrategy::{safe}) {plan.stem}s{index} = {{}};"
5921
+ )
5922
+
5872
5923
  # 9. _var_initialized flag
5873
5924
  if self.ctx.var_members:
5874
5925
  lines.append(" bool _var_initialized = false;")
@@ -6517,18 +6568,18 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
6517
6568
  # double-call ``tf_to_seconds(script_tf_)()``. Property-form members
6518
6569
  # (``timeframe.isdaily``) never carry ``()`` so the optional group is
6519
6570
  # a no-op for them.
6520
- expr = re.sub(r"\btimeframe\.(\w+)(?:\(\))?", _tf_rep, expr)
6571
+ expr = sub_outside_literals(r"\btimeframe\.(\w+)(?:\(\))?", _tf_rep, expr)
6521
6572
 
6522
6573
  # 2) math.<member> -> C++ (constants + std::* functions).
6523
6574
  def _math_rep(p: re.Match) -> str:
6524
6575
  mem = p.group(1)
6525
6576
  return self._MATH_MEMBER_CPP.get(mem, p.group(0))
6526
- expr = re.sub(r"\bmath\.(\w+)", _math_rep, expr)
6577
+ expr = sub_outside_literals(r"\bmath\.(\w+)", _math_rep, expr)
6527
6578
 
6528
6579
  # 3) Pine word-logical operators -> C++ operators (after timeframe/math
6529
6580
  # substitution so we don't rewrite inside their C++ expansions).
6530
6581
  for pine_op, cpp_op in self._PINE_LOGICAL_OPS.items():
6531
- expr = re.sub(rf"\b{pine_op}\b", cpp_op, expr)
6582
+ expr = sub_outside_literals(rf"\b{pine_op}\b", cpp_op, expr)
6532
6583
 
6533
6584
  # 4) Substitute input-backed vars with override-aware get_input_*()
6534
6585
  # reads, and inline known compile-time consts (non-input) as literals.
@@ -6554,7 +6605,9 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
6554
6605
  return f'std::string("{self._cpp_string_escape(val)}")'
6555
6606
  return name
6556
6607
 
6557
- rewritten = ident_re.sub(_sub, expr)
6608
+ # Identifier tokens only: a string literal's text is not identifiers
6609
+ # (a literal that spells an input's or a constant's name stays as is).
6610
+ rewritten = sub_identifiers(expr, _sub)
6558
6611
 
6559
6612
  # 5) Pine auto-converts floats to ints for TA lengths; C++ does not.
6560
6613
  # If any math.* function appears (returns double) OR a timeframe.*