pineforge-codegen 1.9.0__py3-none-any.whl → 1.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pineforge_codegen/__init__.py +14 -1
- pineforge_codegen/analyzer/call_handlers.py +3 -2
- pineforge_codegen/codegen/ascii.py +101 -0
- pineforge_codegen/codegen/base.py +60 -7
- pineforge_codegen/codegen/closed_trade_fold.py +399 -0
- pineforge_codegen/codegen/emit_top.py +34 -3
- pineforge_codegen/codegen/helpers.py +2 -0
- pineforge_codegen/codegen/security.py +2 -1
- pineforge_codegen/codegen/tables.py +7 -4
- pineforge_codegen/codegen/types.py +5 -3
- pineforge_codegen/codegen/visit_call.py +16 -10
- pineforge_codegen/codegen/visit_expr.py +3 -0
- pineforge_codegen/codegen/visit_stmt.py +94 -1
- pineforge_codegen/cpp_allowlist.py +18 -0
- pineforge_codegen/cpp_names.py +17 -0
- pineforge_codegen/diagnostics_catalog.json +27 -0
- pineforge_codegen/library_inline.py +5 -0
- pineforge_codegen/pine_spelling.py +18 -0
- pineforge_codegen/symbols.py +3 -2
- {pineforge_codegen-1.9.0.dist-info → pineforge_codegen-1.9.1.dist-info}/METADATA +1 -1
- {pineforge_codegen-1.9.0.dist-info → pineforge_codegen-1.9.1.dist-info}/RECORD +24 -21
- {pineforge_codegen-1.9.0.dist-info → pineforge_codegen-1.9.1.dist-info}/WHEEL +0 -0
- {pineforge_codegen-1.9.0.dist-info → pineforge_codegen-1.9.1.dist-info}/entry_points.txt +0 -0
- {pineforge_codegen-1.9.0.dist-info → pineforge_codegen-1.9.1.dist-info}/licenses/LICENSE +0 -0
pineforge_codegen/__init__.py
CHANGED
|
@@ -6,8 +6,9 @@ from .lexer import Lexer
|
|
|
6
6
|
from .parser import Parser
|
|
7
7
|
from .analyzer import Analyzer
|
|
8
8
|
from .codegen import CodeGen
|
|
9
|
+
from .cpp_allowlist import REASONS, check_generated_cpp
|
|
9
10
|
from .diagnostic_codes import diagnostics_catalog, render_diagnostic
|
|
10
|
-
from .errors import CompileError, Level, Phase
|
|
11
|
+
from .errors import CompileError, Diagnostic, Level, Phase, SourceLocation
|
|
11
12
|
from .external_requests import lower_no_data_requests
|
|
12
13
|
from .builtin_keywords import bind_builtin_keywords
|
|
13
14
|
from .finite_ta_length import expand_finite_choice_extrema_lengths
|
|
@@ -99,6 +100,18 @@ def _generate(pine_source: str, check_support: bool, filename: str,
|
|
|
99
100
|
continue
|
|
100
101
|
needed = gen.session_functions_needing_clones
|
|
101
102
|
if not needed:
|
|
103
|
+
findings = check_generated_cpp(cpp)
|
|
104
|
+
if findings:
|
|
105
|
+
raise CompileError([
|
|
106
|
+
Diagnostic(
|
|
107
|
+
level=Level.ERROR,
|
|
108
|
+
phase=Phase.CODEGEN,
|
|
109
|
+
location=SourceLocation(file=filename, line=1, col=1, end_col=1),
|
|
110
|
+
message=REASONS[finding.rule],
|
|
111
|
+
hint="Report this generated C++ allowlist failure to PineForge.",
|
|
112
|
+
)
|
|
113
|
+
for finding in findings[:20]
|
|
114
|
+
])
|
|
102
115
|
return gen, ctx, cpp, support_diagnostics, sites
|
|
103
116
|
# Only uncloned functions ask, so the set grows each time and the
|
|
104
117
|
# loop ends by the time every function that reads a flag at an
|
|
@@ -71,6 +71,7 @@ from ..ast_nodes import (
|
|
|
71
71
|
Ternary, TupleAssign, TupleLiteral, UnaryOp, VarDecl,
|
|
72
72
|
)
|
|
73
73
|
from ..method_binding import bind_function_defaults
|
|
74
|
+
from ..pine_spelling import sub_outside_literals
|
|
74
75
|
from ..external_requests import FEED_LOWERING, LOWERING_ANNOTATION
|
|
75
76
|
from ..security_contexts import CONTEXT_ANNOTATION, DEAD_ANNOTATION
|
|
76
77
|
from ..symbols import PineType
|
|
@@ -1524,7 +1525,7 @@ class CallHandlers:
|
|
|
1524
1525
|
result = arg
|
|
1525
1526
|
for param, value in sorted(
|
|
1526
1527
|
pmap.items(), key=lambda item: len(item[0]), reverse=True):
|
|
1527
|
-
result =
|
|
1528
|
+
result = sub_outside_literals(rf'\b{re.escape(param)}\b', value, result)
|
|
1528
1529
|
return result
|
|
1529
1530
|
|
|
1530
1531
|
def _expand_locals(arg: str) -> str:
|
|
@@ -1537,7 +1538,7 @@ class CallHandlers:
|
|
|
1537
1538
|
if name in local_defs:
|
|
1538
1539
|
return "(" + local_defs[name] + ")"
|
|
1539
1540
|
return name
|
|
1540
|
-
expanded =
|
|
1541
|
+
expanded = sub_outside_literals(r"[A-Za-z_][A-Za-z_0-9]*", _rep, arg)
|
|
1541
1542
|
if expanded == arg:
|
|
1542
1543
|
break
|
|
1543
1544
|
arg = expanded
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""ASCII spelling of the completed C++ translation unit.
|
|
2
|
+
|
|
3
|
+
Keep the compiler's internal names and string values intact until emission is
|
|
4
|
+
complete: generated history, callable and library names can contain an authored
|
|
5
|
+
name, and settings metadata reads the emitter's original string spelling.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from .helpers import BUILTIN_ACCESSOR_NAMES, CPP_RESERVED, is_emitter_temporary
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# These are the tokens emitted by codegen, including ordinary character
|
|
17
|
+
# literals. Comments and quoted text must be consumed before identifiers:
|
|
18
|
+
# a name inside either is data, even when it spells a bound identifier.
|
|
19
|
+
# Consume preprocessing numbers before character literals so a digit separator
|
|
20
|
+
# cannot pair with a later apostrophe and hide the intervening code.
|
|
21
|
+
_TOKENS = re.compile(
|
|
22
|
+
r"(?P<comment>//[^\r\n]*|/\*.*?(?:\*/|\Z))"
|
|
23
|
+
r"|(?P<number>(?:[0-9]|\.[0-9])(?:[eEpP][+-]|[\w.]|'[\w])*)"
|
|
24
|
+
r"""|(?P<prefixed>(?:u8|[LuU])?R(?=")|(?:u8|[LuU])(?=["']))"""
|
|
25
|
+
r"""|(?P<quoted>"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')"""
|
|
26
|
+
r"|(?P<identifier>[^\W\d]\w*)",
|
|
27
|
+
re.DOTALL,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _escape_utf8(text: str) -> str:
|
|
32
|
+
"""Spell non-ASCII UTF-8 bytes as fixed-width C++ octal escapes.
|
|
33
|
+
|
|
34
|
+
Three digits terminate each escape even before an authored digit. ASCII
|
|
35
|
+
escapes already emitted for quotes, backslashes, whitespace and NUL remain
|
|
36
|
+
byte-identical. In comments the same spelling makes the text visible
|
|
37
|
+
without introducing a comment delimiter or a line splice.
|
|
38
|
+
"""
|
|
39
|
+
return "".join(
|
|
40
|
+
ch if ch.isascii() else "".join(f"\\{byte:03o}" for byte in ch.encode("utf-8"))
|
|
41
|
+
for ch in text
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def ascii_cpp(cpp: str, *, reserved: Iterable[str] = ()) -> str:
|
|
46
|
+
"""Encode non-ASCII tokens without changing any existing ASCII token.
|
|
47
|
+
|
|
48
|
+
UTF-8 hex is injective. Reserve the complete translation unit's identifiers,
|
|
49
|
+
all authored names (including unused ones), and engine/emitter names before
|
|
50
|
+
allocating encodings in sorted order. A suffix avoids an ASCII author
|
|
51
|
+
deliberately spelling an encoding; allocated names are reserved too.
|
|
52
|
+
|
|
53
|
+
Prefixed literals and non-ASCII character literals have no byte-preserving
|
|
54
|
+
rewrite here. Those forms, or any non-ASCII code left after substitution,
|
|
55
|
+
violate emitter invariants and raise AssertionError.
|
|
56
|
+
"""
|
|
57
|
+
if cpp.isascii():
|
|
58
|
+
return cpp
|
|
59
|
+
|
|
60
|
+
identifiers = set()
|
|
61
|
+
for match in _TOKENS.finditer(cpp):
|
|
62
|
+
text = match.group()
|
|
63
|
+
if match.lastgroup == "prefixed":
|
|
64
|
+
line = cpp.count("\n", 0, match.start()) + 1
|
|
65
|
+
raise AssertionError(f"unexpected prefixed C++ literal at line {line}")
|
|
66
|
+
if (match.lastgroup == "quoted" and text.startswith("'")
|
|
67
|
+
and not text.isascii()):
|
|
68
|
+
line = cpp.count("\n", 0, match.start()) + 1
|
|
69
|
+
raise AssertionError(f"non-ASCII C++ character literal at line {line}")
|
|
70
|
+
if match.lastgroup == "identifier":
|
|
71
|
+
identifiers.add(text)
|
|
72
|
+
occupied = identifiers | set(reserved) | CPP_RESERVED | BUILTIN_ACCESSOR_NAMES
|
|
73
|
+
names: dict[str, str] = {}
|
|
74
|
+
for name in sorted(identifiers):
|
|
75
|
+
if name.isascii():
|
|
76
|
+
continue
|
|
77
|
+
base = f"pf_utf8_{name.encode('utf-8').hex()}"
|
|
78
|
+
candidate = base
|
|
79
|
+
suffix = 2
|
|
80
|
+
while candidate in occupied or is_emitter_temporary(candidate):
|
|
81
|
+
candidate = f"{base}_{suffix}"
|
|
82
|
+
suffix += 1
|
|
83
|
+
names[name] = candidate
|
|
84
|
+
occupied.add(candidate)
|
|
85
|
+
|
|
86
|
+
def replace(match: re.Match[str]) -> str:
|
|
87
|
+
text = match.group()
|
|
88
|
+
if match.lastgroup == "identifier":
|
|
89
|
+
return names.get(text, text)
|
|
90
|
+
if match.lastgroup == "number":
|
|
91
|
+
return text
|
|
92
|
+
return text if text.isascii() else _escape_utf8(text)
|
|
93
|
+
|
|
94
|
+
result = _TOKENS.sub(replace, cpp)
|
|
95
|
+
if not result.isascii():
|
|
96
|
+
first = next(index for index, ch in enumerate(result) if not ch.isascii())
|
|
97
|
+
line = result.count("\n", 0, first) + 1
|
|
98
|
+
raise AssertionError(
|
|
99
|
+
f"non-ASCII C++ code point U+{ord(result[first]):04X} at line {line}"
|
|
100
|
+
)
|
|
101
|
+
return result
|
|
@@ -37,7 +37,7 @@ from ..session_reads import emitted_session_reads
|
|
|
37
37
|
from ..block_locals import block_declarations, decl_key
|
|
38
38
|
from ..pine_spelling import (
|
|
39
39
|
blank_string_literals, input_call_spans, pine_string_literal,
|
|
40
|
-
spell_input_call, sub_identifiers,
|
|
40
|
+
spell_input_call, sub_identifiers, sub_outside_literals,
|
|
41
41
|
)
|
|
42
42
|
|
|
43
43
|
|
|
@@ -299,6 +299,7 @@ TA_TUPLE_RESULT_TYPES = {
|
|
|
299
299
|
|
|
300
300
|
# CPP_RESERVED + the NamingHelper mixin are pulled in from helpers.py so the
|
|
301
301
|
# small naming/walk utilities can be shared with future visitor mixins.
|
|
302
|
+
from .ascii import ascii_cpp
|
|
302
303
|
from .collection_history import CollectionHistoryEmitter
|
|
303
304
|
from .helpers import (
|
|
304
305
|
CPP_RESERVED, INLINE_HISTORY_KINDS, SESSION_FLAG_MEMBERS, NamingHelper,
|
|
@@ -787,6 +788,8 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
787
788
|
self._security_arg_call_values: dict[tuple, str] = {}
|
|
788
789
|
self._random_call_counter = 0
|
|
789
790
|
self._for_counter = 0
|
|
791
|
+
# Retain each loop node alongside its plan; ids are lookup keys only.
|
|
792
|
+
self._closed_trade_folds: dict[int, tuple[ForStmt, object]] = {}
|
|
790
793
|
self._tuple_assign_counter = 0
|
|
791
794
|
# Synthetic history buffers used by inline call-history and by scalar
|
|
792
795
|
# expressions passed to UDF series parameters. They are pre-registered
|
|
@@ -4635,7 +4638,7 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
4635
4638
|
for type_name in self._udt_defs:
|
|
4636
4639
|
self._udt_record_cpp_names[type_name] = (
|
|
4637
4640
|
self._allocate_generated_cpp_name(
|
|
4638
|
-
|
|
4641
|
+
self._udt_record_cpp_type(type_name), global_used
|
|
4639
4642
|
)
|
|
4640
4643
|
)
|
|
4641
4644
|
|
|
@@ -4964,7 +4967,7 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
4964
4967
|
raise first from None
|
|
4965
4968
|
if self._deferred_settings_errors:
|
|
4966
4969
|
raise self._deferred_settings_errors[0]
|
|
4967
|
-
return cpp
|
|
4970
|
+
return ascii_cpp(cpp, reserved=self._safe_name_occupied)
|
|
4968
4971
|
finally:
|
|
4969
4972
|
self._deferred_settings_errors = None
|
|
4970
4973
|
|
|
@@ -4984,6 +4987,44 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
4984
4987
|
return held[0]
|
|
4985
4988
|
return None
|
|
4986
4989
|
|
|
4990
|
+
def _prepare_closed_trade_folds(self) -> None:
|
|
4991
|
+
"""Keep planned loops whose bindings use ordinary scalar storage."""
|
|
4992
|
+
from .closed_trade_fold import plan_closed_trade_folds
|
|
4993
|
+
|
|
4994
|
+
plans = plan_closed_trade_folds(self.ctx.ast, self._safe_name)
|
|
4995
|
+
self._closed_trade_folds = {}
|
|
4996
|
+
if not plans:
|
|
4997
|
+
return
|
|
4998
|
+
|
|
4999
|
+
loops: dict[int, ForStmt] = {}
|
|
5000
|
+
for statement in self.ctx.ast.body:
|
|
5001
|
+
if self._budget is not None:
|
|
5002
|
+
self._budget.check(statement.loc, Phase.CODEGEN)
|
|
5003
|
+
if isinstance(statement, IfStmt):
|
|
5004
|
+
for child in statement.body:
|
|
5005
|
+
if isinstance(child, ForStmt):
|
|
5006
|
+
loops[id(child)] = child
|
|
5007
|
+
|
|
5008
|
+
special_storage = set(self.ctx.series_vars) | self._var_names
|
|
5009
|
+
excluded_accumulators = (
|
|
5010
|
+
special_storage | set(self._known_vars) | self._nonfinite_int_names()
|
|
5011
|
+
)
|
|
5012
|
+
excluded_locals = special_storage | self._global_member_vars
|
|
5013
|
+
for plan in plans:
|
|
5014
|
+
node = loops.get(plan.for_node_id)
|
|
5015
|
+
if node is None:
|
|
5016
|
+
continue
|
|
5017
|
+
if self._budget is not None:
|
|
5018
|
+
self._budget.check(node.loc, Phase.CODEGEN)
|
|
5019
|
+
if any(
|
|
5020
|
+
name in excluded_accumulators or name not in self._global_member_vars
|
|
5021
|
+
for name, _safe in plan.accumulators
|
|
5022
|
+
):
|
|
5023
|
+
continue
|
|
5024
|
+
if any(name in excluded_locals for name, _accessor, _cpp in plan.locals):
|
|
5025
|
+
continue
|
|
5026
|
+
self._closed_trade_folds[plan.for_node_id] = (node, plan)
|
|
5027
|
+
|
|
4987
5028
|
def _generate_cpp(self) -> str:
|
|
4988
5029
|
# Every input is keyed by its title: refuse a non-constant one first,
|
|
4989
5030
|
# then flag inputs one override key would reach together.
|
|
@@ -5006,6 +5047,7 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
5006
5047
|
self._prepare_array_journal()
|
|
5007
5048
|
self._prepare_history_sharing()
|
|
5008
5049
|
graph_journal.prepare(self)
|
|
5050
|
+
self._prepare_closed_trade_folds()
|
|
5009
5051
|
self._security_ohlc_hist_fields_by_sec: dict[int, set[str]] = {}
|
|
5010
5052
|
# A source input's selected series read at a history offset in a
|
|
5011
5053
|
# payload: (key, default) -> (the input call, its history field).
|
|
@@ -5869,6 +5911,15 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
5869
5911
|
f"{self._collection_history_capacity(variable)};"
|
|
5870
5912
|
)
|
|
5871
5913
|
|
|
5914
|
+
# Closed-trade prefixes belong to the declaration-derived checkpoint.
|
|
5915
|
+
for _node, plan in self._closed_trade_folds.values():
|
|
5916
|
+
lines.append(f" int {plan.stem}settled = 0;")
|
|
5917
|
+
lines.append(f" int {plan.stem}seen = 0;")
|
|
5918
|
+
for index, (_name, safe) in enumerate(plan.accumulators):
|
|
5919
|
+
lines.append(
|
|
5920
|
+
f" decltype(GeneratedStrategy::{safe}) {plan.stem}s{index} = {{}};"
|
|
5921
|
+
)
|
|
5922
|
+
|
|
5872
5923
|
# 9. _var_initialized flag
|
|
5873
5924
|
if self.ctx.var_members:
|
|
5874
5925
|
lines.append(" bool _var_initialized = false;")
|
|
@@ -6517,18 +6568,18 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
6517
6568
|
# double-call ``tf_to_seconds(script_tf_)()``. Property-form members
|
|
6518
6569
|
# (``timeframe.isdaily``) never carry ``()`` so the optional group is
|
|
6519
6570
|
# a no-op for them.
|
|
6520
|
-
expr =
|
|
6571
|
+
expr = sub_outside_literals(r"\btimeframe\.(\w+)(?:\(\))?", _tf_rep, expr)
|
|
6521
6572
|
|
|
6522
6573
|
# 2) math.<member> -> C++ (constants + std::* functions).
|
|
6523
6574
|
def _math_rep(p: re.Match) -> str:
|
|
6524
6575
|
mem = p.group(1)
|
|
6525
6576
|
return self._MATH_MEMBER_CPP.get(mem, p.group(0))
|
|
6526
|
-
expr =
|
|
6577
|
+
expr = sub_outside_literals(r"\bmath\.(\w+)", _math_rep, expr)
|
|
6527
6578
|
|
|
6528
6579
|
# 3) Pine word-logical operators -> C++ operators (after timeframe/math
|
|
6529
6580
|
# substitution so we don't rewrite inside their C++ expansions).
|
|
6530
6581
|
for pine_op, cpp_op in self._PINE_LOGICAL_OPS.items():
|
|
6531
|
-
expr =
|
|
6582
|
+
expr = sub_outside_literals(rf"\b{pine_op}\b", cpp_op, expr)
|
|
6532
6583
|
|
|
6533
6584
|
# 4) Substitute input-backed vars with override-aware get_input_*()
|
|
6534
6585
|
# reads, and inline known compile-time consts (non-input) as literals.
|
|
@@ -6554,7 +6605,9 @@ class CodeGen(CallVisitor, ExprVisitor, StmtVisitor, TopLevelEmitter, SecurityEm
|
|
|
6554
6605
|
return f'std::string("{self._cpp_string_escape(val)}")'
|
|
6555
6606
|
return name
|
|
6556
6607
|
|
|
6557
|
-
|
|
6608
|
+
# Identifier tokens only: a string literal's text is not identifiers
|
|
6609
|
+
# (a literal that spells an input's or a constant's name stays as is).
|
|
6610
|
+
rewritten = sub_identifiers(expr, _sub)
|
|
6558
6611
|
|
|
6559
6612
|
# 5) Pine auto-converts floats to ints for TA lengths; C++ does not.
|
|
6560
6613
|
# If any math.* function appears (returns double) OR a timeframe.*
|