pgkiln 0.51.0 → 0.53.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/README.md +1 -1
  2. package/SECURITY.md +10 -1
  3. package/db/migrations/121_ai_agents.sql +94 -0
  4. package/db/migrations/122_team_development.sql +186 -0
  5. package/db/migrations/123_workflow_outcomes.sql +407 -0
  6. package/db/migrations/124_component_details.sql +51 -0
  7. package/db/migrations/125_app_change.sql +23 -0
  8. package/db/migrations/126_mail_queue_network_access.sql +119 -0
  9. package/docs/apex-feature-parity.md +50 -19
  10. package/docs/guide/01-installation.md +8 -4
  11. package/docs/guide/03-builder.md +197 -10
  12. package/docs/guide/04-pages-and-regions.md +71 -21
  13. package/docs/guide/05-items.md +7 -1
  14. package/docs/guide/06-processing.md +129 -9
  15. package/docs/guide/07-dynamic-actions.md +34 -0
  16. package/docs/guide/08-security.md +55 -0
  17. package/docs/guide/09-reference.md +5 -2
  18. package/docs/guide/11-from-apex.md +3 -1
  19. package/docs/guide/12-development.md +87 -16
  20. package/docs/guide/14-globalization.md +49 -1
  21. package/docs/guide/18-cli.md +21 -1
  22. package/docs/guide/19-rest-data-sources.md +12 -0
  23. package/docs/guide/20-ai-agents.md +12 -4
  24. package/examples/hr/README.md +2 -0
  25. package/examples/hr/hr_64_ai_agent.sql +97 -0
  26. package/examples/hr/hr_65_ask_reports.sql +21 -0
  27. package/examples/hr/hr_66_team_development.sql +58 -0
  28. package/examples/hr/hr_67_workflow_outcomes.sql +113 -0
  29. package/examples/hr/hr_68_component_details.sql +96 -0
  30. package/examples/hr/hr_70_data_security.sql +79 -0
  31. package/package.json +1 -1
  32. package/public/app.css +135 -21
  33. package/public/app.js +134 -20
  34. package/public/builder.css +133 -1
  35. package/public/builder.js +51 -0
  36. package/src/ai/chat.ts +18 -18
  37. package/src/ai/service.ts +35 -11
  38. package/src/ai/types.ts +7 -2
  39. package/src/app.ts +15 -0
  40. package/src/builder/advisor.ts +6 -3
  41. package/src/builder/ai-agents.ts +372 -0
  42. package/src/builder/ai-builder.ts +3 -3
  43. package/src/builder/ai.ts +13 -9
  44. package/src/builder/app-updates.ts +55 -0
  45. package/src/builder/assistant.ts +22 -7
  46. package/src/builder/components.ts +38 -9
  47. package/src/builder/data-security.ts +182 -0
  48. package/src/builder/designer.ts +8 -3
  49. package/src/builder/feedback.ts +6 -1
  50. package/src/builder/home.ts +16 -5
  51. package/src/builder/locks.ts +7 -1
  52. package/src/builder/mail-queue.ts +212 -0
  53. package/src/builder/mail.ts +72 -45
  54. package/src/builder/network-access.ts +140 -0
  55. package/src/builder/object-actions.ts +1 -0
  56. package/src/builder/object-browser.ts +1 -1
  57. package/src/builder/region-help.ts +2 -2
  58. package/src/builder/rest-catalogs.ts +19 -4
  59. package/src/builder/routes.ts +4 -1
  60. package/src/builder/shared.ts +1 -1
  61. package/src/builder/sql.ts +2 -0
  62. package/src/builder/team.ts +991 -0
  63. package/src/builder/themeroller.ts +47 -6
  64. package/src/builder/ui.ts +7 -1
  65. package/src/builder/websources.ts +1 -1
  66. package/src/builder/workflow-migrate.ts +210 -0
  67. package/src/builder/workflows.ts +29 -6
  68. package/src/builder/workspace-home.ts +2 -0
  69. package/src/builder/workspace-mail.ts +121 -0
  70. package/src/builder/workspace-transfer.ts +1 -1
  71. package/src/builder/workspaces.ts +1 -0
  72. package/src/cli/main.ts +25 -3
  73. package/src/cli/mcp.ts +40 -4
  74. package/src/cli/only.ts +52 -0
  75. package/src/cli/replace.ts +11 -1
  76. package/src/datasec.ts +343 -0
  77. package/src/i18n/ar.ts +17 -0
  78. package/src/i18n/cs.ts +17 -0
  79. package/src/i18n/da.ts +17 -0
  80. package/src/i18n/de.ts +17 -0
  81. package/src/i18n/el.ts +17 -0
  82. package/src/i18n/es.ts +17 -0
  83. package/src/i18n/fi.ts +17 -0
  84. package/src/i18n/fr.ts +17 -0
  85. package/src/i18n/he.ts +17 -0
  86. package/src/i18n/it.ts +17 -0
  87. package/src/i18n/ja.ts +17 -0
  88. package/src/i18n/ko.ts +17 -0
  89. package/src/i18n/nb.ts +17 -0
  90. package/src/i18n/pl.ts +17 -0
  91. package/src/i18n/pt.ts +17 -0
  92. package/src/i18n/ru.ts +17 -0
  93. package/src/i18n/sv.ts +17 -0
  94. package/src/i18n/tr.ts +17 -0
  95. package/src/i18n/uk.ts +17 -0
  96. package/src/i18n/zh.ts +17 -0
  97. package/src/i18n.ts +34 -0
  98. package/src/instance.ts +2 -0
  99. package/src/mail/mail.ts +50 -11
  100. package/src/mail/providers.ts +6 -3
  101. package/src/metadata.ts +11 -3
  102. package/src/network-acl.ts +126 -0
  103. package/src/objectstore.ts +3 -0
  104. package/src/runtime/ai-agents.ts +365 -0
  105. package/src/runtime/ai-filter.ts +385 -107
  106. package/src/runtime/assistant.ts +64 -11
  107. package/src/runtime/context.ts +2 -0
  108. package/src/runtime/engine.ts +3 -1
  109. package/src/runtime/ir-phrases.ts +812 -0
  110. package/src/runtime/ir-plan.ts +449 -0
  111. package/src/runtime/items.ts +1 -1
  112. package/src/runtime/regions.ts +6 -2
  113. package/src/runtime/render.ts +5 -3
  114. package/src/runtime/report.ts +1 -1
  115. package/src/runtime/styles.ts +54 -12
  116. package/src/runtime/tasks.ts +63 -12
  117. package/src/task-events.ts +232 -0
  118. package/src/team-samples.ts +74 -0
  119. package/src/team.ts +342 -0
  120. package/src/webclient.ts +45 -4
  121. package/src/webrequests.ts +5 -0
  122. package/src/websources.ts +30 -12
  123. package/src/workflow.ts +135 -18
  124. package/src/workspace-export.ts +16 -1
@@ -0,0 +1,812 @@
1
+ import type { Kind, RawPlan } from './ir-plan.ts';
2
+
3
+ // (sprint 112) "Ask in your own words" without an AI service (APEX 26.2: AI
4
+ // interactive reports work with zero AI). A small rule-based parser turns
5
+ // common phrases into the same answer an AI service gives (a RawPlan, checked
6
+ // by checkPlan in ir-plan.ts):
7
+ //
8
+ // salary over 3000 sort by hire date, newest first only clerks
9
+ // sum of salary by department where job is clerk bonus is 10% of salary
10
+ // highlight salary over 4000 in red pie chart of count by department start over
11
+ //
12
+ // The words live in phrase tables, one per language (English, Dutch and
13
+ // German here; another language adds a table to PHRASES). A question is cut
14
+ // into clauses (at commas and words such as "where", "and", "with"); each
15
+ // clause is matched against the patterns in turn. Columns are found by name
16
+ // or heading (also without spaces, and a verb such as "hired" for "hiredate");
17
+ // a bare value ("clerks", "in sales") is looked up among the report's own
18
+ // values through the lookup function (a query parameter, never SQL text).
19
+ // What isn't understood is reported, not guessed.
20
+
21
+ export interface PhraseColumn {
22
+ name: string;
23
+ label: string;
24
+ kind: Kind;
25
+ }
26
+
27
+ /** A stored value matching one of the candidates (lower case), in the given column or any text column. */
28
+ export type Lookup = (column: string | null, candidates: string[]) => Promise<{ column: string; value: string; candidate: string } | null>;
29
+
30
+ export interface Phrases {
31
+ /** words that start a new clause (besides , and ;) */
32
+ split: string[];
33
+ /** phrases a new clause starts at (they stay in it) */
34
+ starts: string[];
35
+ /** "between X and Y" */
36
+ between: [string, string];
37
+ /** words dropped at the start of a clause */
38
+ stop: string[];
39
+ reset: RegExp;
40
+ /** sort by <column> [direction]: group 1 is the rest */
41
+ sort: RegExp;
42
+ desc: string[];
43
+ asc: string[];
44
+ /** "highest salary first" (desc) / "lowest salary first" (asc): group 1 is the column */
45
+ firstDesc: RegExp;
46
+ firstAsc: RegExp;
47
+ /** highlight <condition> [in <color>]: group 1 is the rest */
48
+ highlight: RegExp;
49
+ colorIn: RegExp;
50
+ colors: Record<string, string>;
51
+ breakBy: RegExp;
52
+ groupBy: RegExp;
53
+ /** "by", "per", … between a measure and the group columns */
54
+ by: string[];
55
+ /** "and" between group or pivot columns */
56
+ and: string[];
57
+ pivot: RegExp;
58
+ /** (kind)? chart (of)? rest: group 1 the kind word (may be empty), group 2 the rest */
59
+ chart: RegExp;
60
+ chartKinds: Record<string, string>;
61
+ functions: [string, string][];
62
+ of: string[];
63
+ /** <name> is <expression>: group 1 the name, group 2 the expression */
64
+ compute: RegExp;
65
+ arith: [RegExp, string][];
66
+ percentOf: RegExp;
67
+ search: RegExp;
68
+ /** operator phrases (longest first is not needed: sorted by length) */
69
+ operators: [string, string][];
70
+ /** "is empty" and the like, after a column */
71
+ empty: [string, string][];
72
+ /** no <column> / without <column>: group 1 */
73
+ without: RegExp;
74
+ /** has <column>: group 1 */
75
+ having: RegExp;
76
+ /** only <values>: group 1 */
77
+ only: RegExp;
78
+ /** not <values> / except <values>: group 1 */
79
+ except: RegExp;
80
+ /** replacements before a question is cut into clauses ("highlight rows where" → "highlight") */
81
+ glue: [RegExp, string][];
82
+ yes: string[];
83
+ no: string[];
84
+ months: Record<string, number>;
85
+ thisYear: string[];
86
+ lastYear: string[];
87
+ /** decimal comma (3,5) and thousands dot (3.000) */
88
+ decimalComma: boolean;
89
+ thousand: string[];
90
+ /** singular forms of a plural word */
91
+ singular: (w: string) => string[];
92
+ }
93
+
94
+ const words = (s: string) => s.split(' ').filter(Boolean);
95
+ const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
96
+
97
+ const en: Phrases = {
98
+ split: ['where', 'whose', 'with', 'who', 'which', 'that', 'and', 'but', 'also', 'then', 'while'],
99
+ starts: ['sorted by', 'sort by', 'order by', 'ordered by', 'sorted on', 'highlight', 'group by', 'grouped by', 'control break', 'break by', 'break on', 'pivot', 'only', 'just', 'except', 'search for'],
100
+ between: ['between', 'and'],
101
+ stop: ['show', 'me', 'list', 'display', 'give', 'get', 'find', 'i', 'want', 'need', 'see', 'please', 'can', 'you', 'all', 'the', 'a', 'an', 'any', 'rows', 'records', 'entries', 'everyone', 'everybody', 'people', 'those', 'them', 'are', 'is', 'was', 'were', 'be', 'who', 'which', 'that', 'whose', 'add', 'make', 'let', 'us'],
102
+ reset: /^(?:reset|clear|clear all|clear everything|clear (?:all )?(?:the )?filters|start over|start again|remove (?:all|everything)|show everything|undo all)$/,
103
+ sort: /^(?:sort|sorted|order|ordered|sorting)\s+(?:it\s+|them\s+)?(?:by|on)\s+(.+)$/,
104
+ desc: ['descending', 'desc', 'high to low', 'highest first', 'largest first', 'biggest first', 'newest first', 'latest first', 'most recent first', 'last first', 'z to a', 'reversed', 'in reverse'],
105
+ asc: ['ascending', 'asc', 'low to high', 'lowest first', 'smallest first', 'oldest first', 'earliest first', 'first first', 'a to z'],
106
+ firstDesc: /^(?:the\s+)?(?:highest|largest|biggest|most|newest|latest|most recent|best paid|top)\s+(.+?)\s+(?:first|on top|at the top)$/,
107
+ firstAsc: /^(?:the\s+)?(?:lowest|smallest|least|oldest|earliest|first)\s+(.+?)\s+(?:first|on top|at the top)$/,
108
+ highlight: /^(?:highlight|mark|colou?r)\s+(?:the\s+)?(?:rows?\s+)?(?:where\s+|with\s+|whose\s+)?(.+)$/,
109
+ colorIn: /\s+(?:in|as|with)\s+(\S+)$/,
110
+ colors: { yellow: 'yellow', green: 'green', red: 'red', blue: 'blue', gray: 'gray', grey: 'gray' },
111
+ breakBy: /^(?:control\s+)?break\s+(?:by|on)\s+(.+)$/,
112
+ groupBy: /^(?:group|grouped)\s+(?:by|on|per)\s+(.+)$/,
113
+ by: ['grouped by', 'for each', 'for every', 'in each', 'by', 'per', 'across'],
114
+ and: ['and', '&', 'by', 'across', 'against', 'versus', 'vs', 'x'],
115
+ pivot: /^pivot(?:\s+table)?\s+(?:of\s+)?(.+)$/,
116
+ chart: /^(?:a\s+)?(?:(\S+)\s+)?(?:chart|graph)\s*(?:of\s+|with\s+|showing\s+)?(.*)$/,
117
+ chartKinds: { bar: 'bar', column: 'column', line: 'line', area: 'area', pie: 'pie', donut: 'donut', doughnut: 'donut' },
118
+ functions: [['sum', 'sum'], ['total', 'sum'], ['average', 'avg'], ['avg', 'avg'], ['mean', 'avg'], ['count', 'count'], ['number', 'count'], ['how many', 'count'], ['minimum', 'min'], ['min', 'min'], ['lowest', 'min'], ['smallest', 'min'], ['maximum', 'max'], ['max', 'max'], ['highest', 'max'], ['largest', 'max']],
119
+ of: ['of', 'the', 'all'],
120
+ compute: /^(?:(?:add|compute|calculate|create|make)\s+(?:a\s+)?(?:new\s+)?(?:column\s+|field\s+)?)?(?:column\s+)?([\p{L}][\p{L}\p{N}_ ]{0,39}?)\s+(?:is|=|as|equals|means)\s+(.+)$/u,
121
+ arith: [[/\b(?:times|multiplied by)\b/g, '*'], [/\bplus\b/g, '+'], [/\b(?:minus|less)\b/g, '-'], [/\bdivided by\b/g, '/'], [/\btwice\b/g, '2 *'], [/\bhalf of\b/g, '0.5 *'], [/\bhalf\b/g, '0.5 *']],
122
+ percentOf: /(\d+(?:\.\d+)?)\s*(?:%|percent|per cent)\s+of\b/g,
123
+ search: /^(?:search|look)\s+(?:for\s+)?(.+)$/,
124
+ operators: [
125
+ ['is not', 'ne'], ["isn't", 'ne'], ['not equal to', 'ne'], ['other than', 'ne'], ['not', 'ne'], ['!=', 'ne'], ['<>', 'ne'], ['≠', 'ne'],
126
+ ['is at least', 'ge'], ['at least', 'ge'], ['is greater than or equal to', 'ge'], ['>=', 'ge'], ['≥', 'ge'], ['from', 'ge'], ['since', 'ge'], ['starting', 'ge'],
127
+ ['is at most', 'le'], ['at most', 'le'], ['up to', 'le'], ['no more than', 'le'], ['<=', 'le'], ['≤', 'le'], ['until', 'le'], ['through', 'le'],
128
+ ['is over', 'gt'], ['over', 'gt'], ['above', 'gt'], ['is more than', 'gt'], ['more than', 'gt'], ['greater than', 'gt'], ['is greater than', 'gt'], ['higher than', 'gt'], ['bigger than', 'gt'], ['larger than', 'gt'], ['exceeds', 'gt'], ['after', 'gt'], ['later than', 'gt'], ['>', 'gt'],
129
+ ['is under', 'lt'], ['under', 'lt'], ['below', 'lt'], ['is less than', 'lt'], ['less than', 'lt'], ['lower than', 'lt'], ['smaller than', 'lt'], ['before', 'lt'], ['earlier than', 'lt'], ['<', 'lt'],
130
+ ['does not contain', 'not_contains'], ["doesn't contain", 'not_contains'], ['not containing', 'not_contains'], ['without the text', 'not_contains'],
131
+ ['contains', 'contains'], ['containing', 'contains'], ['includes', 'contains'], ['like', 'contains'], ['mentions', 'contains'], ['matching', 'contains'],
132
+ ['is in', 'in'], ['in', 'in'], ['during', 'in'], ['of', 'eq'],
133
+ ['is equal to', 'eq'], ['equal to', 'eq'], ['equals', 'eq'], ['is', 'eq'], ['==', 'eq'], ['=', 'eq'], [':', 'eq'],
134
+ ],
135
+ empty: [['is not empty', 'not_null'], ['is not blank', 'not_null'], ['is set', 'not_null'], ['is known', 'not_null'], ['is empty', 'null'], ['is blank', 'null'], ['is missing', 'null'], ['is unknown', 'null'], ['is null', 'null'], ['is not null', 'not_null']],
136
+ without: /^(?:no|without|missing|lacking|has no|with no|without a|without an)\s+(.+)$/,
137
+ having: /^(?:having|has|having a|has a|has an|with a|with an)\s+(.+)$/,
138
+ only: /^(?:only|just|all|show only|everyone in|in)\s+(.+)$/,
139
+ except: /^(?:not|except|excluding|apart from|other than|no)\s+(.+)$/,
140
+ glue: [[/\b(highlight|mark|colou?r)\s+(?:the\s+)?(?:rows?\s+)?(?:where|with|whose|that have)\s+/g, '$1 ']],
141
+ yes: ['yes', 'true', 'y', 'on'],
142
+ no: ['no', 'false', 'n', 'off'],
143
+ months: { january: 1, jan: 1, february: 2, feb: 2, march: 3, mar: 3, april: 4, apr: 4, may: 5, june: 6, jun: 6, july: 7, jul: 7, august: 8, aug: 8, september: 9, sep: 9, sept: 9, october: 10, oct: 10, november: 11, nov: 11, december: 12, dec: 12 },
144
+ thisYear: ['this year'],
145
+ lastYear: ['last year'],
146
+ decimalComma: false,
147
+ thousand: ['k', 'thousand'],
148
+ singular: (w) => {
149
+ const out: string[] = [];
150
+ if (/men$/.test(w)) out.push(w.replace(/men$/, 'man'));
151
+ if (/ies$/.test(w)) out.push(w.replace(/ies$/, 'y'));
152
+ if (/(ses|xes|ches|shes)$/.test(w)) out.push(w.slice(0, -2));
153
+ if (/s$/.test(w) && !/ss$/.test(w)) out.push(w.slice(0, -1));
154
+ return out;
155
+ },
156
+ };
157
+
158
+ const nl: Phrases = {
159
+ split: ['waar', 'waarvan', 'met', 'die', 'dat', 'en', 'maar', 'ook', 'dan'],
160
+ starts: ['gesorteerd op', 'sorteer op', 'sorteren op', 'orden op', 'markeer', 'groepeer op', 'gegroepeerd op', 'splits op', 'draaitabel', 'alleen', 'behalve', 'zoek naar'],
161
+ between: ['tussen', 'en'],
162
+ stop: ['toon', 'laat', 'zien', 'me', 'mij', 'geef', 'lijst', 'alle', 'de', 'het', 'een', 'graag', 'rijen', 'iedereen', 'mensen', 'zijn', 'is', 'die', 'dat', 'wie', 'voeg', 'toe', 'maak'],
163
+ reset: /^(?:reset|wis alles|alles wissen|wis (?:alle )?filters|begin opnieuw|opnieuw beginnen|toon alles|alles tonen)$/,
164
+ sort: /^(?:sorteer|gesorteerd|sorteren|orden|geordend)\s+(?:op|naar)\s+(.+)$/,
165
+ desc: ['aflopend', 'hoog naar laag', 'hoogste eerst', 'grootste eerst', 'nieuwste eerst', 'laatste eerst', 'z tot a'],
166
+ asc: ['oplopend', 'laag naar hoog', 'laagste eerst', 'kleinste eerst', 'oudste eerst', 'eerste eerst', 'a tot z'],
167
+ firstDesc: /^(?:de\s+|het\s+)?(?:hoogste|grootste|meeste|nieuwste|laatste|best betaalde)\s+(.+?)\s+(?:eerst|bovenaan)$/,
168
+ firstAsc: /^(?:de\s+|het\s+)?(?:laagste|kleinste|minste|oudste|eerste)\s+(.+?)\s+(?:eerst|bovenaan)$/,
169
+ highlight: /^(?:markeer|accentueer|kleur)\s+(?:de\s+)?(?:rijen\s+)?(?:waar\s+|met\s+|waarvan\s+)?(.+)$/,
170
+ colorIn: /\s+(?:in|als|met)\s+(\S+)$/,
171
+ colors: { geel: 'yellow', groen: 'green', rood: 'red', blauw: 'blue', grijs: 'gray' },
172
+ breakBy: /^(?:splits|onderbreek|breek)\s+(?:op|per)\s+(.+)$/,
173
+ groupBy: /^(?:groepeer|gegroepeerd|groeperen)\s+(?:op|per|naar)\s+(.+)$/,
174
+ by: ['gegroepeerd op', 'voor elke', 'voor iedere', 'per', 'op', 'naar'],
175
+ and: ['en', '&', 'per', 'op', 'tegen', 'x'],
176
+ pivot: /^(?:draaitabel|pivot)\s+(?:van\s+)?(.+)$/,
177
+ chart: /^(?:een\s+)?(?:(\S+?)\s*)?(?:grafiek|diagram)\s*(?:van\s+|met\s+)?(.*)$/,
178
+ chartKinds: { staaf: 'bar', staafdiagram: 'bar', kolom: 'column', lijn: 'line', vlak: 'area', taart: 'pie', cirkel: 'pie', donut: 'donut' },
179
+ functions: [['som', 'sum'], ['totaal', 'sum'], ['totale', 'sum'], ['gemiddelde', 'avg'], ['gemiddeld', 'avg'], ['aantal', 'count'], ['hoeveel', 'count'], ['minimum', 'min'], ['laagste', 'min'], ['kleinste', 'min'], ['maximum', 'max'], ['hoogste', 'max'], ['grootste', 'max']],
180
+ of: ['van', 'de', 'het', 'alle'],
181
+ compute: /^(?:(?:voeg|bereken|maak)\s+(?:een\s+)?(?:nieuwe\s+)?(?:kolom\s+)?)?(?:kolom\s+)?([\p{L}][\p{L}\p{N}_ ]{0,39}?)\s+(?:is|=|als|wordt)\s+(.+?)(?:\s+toe)?$/u,
182
+ arith: [[/\b(?:keer|maal)\b/g, '*'], [/\bplus\b/g, '+'], [/\bmin\b/g, '-'], [/\bgedeeld door\b/g, '/'], [/\bhet dubbele van\b/g, '2 *'], [/\bde helft van\b/g, '0.5 *']],
183
+ percentOf: /(\d+(?:\.\d+)?)\s*(?:%|procent)\s+van\b/g,
184
+ search: /^(?:zoek|zoeken)\s+(?:naar\s+|op\s+)?(.+)$/,
185
+ operators: [
186
+ ['is niet', 'ne'], ['niet gelijk aan', 'ne'], ['anders dan', 'ne'], ['niet', 'ne'], ['!=', 'ne'], ['<>', 'ne'], ['≠', 'ne'],
187
+ ['is minstens', 'ge'], ['minstens', 'ge'], ['ten minste', 'ge'], ['minimaal', 'ge'], ['vanaf', 'ge'], ['sinds', 'ge'], ['>=', 'ge'], ['≥', 'ge'],
188
+ ['is hoogstens', 'le'], ['hoogstens', 'le'], ['ten hoogste', 'le'], ['maximaal', 'le'], ['tot en met', 'le'], ['<=', 'le'], ['≤', 'le'],
189
+ ['boven', 'gt'], ['meer dan', 'gt'], ['groter dan', 'gt'], ['hoger dan', 'gt'], ['is meer dan', 'gt'], ['is groter dan', 'gt'], ['na', 'gt'], ['later dan', 'gt'], ['>', 'gt'],
190
+ ['onder', 'lt'], ['minder dan', 'lt'], ['kleiner dan', 'lt'], ['lager dan', 'lt'], ['is minder dan', 'lt'], ['voor', 'lt'], ['vóór', 'lt'], ['eerder dan', 'lt'], ['<', 'lt'],
191
+ ['bevat niet', 'not_contains'], ['bevat', 'contains'], ['met de tekst', 'contains'], ['lijkt op', 'contains'],
192
+ ['is in', 'in'], ['in', 'in'], ['tijdens', 'in'],
193
+ ['is gelijk aan', 'eq'], ['gelijk aan', 'eq'], ['is', 'eq'], ['=', 'eq'], [':', 'eq'],
194
+ ],
195
+ empty: [['is niet leeg', 'not_null'], ['is ingevuld', 'not_null'], ['is bekend', 'not_null'], ['is leeg', 'null'], ['ontbreekt', 'null'], ['is onbekend', 'null']],
196
+ without: /^(?:geen|zonder)\s+(.+)$/,
197
+ having: /^(?:heeft|hebben|met een)\s+(.+)$/,
198
+ only: /^(?:alleen|enkel|uitsluitend|iedereen in|in|bij)\s+(.+)$/,
199
+ except: /^(?:niet|behalve|zonder|geen)\s+(.+)$/,
200
+ glue: [[/\b(markeer|accentueer|kleur)\s+(?:de\s+)?(?:rijen\s+)?(?:waar|met|waarvan)\s+/g, '$1 ']],
201
+ yes: ['ja', 'waar', 'true'],
202
+ no: ['nee', 'onwaar', 'false'],
203
+ months: { januari: 1, jan: 1, februari: 2, feb: 2, maart: 3, mrt: 3, april: 4, apr: 4, mei: 5, juni: 6, jun: 6, juli: 7, jul: 7, augustus: 8, aug: 8, september: 9, sep: 9, sept: 9, oktober: 10, okt: 10, november: 11, nov: 11, december: 12, dec: 12 },
204
+ thisYear: ['dit jaar'],
205
+ lastYear: ['vorig jaar', 'afgelopen jaar'],
206
+ decimalComma: true,
207
+ thousand: ['k', 'duizend'],
208
+ singular: (w) => {
209
+ const out: string[] = [];
210
+ if (/'s$/.test(w)) out.push(w.slice(0, -2));
211
+ if (/s$/.test(w)) out.push(w.slice(0, -1));
212
+ if (/en$/.test(w)) out.push(w.slice(0, -2));
213
+ return out;
214
+ },
215
+ };
216
+
217
+ const de: Phrases = {
218
+ split: ['wo', 'wobei', 'mit', 'und', 'aber', 'auch', 'dann', 'deren', 'dessen'],
219
+ starts: ['sortiert nach', 'sortiere nach', 'sortieren nach', 'ordne nach', 'markiere', 'hebe', 'gruppiere nach', 'gruppiert nach', 'kontrollgruppe', 'pivot', 'nur', 'außer', 'suche nach'],
220
+ between: ['zwischen', 'und'],
221
+ stop: ['zeige', 'zeig', 'mir', 'liste', 'alle', 'gib', 'bitte', 'den', 'die', 'der', 'das', 'ein', 'eine', 'einen', 'zeilen', 'datensätze', 'jeder', 'leute', 'sind', 'ist', 'füge', 'hinzu', 'erstelle'],
222
+ reset: /^(?:zurücksetzen|alles zurücksetzen|alle filter entfernen|filter löschen|neu anfangen|von vorne|alles zeigen|alles anzeigen)$/,
223
+ sort: /^(?:sortiere|sortiert|sortieren|ordne|geordnet)\s+(?:nach|auf)\s+(.+)$/,
224
+ desc: ['absteigend', 'hoch nach niedrig', 'höchste zuerst', 'größte zuerst', 'neueste zuerst', 'z bis a'],
225
+ asc: ['aufsteigend', 'niedrig nach hoch', 'niedrigste zuerst', 'kleinste zuerst', 'älteste zuerst', 'a bis z'],
226
+ firstDesc: /^(?:das\s+|die\s+|der\s+)?(?:höchste|höchstes|höchsten|größte|größtes|meiste|neueste|neuesten|best bezahlte)\s+(.+?)\s+(?:zuerst|oben)$/,
227
+ firstAsc: /^(?:das\s+|die\s+|der\s+)?(?:niedrigste|niedrigstes|kleinste|wenigste|älteste|ältesten|erste)\s+(.+?)\s+(?:zuerst|oben)$/,
228
+ highlight: /^(?:markiere|hebe|färbe)\s+(?:die\s+)?(?:zeilen\s+)?(?:wo\s+|mit\s+|deren\s+)?(.+?)(?:\s+hervor)?$/,
229
+ colorIn: /\s+(?:in|als|mit)\s+(\S+)$/,
230
+ colors: { gelb: 'yellow', grün: 'green', gruen: 'green', rot: 'red', blau: 'blue', grau: 'gray' },
231
+ breakBy: /^(?:kontrollgruppe|gruppenwechsel|trenne)\s+(?:nach|auf|bei)\s+(.+)$/,
232
+ groupBy: /^(?:gruppiere|gruppiert|gruppieren)\s+(?:nach|pro)\s+(.+)$/,
233
+ by: ['gruppiert nach', 'für jede', 'für jeden', 'pro', 'je', 'nach'],
234
+ and: ['und', '&', 'nach', 'gegen', 'x'],
235
+ pivot: /^pivot(?:tabelle)?\s+(?:von\s+|aus\s+)?(.+)$/,
236
+ chart: /^(?:ein\s+|eine\s+)?(?:(\S+?)\s*)?(?:diagramm|grafik|chart)\s*(?:von\s+|mit\s+|über\s+)?(.*)$/,
237
+ chartKinds: { balken: 'bar', säulen: 'column', saeulen: 'column', linien: 'line', linie: 'line', flächen: 'area', kreis: 'pie', torten: 'pie', donut: 'donut', ring: 'donut' },
238
+ functions: [['summe', 'sum'], ['gesamt', 'sum'], ['durchschnitt', 'avg'], ['mittelwert', 'avg'], ['durchschnittliche', 'avg'], ['anzahl', 'count'], ['wie viele', 'count'], ['minimum', 'min'], ['niedrigste', 'min'], ['kleinste', 'min'], ['maximum', 'max'], ['höchste', 'max'], ['größte', 'max']],
239
+ of: ['von', 'der', 'des', 'die', 'das', 'aller'],
240
+ compute: /^(?:(?:füge|berechne|erstelle)\s+(?:eine\s+)?(?:neue\s+)?(?:spalte\s+)?)?(?:spalte\s+)?([\p{L}][\p{L}\p{N}_ ]{0,39}?)\s+(?:ist|=|als|sind)\s+(.+?)(?:\s+hinzu)?$/u,
241
+ arith: [[/\b(?:mal|multipliziert mit)\b/g, '*'], [/\bplus\b/g, '+'], [/\bminus\b/g, '-'], [/\bgeteilt durch\b/g, '/'], [/\bdas doppelte von\b/g, '2 *'], [/\bdie hälfte von\b/g, '0.5 *']],
242
+ percentOf: /(\d+(?:\.\d+)?)\s*(?:%|prozent)\s+(?:von|des|der)\b/g,
243
+ search: /^(?:suche|suchen)\s+(?:nach\s+)?(.+)$/,
244
+ operators: [
245
+ ['ist nicht', 'ne'], ['ungleich', 'ne'], ['nicht', 'ne'], ['!=', 'ne'], ['<>', 'ne'], ['≠', 'ne'],
246
+ ['ist mindestens', 'ge'], ['mindestens', 'ge'], ['ab', 'ge'], ['seit', 'ge'], ['>=', 'ge'], ['≥', 'ge'],
247
+ ['ist höchstens', 'le'], ['höchstens', 'le'], ['bis', 'le'], ['<=', 'le'], ['≤', 'le'],
248
+ ['über', 'gt'], ['mehr als', 'gt'], ['größer als', 'gt'], ['höher als', 'gt'], ['ist größer als', 'gt'], ['nach', 'gt'], ['später als', 'gt'], ['>', 'gt'],
249
+ ['unter', 'lt'], ['weniger als', 'lt'], ['kleiner als', 'lt'], ['niedriger als', 'lt'], ['ist kleiner als', 'lt'], ['vor', 'lt'], ['früher als', 'lt'], ['<', 'lt'],
250
+ ['enthält nicht', 'not_contains'], ['enthält', 'contains'], ['mit dem text', 'contains'], ['wie', 'contains'],
251
+ ['ist in', 'in'], ['in', 'in'], ['im', 'in'], ['während', 'in'],
252
+ ['ist gleich', 'eq'], ['gleich', 'eq'], ['ist', 'eq'], ['=', 'eq'], [':', 'eq'],
253
+ ],
254
+ empty: [['ist nicht leer', 'not_null'], ['ist gesetzt', 'not_null'], ['ist bekannt', 'not_null'], ['ist leer', 'null'], ['fehlt', 'null'], ['ist unbekannt', 'null']],
255
+ without: /^(?:kein|keine|keinen|ohne)\s+(.+)$/,
256
+ having: /^(?:hat|haben|mit einem|mit einer)\s+(.+)$/,
257
+ only: /^(?:nur|lediglich|ausschließlich|alle in|in|bei|im)\s+(.+)$/,
258
+ except: /^(?:nicht|außer|ausser|ohne|kein|keine)\s+(.+)$/,
259
+ glue: [[/\b(markiere|hebe|färbe)\s+(?:die\s+)?(?:zeilen\s+)?(?:wo|mit|deren)\s+/g, '$1 ']],
260
+ yes: ['ja', 'wahr', 'true'],
261
+ no: ['nein', 'falsch', 'false'],
262
+ months: { januar: 1, jan: 1, februar: 2, feb: 2, märz: 3, maerz: 3, mär: 3, april: 4, apr: 4, mai: 5, juni: 6, jun: 6, juli: 7, jul: 7, august: 8, aug: 8, september: 9, sep: 9, sept: 9, oktober: 10, okt: 10, november: 11, nov: 11, dezember: 12, dez: 12 },
263
+ thisYear: ['dieses jahr', 'in diesem jahr'],
264
+ lastYear: ['letztes jahr', 'im letzten jahr', 'vorjahr'],
265
+ decimalComma: true,
266
+ thousand: ['k', 'tausend'],
267
+ singular: (w) => {
268
+ const out: string[] = [];
269
+ if (/en$/.test(w)) out.push(w.slice(0, -2), w.slice(0, -1));
270
+ if (/[en]$/.test(w)) out.push(w.slice(0, -1));
271
+ if (/s$/.test(w)) out.push(w.slice(0, -1));
272
+ return out;
273
+ },
274
+ };
275
+
276
+ /** The phrase tables by language (base language code). English is always tried too. */
277
+ export const PHRASES: Record<string, Phrases> = { en, nl, de };
278
+
279
+ // ------------------------------------------------------------------ parsing
280
+
281
+ interface State {
282
+ P: Phrases;
283
+ columns: PhraseColumn[];
284
+ lookup: Lookup;
285
+ today: string;
286
+ plan: Required<Pick<RawPlan, 'filters' | 'highlights' | 'aggregates' | 'group_by' | 'group_functions' | 'computed_columns'>> & RawPlan;
287
+ left: string[];
288
+ understood: number;
289
+ /** the last column a condition was about ("salary over 1000 and under 3000") */
290
+ lastCol: PhraseColumn | null;
291
+ /** the previous clause set a group by (a bare column after it adds to it) */
292
+ lastGroup: boolean;
293
+ /** the previous clause was a sort (a bare direction after it applies to it) */
294
+ lastSort: boolean;
295
+ /** quoted text, put back when a value is read */
296
+ quoted: string[];
297
+ /** "pivot sum of salary by department" waiting for its pivot column ("and job" became the next clause) */
298
+ pendingPivot: { fn: string; value: string; row: string } | null;
299
+ }
300
+
301
+ const flat = (s: string) => s.replace(/[\s_]+/g, '');
302
+ const startsWithWord = (s: string, w: string) => s === w || s.startsWith(`${w} `) || (/^\W/.test(w) && s.startsWith(w));
303
+ const afterWord = (s: string, w: string) => s.slice(w.length).trim();
304
+
305
+ /** Column aliases: name, name with spaces, heading (all without spaces for comparing). */
306
+ function aliases(c: PhraseColumn) {
307
+ const label = c.label.toLowerCase().replace(/[^\p{L}\p{N} ]/gu, ' ');
308
+ return [...new Set([flat(c.name.toLowerCase()), flat(label)].filter(Boolean))];
309
+ }
310
+
311
+ /** The column a text starts with (longest match), and the rest. */
312
+ function matchColumn(st: State, s: string): { col: PhraseColumn; rest: string } | null {
313
+ const w = words(s);
314
+ for (let n = Math.min(4, w.length); n >= 1; n--) {
315
+ const phrase = flat(w.slice(0, n).join(' '));
316
+ const forms = [phrase, ...st.P.singular(phrase)];
317
+ const col = st.columns.find((c) => aliases(c).some((a) => forms.includes(a)));
318
+ if (col) return { col, rest: w.slice(n).join(' ') };
319
+ }
320
+ // a verb or a part of a name: "hired" → hiredate, "commission" → comm (one column only)
321
+ const first = w[0] ?? '';
322
+ if (first.length >= 4) {
323
+ const stems = [first, first.replace(/(ed|d|ing|s)$/, ''), first.replace(/(ed|ing)$/, '')].filter((x) => x.length >= 4);
324
+ const found = st.columns.filter((c) => aliases(c).some((a) => stems.some((x) => a.startsWith(x) || (a.length >= 4 && x.startsWith(a)))));
325
+ if (found.length === 1) return { col: found[0], rest: w.slice(1).join(' ') };
326
+ }
327
+ return null;
328
+ }
329
+
330
+ const stripStop = (st: State, s: string) => {
331
+ let w = words(s);
332
+ while (w.length > 1 && st.P.stop.includes(w[0])) w = w.slice(1);
333
+ return w.join(' ');
334
+ };
335
+
336
+ const unquote = (st: State, s: string) => s.replace(/\u0002(\d+)\u0002/g, (_, i) => st.quoted[Number(i)] ?? '').trim();
337
+
338
+ function parseNumber(st: State, s: string): { n: string; rest: string } | null {
339
+ const m = /^(?:[$€£]\s*)?(-?\d+(?:\.\d+)?)\s*([a-z]+)?\b(.*)$/.exec(s);
340
+ if (!m) return null;
341
+ let n = Number(m[1]);
342
+ let rest = `${m[2] ?? ''} ${m[3]}`.trim();
343
+ const unit = m[2] ?? '';
344
+ if (unit && st.P.thousand.includes(unit)) (n *= 1000), (rest = m[3].trim());
345
+ return { n: String(n), rest: rest.replace(/^(?:euro|eur|dollars?|usd)\b\s*/, '') };
346
+ }
347
+
348
+ const pad = (n: number) => String(n).padStart(2, '0');
349
+ const lastDay = (y: number, m: number) => new Date(Date.UTC(y, m, 0)).getUTCDate();
350
+
351
+ /** A date or period at the start of the text: its first and last day. */
352
+ function parseDate(st: State, s: string): { start: string; end: string; rest: string } | null {
353
+ const year = Number(st.today.slice(0, 4));
354
+ for (const [list, y] of [[st.P.thisYear, year], [st.P.lastYear, year - 1]] as [string[], number][])
355
+ for (const p of list) if (startsWithWord(s, p)) return { start: `${y}-01-01`, end: `${y}-12-31`, rest: afterWord(s, p) };
356
+ let m = /^(\d{4})-(\d{1,2})-(\d{1,2})\b(.*)$/.exec(s);
357
+ if (m) {
358
+ const d = `${m[1]}-${pad(+m[2])}-${pad(+m[3])}`;
359
+ return +m[2] >= 1 && +m[2] <= 12 && +m[3] >= 1 && +m[3] <= 31 ? { start: d, end: d, rest: m[4].trim() } : null;
360
+ }
361
+ const month = (w: string) => st.P.months[w.replace(/\.$/, '')];
362
+ // 1 june 1981 / june 1 1981 / june 1981 / 1981
363
+ m = /^(\d{1,2})(?:st|nd|rd|th|e|\.)?\s+(\p{L}+)\.?\s+(\d{4})\b(.*)$/u.exec(s);
364
+ if (m && month(m[2])) return { start: `${m[3]}-${pad(month(m[2]))}-${pad(+m[1])}`, end: `${m[3]}-${pad(month(m[2]))}-${pad(+m[1])}`, rest: m[4].trim() };
365
+ m = /^(\p{L}+)\.?\s+(\d{1,2})(?:st|nd|rd|th)?\s+(\d{4})\b(.*)$/u.exec(s);
366
+ if (m && month(m[1])) return { start: `${m[3]}-${pad(month(m[1]))}-${pad(+m[2])}`, end: `${m[3]}-${pad(month(m[1]))}-${pad(+m[2])}`, rest: m[4].trim() };
367
+ m = /^(\p{L}+)\.?\s+(\d{4})\b(.*)$/u.exec(s);
368
+ if (m && month(m[1])) {
369
+ const mm = month(m[1]);
370
+ return { start: `${m[2]}-${pad(mm)}-01`, end: `${m[2]}-${pad(mm)}-${pad(lastDay(+m[2], mm))}`, rest: m[3].trim() };
371
+ }
372
+ m = /^(1[89]\d\d|2\d\d\d)\b(.*)$/.exec(s);
373
+ if (m) return { start: `${m[1]}-01-01`, end: `${m[1]}-12-31`, rest: m[2].trim() };
374
+ return null;
375
+ }
376
+
377
+ type Filter = { column: string; operator: string; value: string };
378
+
379
+ /** Candidates for a value lookup: the first 1–5 words (and their singular forms), longest first. */
380
+ function candidates(st: State, s: string) {
381
+ const w = words(unquote(st, s));
382
+ const out: { text: string; n: number }[] = [];
383
+ for (let n = Math.min(5, w.length); n >= 1; n--) {
384
+ const text = w.slice(0, n).join(' ');
385
+ out.push({ text, n });
386
+ for (const x of st.P.singular(w[n - 1])) out.push({ text: [...w.slice(0, n - 1), x].join(' '), n });
387
+ }
388
+ return { out, w };
389
+ }
390
+
391
+ /** A value of a column at the start of the text: the conditions and the rest. */
392
+ async function valueCondition(st: State, col: PhraseColumn, op: string, s: string): Promise<{ filters: Filter[]; rest: string } | null> {
393
+ s = s.trim();
394
+ if (!s) return null;
395
+ if (col.kind === 'number') {
396
+ const n = parseNumber(st, s);
397
+ if (!n) return null;
398
+ if (op === 'in' || op === 'eq') return { filters: [{ column: col.name, operator: 'eq', value: n.n }], rest: n.rest };
399
+ return { filters: [{ column: col.name, operator: op, value: n.n }], rest: n.rest };
400
+ }
401
+ if (col.kind === 'date') {
402
+ const d = parseDate(st, s);
403
+ if (!d) return null;
404
+ const one = d.start === d.end;
405
+ const f = (operator: string, value: string): Filter => ({ column: col.name, operator, value });
406
+ switch (op) {
407
+ case 'gt': return { filters: [f('gt', d.end)], rest: d.rest };
408
+ case 'ge': return { filters: [f('ge', d.start)], rest: d.rest };
409
+ case 'lt': return { filters: [f('lt', d.start)], rest: d.rest };
410
+ case 'le': return { filters: [f('le', d.end)], rest: d.rest };
411
+ case 'ne': return one ? { filters: [f('ne', d.start)], rest: d.rest } : null;
412
+ default: return { filters: one ? [f('eq', d.start)] : [f('ge', d.start), f('le', d.end)], rest: d.rest };
413
+ }
414
+ }
415
+ if (col.kind === 'boolean') {
416
+ const w = words(s);
417
+ const yes = st.P.yes.includes(w[0]);
418
+ if (!yes && !st.P.no.includes(w[0])) return null;
419
+ return { filters: [{ column: col.name, operator: op === 'ne' ? 'ne' : 'eq', value: String(yes) }], rest: w.slice(1).join(' ') };
420
+ }
421
+ // text: "contains" takes the rest; otherwise a stored value (case as stored), else the words as a contains
422
+ if (op === 'contains' || op === 'not_contains') return { filters: [{ column: col.name, operator: op, value: unquote(st, s) }], rest: '' };
423
+ const { out, w } = candidates(st, s);
424
+ const found = await st.lookup(col.name, out.map((c) => c.text.toLowerCase()));
425
+ if (found) {
426
+ const n = out.find((c) => c.text.toLowerCase() === found.candidate)?.n ?? w.length;
427
+ const operator = op === 'ne' ? 'ne' : ['gt', 'ge', 'lt', 'le'].includes(op) ? op : 'eq';
428
+ return { filters: [{ column: col.name, operator, value: found.value }], rest: w.slice(n).join(' ') };
429
+ }
430
+ const value = unquote(st, s);
431
+ if (op === 'ne') return { filters: [{ column: col.name, operator: 'not_contains', value }], rest: '' };
432
+ if (['gt', 'ge', 'lt', 'le'].includes(op)) return { filters: [{ column: col.name, operator: op, value: w[0] }], rest: w.slice(1).join(' ') };
433
+ return { filters: [{ column: col.name, operator: 'contains', value }], rest: '' };
434
+ }
435
+
436
+ /** A condition on a column: "<op> <value>", "between a and b", "is empty", or a bare value. */
437
+ async function condition(st: State, col: PhraseColumn, s: string): Promise<{ filters: Filter[]; rest: string } | null> {
438
+ s = s.trim();
439
+ const P = st.P;
440
+ for (const [phrase, op] of [...P.empty].sort((a, b) => b[0].length - a[0].length))
441
+ if (startsWithWord(s, phrase)) return { filters: [{ column: col.name, operator: op, value: '' }], rest: afterWord(s, phrase) };
442
+ const [bw, band] = P.between;
443
+ if (startsWithWord(s, bw)) {
444
+ const m = new RegExp(`^${esc(bw)}\\s+(.+?)\\s+${esc(band)}\\s+(.+)$`).exec(s);
445
+ if (!m) return null;
446
+ const a = await valueCondition(st, col, 'ge', m[1]);
447
+ const b = await valueCondition(st, col, 'le', m[2]);
448
+ if (!a || !b || a.rest) return null;
449
+ return { filters: [...a.filters.slice(0, 1), ...b.filters.slice(-1)], rest: b.rest };
450
+ }
451
+ if (!s) return col.kind === 'boolean' ? { filters: [{ column: col.name, operator: 'eq', value: 'true' }], rest: '' } : null;
452
+ for (const [phrase, op] of [...P.operators].sort((a, b) => b[0].length - a[0].length))
453
+ if (startsWithWord(s, phrase)) {
454
+ const r = await valueCondition(st, col, op, afterWord(s, phrase));
455
+ if (r) return r;
456
+ }
457
+ return valueCondition(st, col, 'eq', s);
458
+ }
459
+
460
+ /** A stored value of any shown text column at the start of the text ("clerks", "sales"). */
461
+ async function bareValue(st: State, s: string, negate: boolean): Promise<{ filters: Filter[]; rest: string } | null> {
462
+ const { out, w } = candidates(st, s);
463
+ if (!out.length) return null;
464
+ const found = await st.lookup(null, out.map((c) => c.text.toLowerCase()));
465
+ if (!found) return null;
466
+ const n = out.find((c) => c.text.toLowerCase() === found.candidate)?.n ?? w.length;
467
+ return { filters: [{ column: found.column, operator: negate ? 'ne' : 'eq', value: found.value }], rest: w.slice(n).join(' ') };
468
+ }
469
+
470
+ /** Function phrase at the start: the function and the rest (after "of", "the"). */
471
+ function matchFunction(st: State, s: string) {
472
+ for (const [phrase, fn] of [...st.P.functions].sort((a, b) => b[0].length - a[0].length))
473
+ if (startsWithWord(s, phrase)) {
474
+ let rest = words(afterWord(s, phrase));
475
+ while (rest.length && st.P.of.includes(rest[0])) rest = rest.slice(1);
476
+ return { fn, rest: rest.join(' ') };
477
+ }
478
+ return null;
479
+ }
480
+
481
+ /** "<left> by <right>": the first group phrase that splits the text. */
482
+ function splitBy(st: State, s: string, phrases = st.P.by): [string, string] | null {
483
+ for (const p of [...phrases].sort((a, b) => b.length - a.length)) {
484
+ const i = ` ${s} `.indexOf(` ${p} `);
485
+ if (i >= 0) return [s.slice(0, Math.max(0, i - 1)).trim(), s.slice(i + p.length).trim()];
486
+ }
487
+ return null;
488
+ }
489
+
490
+ /** Columns separated by "and", "," or the like; null unless all are columns. */
491
+ function columnList(st: State, s: string): PhraseColumn[] | null {
492
+ const out: PhraseColumn[] = [];
493
+ let rest = s.trim();
494
+ while (rest) {
495
+ const m = matchColumn(st, stripStop(st, rest));
496
+ if (!m) return null;
497
+ out.push(m.col);
498
+ rest = m.rest;
499
+ const sep = st.P.and.find((a) => startsWithWord(rest, a));
500
+ if (sep) rest = afterWord(rest, sep);
501
+ else if (rest) return null;
502
+ }
503
+ return out.length ? out : null;
504
+ }
505
+
506
+ /** A measure: "<fn> of <column>", "<column>" (sum or count), or nothing (count of the first column). */
507
+ function measure(st: State, s: string): { fn: string; column: string } | null {
508
+ const first = st.columns[0];
509
+ s = stripStop(st, s);
510
+ const f = matchFunction(st, s);
511
+ if (f) {
512
+ const m = f.rest ? matchColumn(st, f.rest) : null;
513
+ if (m && !m.rest) return { fn: f.fn, column: m.col.name };
514
+ return f.fn === 'count' && first ? { fn: 'count', column: first.name } : null;
515
+ }
516
+ if (!s) return first ? { fn: 'count', column: first.name } : null;
517
+ const m = matchColumn(st, s);
518
+ if (m && !m.rest) return { fn: m.col.kind === 'number' ? 'sum' : 'count', column: m.col.name };
519
+ return null;
520
+ }
521
+
522
+ /** An expression in words ("10% of salary", "salary times 12") as a computed column expression, or null. */
523
+ function expression(st: State, s: string): string | null {
524
+ let e = unquote(st, s).replace(st.P.percentOf, (_, n) => `${Number(n) / 100} * `);
525
+ for (const [re, op] of st.P.arith) e = e.replace(re, ` ${op} `);
526
+ // words → column names; numbers, operators, parentheses and the compute functions stay
527
+ const tokens = e.match(/"[^"]*"|'[^']*'|\d+(?:\.\d+)?|\|\||[-+*/(),]|[\p{L}_][\p{L}\p{N}_$]*/gu) ?? [];
528
+ const out: string[] = [];
529
+ let usesColumn = false;
530
+ for (let i = 0; i < tokens.length; i++) {
531
+ const tk = tokens[i];
532
+ if (!/^[\p{L}_]/u.test(tk)) {
533
+ out.push(tk);
534
+ continue;
535
+ }
536
+ if (tokens[i + 1] === '(') {
537
+ out.push(tk);
538
+ continue;
539
+ }
540
+ if (st.P.of.includes(tk)) continue;
541
+ // the longest run of words that is a column
542
+ let j = i;
543
+ while (j + 1 < tokens.length && /^[\p{L}_]/u.test(tokens[j + 1]) && j - i < 3) j++;
544
+ let done = false;
545
+ for (let k = j; k >= i; k--) {
546
+ const m = matchColumn(st, tokens.slice(i, k + 1).join(' '));
547
+ if (m && !m.rest) {
548
+ out.push(/^[a-z_][a-z0-9_$]*$/.test(m.col.name) ? m.col.name : `"${m.col.name.replace(/"/g, '')}"`);
549
+ usesColumn = true;
550
+ i = k;
551
+ done = true;
552
+ break;
553
+ }
554
+ }
555
+ if (!done) return null;
556
+ }
557
+ return usesColumn && out.length ? out.join(' ').replace(/\( /g, '(').replace(/ \)/g, ')').replace(/ ,/g, ',') : null;
558
+ }
559
+
560
+ /** The rest of a clause after something understood: parsed as a clause of its own. */
561
+ async function more(st: State, rest: string) {
562
+ if (!(await clause(st, rest))) st.left.push(unquote(st, rest));
563
+ return true;
564
+ }
565
+
566
+ /** One clause; true when it was understood. */
567
+ async function clause(st: State, raw: string): Promise<boolean> {
568
+ const P = st.P;
569
+ const plan = st.plan;
570
+ let s = stripStop(st, raw.trim());
571
+ if (!s) return true;
572
+ const wasGroup = st.lastGroup;
573
+ const pivot = st.pendingPivot;
574
+ st.pendingPivot = null;
575
+ if (pivot) {
576
+ const c = matchColumn(st, s);
577
+ if (c && !c.rest && c.col.name !== pivot.row) return (plan.pivot = { row: pivot.row, column: c.col.name, function: pivot.fn, value: pivot.value }), true;
578
+ st.left.push(`pivot ${pivot.row}`);
579
+ }
580
+ const wasSort = st.lastSort;
581
+ st.lastGroup = false;
582
+ st.lastSort = false;
583
+ if (P.reset.test(s)) return (plan.reset = true);
584
+
585
+ // a direction after a sort ("sort by hire date, newest first")
586
+ if (wasSort && [...P.desc, ...P.asc].includes(s)) return (plan.sort_descending = P.desc.includes(s)), (st.lastSort = true);
587
+ let m: RegExpExecArray | null;
588
+ if ((m = P.sort.exec(s))) {
589
+ const c = matchColumn(st, stripStop(st, m[1]));
590
+ if (!c) return false;
591
+ plan.sort_column = c.col.name;
592
+ plan.sort_descending = P.desc.includes(c.rest);
593
+ if (c.rest && !P.desc.includes(c.rest) && !P.asc.includes(c.rest)) return more(st, c.rest);
594
+ return (st.lastSort = true);
595
+ }
596
+ if ((m = P.firstDesc.exec(s)) || (m = P.firstAsc.exec(s))) {
597
+ const c = matchColumn(st, stripStop(st, m[1]));
598
+ if (c && !c.rest) {
599
+ plan.sort_column = c.col.name;
600
+ plan.sort_descending = P.firstDesc.test(s);
601
+ return true;
602
+ }
603
+ }
604
+ if ((m = P.search.exec(s))) {
605
+ plan.search = unquote(st, m[1]);
606
+ return true;
607
+ }
608
+ if ((m = P.highlight.exec(s))) {
609
+ let rest = m[1];
610
+ let color = 'yellow';
611
+ const cm = P.colorIn.exec(rest);
612
+ if (cm && P.colors[cm[1]]) (color = P.colors[cm[1]]), (rest = rest.slice(0, cm.index));
613
+ const w = words(rest);
614
+ if (P.colors[w[0]]) (color = P.colors[w[0]]), (rest = w.slice(1).join(' '));
615
+ const c = matchColumn(st, stripStop(st, rest));
616
+ const r = c ? await condition(st, c.col, c.rest) : await bareValue(st, rest, false);
617
+ if (!r || r.rest) return false;
618
+ for (const f of r.filters) plan.highlights.push({ ...f, color });
619
+ return true;
620
+ }
621
+ if ((m = P.breakBy.exec(s))) {
622
+ const c = matchColumn(st, stripStop(st, m[1]));
623
+ if (!c || c.rest) return false;
624
+ plan.control_break = c.col.name;
625
+ return true;
626
+ }
627
+ if ((m = P.groupBy.exec(s))) {
628
+ const cols = columnList(st, m[1]);
629
+ if (!cols) return false;
630
+ plan.group_by.push(...cols.map((c) => c.name));
631
+ return (st.lastGroup = true);
632
+ }
633
+ if ((m = P.pivot.exec(s))) {
634
+ const by = splitBy(st, m[1]);
635
+ if (!by) return false;
636
+ // "pivot sum of salary by department and job" or "pivot department by job"
637
+ const left = measure(st, by[0]);
638
+ const cols = columnList(st, by[1]);
639
+ const asRow = matchColumn(st, by[0]);
640
+ if (left && cols?.length === 2 && matchFunction(st, by[0])) {
641
+ plan.pivot = { row: cols[0].name, column: cols[1].name, function: left.fn, value: left.column };
642
+ return true;
643
+ }
644
+ if (left && cols?.length === 1 && matchFunction(st, by[0])) {
645
+ st.pendingPivot = { fn: left.fn, value: left.column, row: cols[0].name };
646
+ return true;
647
+ }
648
+ if (asRow && !asRow.rest && cols?.length === 1 && st.columns[0]) {
649
+ plan.pivot = { row: asRow.col.name, column: cols[0].name, function: 'count', value: st.columns[0].name };
650
+ return true;
651
+ }
652
+ return false;
653
+ }
654
+ if ((m = P.chart.exec(s)) && (!m[1] || P.chartKinds[m[1]] || P.of.includes(m[1]) || P.stop.includes(m[1]))) {
655
+ const kind = (m[1] && P.chartKinds[m[1]]) || 'bar';
656
+ const by = splitBy(st, m[2]);
657
+ if (!by) return false;
658
+ const label = matchColumn(st, stripStop(st, by[1]));
659
+ const v = measure(st, by[0]);
660
+ if (!label || label.rest || !v) return false;
661
+ plan.chart = { type: kind, label: label.col.name, function: v.fn, value: v.column };
662
+ return true;
663
+ }
664
+ // an aggregate: "sum of salary by department" (group by), "total salary" (a totals row)
665
+ const fm = matchFunction(st, s);
666
+ if (fm) {
667
+ const by = splitBy(st, fm.rest);
668
+ if (by) {
669
+ const v = by[0] ? matchColumn(st, stripStop(st, by[0])) : null;
670
+ const first = st.columns[0];
671
+ const fn = v && !v.rest ? { fn: fm.fn, column: v.col.name } : fm.fn === 'count' && first ? { fn: 'count', column: first.name } : null;
672
+ const cols = columnList(st, by[1]);
673
+ if (cols && fn) {
674
+ plan.group_by.push(...cols.map((c) => c.name));
675
+ plan.group_functions.push({ function: fn.fn, column: fn.column });
676
+ return (st.lastGroup = true);
677
+ }
678
+ } else {
679
+ const c = fm.rest ? matchColumn(st, fm.rest) : null;
680
+ if (c && !c.rest) {
681
+ plan.aggregates.push({ function: fm.fn, column: c.col.name });
682
+ return true;
683
+ }
684
+ }
685
+ }
686
+ // a computed column: "bonus is 10% of salary" (the name is not a column)
687
+ if ((m = P.compute.exec(s)) && !matchColumn(st, m[1])) {
688
+ const expr = expression(st, m[2]);
689
+ if (expr) {
690
+ plan.computed_columns.push({ name: m[1].trim(), expression: expr });
691
+ return true;
692
+ }
693
+ }
694
+ // no <column> / has <column>
695
+ if ((m = P.without.exec(s))) {
696
+ const c = matchColumn(st, stripStop(st, m[1]));
697
+ if (c && !c.rest) return plan.filters.push({ column: c.col.name, operator: 'null', value: '' }), true;
698
+ }
699
+ if ((m = P.having.exec(s))) {
700
+ const c = matchColumn(st, stripStop(st, m[1]));
701
+ if (c && !c.rest) return plan.filters.push({ column: c.col.name, operator: 'not_null', value: '' }), true;
702
+ }
703
+ // a column and a condition
704
+ const c = matchColumn(st, s);
705
+ if (c) {
706
+ // a bare column after a group by adds to it ("by department and job")
707
+ if (!c.rest && wasGroup) return plan.group_by.push(c.col.name), (st.lastGroup = true);
708
+ const r = await condition(st, c.col, c.rest);
709
+ if (r) {
710
+ st.lastCol = c.col;
711
+ plan.filters.push(...r.filters);
712
+ return r.rest ? more(st, r.rest) : true;
713
+ }
714
+ }
715
+ // a condition on the column before ("salary over 1000 and under 3000")
716
+ if (st.lastCol && P.operators.some(([p]) => startsWithWord(s, p))) {
717
+ const r = await condition(st, st.lastCol, s);
718
+ if (r) {
719
+ plan.filters.push(...r.filters);
720
+ return r.rest ? more(st, r.rest) : true;
721
+ }
722
+ }
723
+ // values: "only clerks", "not clerks", "clerks", "in sales"
724
+ const neg = P.except.exec(s);
725
+ const only = P.only.exec(s);
726
+ if (only || neg) {
727
+ const b = matchColumn(st, stripStop(st, (only ?? neg)![1]));
728
+ if (b && !b.rest && b.col.kind === 'boolean') return plan.filters.push({ column: b.col.name, operator: 'eq', value: String(!neg) }), true;
729
+ }
730
+ const r = neg ? await bareValue(st, neg[1], true) : await bareValue(st, only ? only[1] : s, false);
731
+ if (r) {
732
+ plan.filters.push(...r.filters);
733
+ return r.rest ? more(st, r.rest) : true;
734
+ }
735
+ return false;
736
+ }
737
+
738
+ /** Cut a question into clauses (outside "between … and …"). */
739
+ function clauses(st: State, q: string) {
740
+ const P = st.P;
741
+ let s = q;
742
+ for (const [re, to] of P.glue) s = s.replace(re, to);
743
+ // between a and b: protect the "and"
744
+ const [bw, band] = P.between;
745
+ s = s.replace(new RegExp(`\\b${esc(bw)}\\s+(\\S+(?:\\s+\\S+){0,2}?)\\s+${esc(band)}\\s+`, 'gu'), `${bw} $1 \u0001 `);
746
+ for (const p of [...P.starts].sort((a, b) => b.length - a.length)) s = s.replace(new RegExp(`(^|\\s)(${esc(p)})(?=\\s|$)`, 'gu'), '$1; $2');
747
+ const sep = new RegExp(`\\s+(?:${P.split.map(esc).join('|')})\\s+`, 'gu');
748
+ return s
749
+ .split(/[;,]/)
750
+ .flatMap((x) => ` ${x} `.split(sep))
751
+ .map((x) => x.replace(/\u0001/g, band).trim())
752
+ .filter(Boolean);
753
+ }
754
+
755
+ /** Normalise a question for one language: lower case, quotes kept aside, numbers without separators. */
756
+ function normalise(P: Phrases, question: string, quoted: string[]) {
757
+ let s = question.replace(/["“”]([^"“”]{1,200})["“”]/g, (_, x) => `\u0002${quoted.push(x) - 1}\u0002`).toLowerCase();
758
+ s = s.replace(/[‘’]/g, "'").replace(/[?!]+/g, ' ').replace(/\.(\s|$)/g, '$1');
759
+ s = s.replace(/(\d{1,2}),\s*(\d{4})\b/g, '$1 $2');
760
+ s = P.decimalComma ? s.replace(/(\d)\.(\d{3})\b/g, '$1$2').replace(/(\d),(\d)/g, '$1.$2') : s.replace(/(\d),(\d{3})\b/g, '$1$2');
761
+ // "salary>3000" → "salary > 3000"
762
+ s = s.replace(/\s*(>=|<=|!=|<>|==|[<>=≠≥≤])\s*/g, ' $1 ');
763
+ return s.replace(/\s+/g, ' ').trim();
764
+ }
765
+
766
+ export interface ParseResult {
767
+ plan: RawPlan;
768
+ /** the clauses not understood */
769
+ left: string[];
770
+ understood: number;
771
+ }
772
+
773
+ async function parseWith(P: Phrases, question: string, columns: PhraseColumn[], lookup: Lookup, today: string): Promise<ParseResult> {
774
+ const st: State = {
775
+ P, columns, lookup, today, quoted: [], left: [], understood: 0, lastCol: null, lastGroup: false, lastSort: false, pendingPivot: null,
776
+ plan: { filters: [], highlights: [], aggregates: [], group_by: [], group_functions: [], computed_columns: [] },
777
+ };
778
+ const s = normalise(P, question, st.quoted);
779
+ for (const c of clauses(st, s)) {
780
+ if (await clause(st, c)) st.understood++;
781
+ else st.left.push(unquote(st, c));
782
+ }
783
+ if (st.pendingPivot) st.left.push(`pivot ${st.pendingPivot.row}`);
784
+ // two values of one column can't both hold (filters are and-ed): keep the first, say so
785
+ const seen = new Set<string>();
786
+ st.plan.filters = st.plan.filters.filter((f) => {
787
+ if (f.operator !== 'eq') return true;
788
+ if (seen.has(f.column)) {
789
+ st.left.push(f.value);
790
+ return false;
791
+ }
792
+ seen.add(f.column);
793
+ return true;
794
+ });
795
+ return { plan: st.plan, left: st.left, understood: st.understood };
796
+ }
797
+
798
+ /**
799
+ * Parse a question with the phrases of the user's language (when pgkiln has
800
+ * them) and English, and keep the reading that understood most.
801
+ */
802
+ export async function parseQuestion(question: string, columns: PhraseColumn[], lang: string, lookup: Lookup, today = new Date().toISOString().slice(0, 10)): Promise<ParseResult> {
803
+ const base = lang.toLowerCase().split('-')[0];
804
+ const tables = [...new Set([base, 'en'])].filter((l) => PHRASES[l]).map((l) => PHRASES[l]);
805
+ let best: ParseResult | null = null;
806
+ for (const P of tables) {
807
+ const r = await parseWith(P, question.slice(0, 500), columns, lookup, today);
808
+ if (!best || r.understood - r.left.length > best.understood - best.left.length) best = r;
809
+ if (!r.left.length && r.understood) break;
810
+ }
811
+ return best!;
812
+ }