typia 14.0.6 → 15.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/lib/internal/_createLlmEvaluation.d.ts +1 -0
- package/lib/internal/_createLlmEvaluation.js +486 -0
- package/lib/internal/_createLlmEvaluation.js.map +1 -0
- package/lib/internal/_createLlmEvaluation.mjs +382 -0
- package/lib/internal/_createLlmEvaluation.mjs.map +1 -0
- package/lib/internal/_stringLengthGte.d.ts +1 -0
- package/lib/internal/_stringLengthGte.js +14 -0
- package/lib/internal/_stringLengthGte.js.map +1 -0
- package/lib/internal/_stringLengthGte.mjs +11 -0
- package/lib/internal/_stringLengthGte.mjs.map +1 -0
- package/lib/internal/_stringLengthLte.d.ts +1 -0
- package/lib/internal/_stringLengthLte.js +14 -0
- package/lib/internal/_stringLengthLte.js.map +1 -0
- package/lib/internal/_stringLengthLte.mjs +11 -0
- package/lib/internal/_stringLengthLte.mjs.map +1 -0
- package/lib/llm.d.ts +54 -1
- package/lib/llm.js +5 -0
- package/lib/llm.js.map +1 -1
- package/lib/llm.mjs +6 -1
- package/lib/llm.mjs.map +1 -1
- package/lib/re-exports.d.ts +1 -1
- package/lib/re-exports.js.map +1 -1
- package/native/cmd/ttsc-typia/build.go +2 -0
- package/native/cmd/ttsc-typia/dynamic_key_tags_transform_test.go +1 -1
- package/native/cmd/ttsc-typia/llm_evaluation_accepts_property_probability_test.go +27 -0
- package/native/cmd/ttsc-typia/llm_evaluation_accepts_unselected_source_probability_test.go +26 -0
- package/native/cmd/ttsc-typia/llm_evaluation_complete_object_shape_test.go +96 -0
- package/native/cmd/ttsc-typia/llm_evaluation_ignores_declaration_probability_without_call_test.go +18 -0
- package/native/cmd/ttsc-typia/llm_evaluation_plan_emission_transform_test.go +90 -0
- package/native/cmd/ttsc-typia/llm_evaluation_program_scan_dependencies_test.go +69 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_alias_comment_test.go +44 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_alias_refinement_test.go +45 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_computed_method_probability_test.go +30 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_imported_declaration_probability_test.go +48 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_invalid_probability_test.go +199 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_refinements_test.go +94 -0
- package/native/cmd/ttsc-typia/llm_evaluation_rejects_undecidable_types_test.go +232 -0
- package/native/cmd/ttsc-typia/metadata_type_tag_sole_malformed_diagnostic_test.go +94 -0
- package/native/cmd/ttsc-typia/template_literal_type_tags_transform_test.go +3 -2
- package/native/cmd/ttsc-typia/transform.go +2 -0
- package/native/cmd/ttsc-typia/transform_helpers_test.go +17 -3
- package/native/core/context/ITypiaContext.go +4 -0
- package/native/core/factories/IdentifierFactory.go +10 -0
- package/native/core/factories/LiteralFactory.go +16 -0
- package/native/core/factories/MetadataCommentTagFactory.go +12 -10
- package/native/core/factories/MetadataTypeTagFactory.go +97 -24
- package/native/core/factories/MetadataTypeTagSchemaFactory.go +5 -0
- package/native/core/factories/internal/metadata/IMetadataIteratorProps.go +6 -2
- package/native/core/factories/internal/metadata/emplace_metadata_object.go +49 -15
- package/native/core/factories/literal_factory_null_survives_objects_test.go +43 -0
- package/native/core/factories/metadata_comment_tag_factory_string_length_helpers_test.go +55 -0
- package/native/core/factories/metadata_type_tag_factory_reports_all_invalid_candidates_test.go +73 -0
- package/native/core/programmers/llm/LlmApplicationProgrammer.go +1 -0
- package/native/core/programmers/llm/LlmEvaluationProgrammer.go +555 -0
- package/native/core/programmers/llm/LlmParametersProgrammer.go +6 -5
- package/native/core/programmers/llm/LlmReferenceDescription.go +113 -0
- package/native/core/programmers/llm/llm_reference_description_test.go +134 -0
- package/native/core/schemas/metadata/MetadataDependency.go +9 -0
- package/native/transform/CallExpressionTransformer.go +1 -0
- package/native/transform/FileTransformer.go +5 -0
- package/native/transform/features/llm/LlmEvaluationProbabilityDeclarations.go +94 -0
- package/native/transform/features/llm/LlmEvaluationProbabilityDeclarations_test.go +50 -0
- package/native/transform/features/llm/LlmEvaluationTransformer.go +59 -0
- package/native/transform/features/llm/LlmSchemaTransformer.go +14 -11
- package/native/transform/features/llm/llm_transformer_entry_coverage_test.go +29 -27
- package/package.json +3 -3
- package/src/internal/_createLlmEvaluation.ts +678 -0
- package/src/internal/_stringLengthGte.ts +6 -0
- package/src/internal/_stringLengthLte.ts +6 -0
- package/src/llm.ts +64 -0
- package/src/re-exports.ts +1 -0
package/native/core/factories/metadata_type_tag_factory_reports_all_invalid_candidates_test.go
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
package factories
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"testing"
|
|
5
|
+
|
|
6
|
+
schemametadata "github.com/samchon/typia/packages/typia/native/core/schemas/metadata"
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
// TestMetadataTypeTagFactoryReportsAllInvalidCandidates verifies a rejected tag
|
|
10
|
+
// still produces a transform diagnostic when no valid tag candidate accompanies
|
|
11
|
+
// it.
|
|
12
|
+
//
|
|
13
|
+
// Analyze used to return as soon as its filtered candidate list was empty. That
|
|
14
|
+
// discarded the messages collected while rejecting the candidates, so a lone
|
|
15
|
+
// malformed tag silently became an ordinary phantom brand while the same tag
|
|
16
|
+
// beside a valid one failed compilation.
|
|
17
|
+
//
|
|
18
|
+
// 1. Build one structurally recognizable tag with a non-literal value.
|
|
19
|
+
// 2. Analyze it without a valid companion and require no emitted metadata tag.
|
|
20
|
+
// 3. Require the collected validation message to reach the factory errors.
|
|
21
|
+
func TestMetadataTypeTagFactoryReportsAllInvalidCandidates(t *testing.T) {
|
|
22
|
+
invalid := metadataTypeTagFactoryAllInvalidContainer("InvalidValue",
|
|
23
|
+
metadataTypeTagFactoryAllInvalidProperty("target", MetadataFactory.SoleLiteral("array")),
|
|
24
|
+
metadataTypeTagFactoryAllInvalidProperty("kind", MetadataFactory.SoleLiteral("default")),
|
|
25
|
+
metadataTypeTagFactoryAllInvalidProperty("value", func() *schemametadata.MetadataSchema {
|
|
26
|
+
metadata := schemametadata.MetadataSchema_initialize()
|
|
27
|
+
metadata.Atomics = append(metadata.Atomics, schemametadata.MetadataAtomic_create(schemametadata.MetadataAtomic{Type: "string"}))
|
|
28
|
+
return metadata
|
|
29
|
+
}()),
|
|
30
|
+
)
|
|
31
|
+
errors := []MetadataFactory_IError{}
|
|
32
|
+
tags := MetadataTypeTagFactory.Analyze(struct {
|
|
33
|
+
Errors *[]MetadataFactory_IError
|
|
34
|
+
Type string
|
|
35
|
+
Objects []*schemametadata.MetadataObjectType
|
|
36
|
+
Explore MetadataFactory_IExplore
|
|
37
|
+
}{
|
|
38
|
+
Errors: &errors,
|
|
39
|
+
Type: "array",
|
|
40
|
+
Objects: []*schemametadata.MetadataObjectType{invalid},
|
|
41
|
+
})
|
|
42
|
+
if len(tags) != 0 {
|
|
43
|
+
t.Fatalf("an invalid value must not produce a metadata tag: %#v", tags)
|
|
44
|
+
}
|
|
45
|
+
if len(errors) != 1 || len(errors[0].Messages) == 0 {
|
|
46
|
+
t.Fatalf("a lone invalid tag must retain its diagnostic: %#v", errors)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
func metadataTypeTagFactoryAllInvalidProperty(key string, value *schemametadata.MetadataSchema) *schemametadata.MetadataProperty {
|
|
51
|
+
return schemametadata.MetadataProperty_create(schemametadata.MetadataProperty{
|
|
52
|
+
Key: MetadataFactory.SoleLiteral(key),
|
|
53
|
+
Value: value,
|
|
54
|
+
})
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
func metadataTypeTagFactoryAllInvalidContainer(name string, properties ...*schemametadata.MetadataProperty) *schemametadata.MetadataObjectType {
|
|
58
|
+
tagValue := schemametadata.MetadataSchema_initialize()
|
|
59
|
+
tagValue.Required = false
|
|
60
|
+
tagValue.Optional = true
|
|
61
|
+
tagValue.Objects = append(tagValue.Objects, schemametadata.MetadataObject_create(schemametadata.MetadataObject{
|
|
62
|
+
Type: schemametadata.MetadataObjectType_create(schemametadata.MetadataObjectType{
|
|
63
|
+
Name: name + "Shape",
|
|
64
|
+
Properties: properties,
|
|
65
|
+
}),
|
|
66
|
+
}))
|
|
67
|
+
return schemametadata.MetadataObjectType_create(schemametadata.MetadataObjectType{
|
|
68
|
+
Name: name,
|
|
69
|
+
Properties: []*schemametadata.MetadataProperty{
|
|
70
|
+
metadataTypeTagFactoryAllInvalidProperty("typia.tag", tagValue),
|
|
71
|
+
},
|
|
72
|
+
})
|
|
73
|
+
}
|
|
@@ -401,6 +401,7 @@ func llmApplicationProgrammer_convertParameters(
|
|
|
401
401
|
output := llmSchemaProgrammer_convert_schema_config(target, components, defs, config)
|
|
402
402
|
output["additionalProperties"] = false
|
|
403
403
|
output["$defs"] = defs
|
|
404
|
+
llmReferenceDescription_assign(output, schema, components)
|
|
404
405
|
return output
|
|
405
406
|
}
|
|
406
407
|
|
|
@@ -0,0 +1,555 @@
|
|
|
1
|
+
package llm
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"fmt"
|
|
5
|
+
"math"
|
|
6
|
+
"reflect"
|
|
7
|
+
"sort"
|
|
8
|
+
"strconv"
|
|
9
|
+
"strings"
|
|
10
|
+
|
|
11
|
+
shimast "github.com/microsoft/typescript-go/shim/ast"
|
|
12
|
+
nativecontext "github.com/samchon/typia/packages/typia/native/core/context"
|
|
13
|
+
nativefactories "github.com/samchon/typia/packages/typia/native/core/factories"
|
|
14
|
+
schemametadata "github.com/samchon/typia/packages/typia/native/core/schemas/metadata"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
type llmEvaluationProgrammerNamespace struct{}
|
|
18
|
+
|
|
19
|
+
// LlmEvaluationProgrammer writes `typia.llm.evaluation<T>()`.
|
|
20
|
+
//
|
|
21
|
+
// The emitted code is one call to the `_createLlmEvaluation` runtime helper
|
|
22
|
+
// with a compile-time plan: one entry per decision leaf of `T`, carrying its
|
|
23
|
+
// property path, question text, and probability requirements. The helper
|
|
24
|
+
// derives both the question map and the checked decoder from that plan,
|
|
25
|
+
// so the question-key encoding has a single owner at runtime.
|
|
26
|
+
var LlmEvaluationProgrammer = llmEvaluationProgrammerNamespace{}
|
|
27
|
+
|
|
28
|
+
type LlmEvaluationProgrammer_IWriteProps struct {
|
|
29
|
+
Context nativecontext.ITypiaContext
|
|
30
|
+
Metadata *schemametadata.MetadataSchema
|
|
31
|
+
Name *string
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// LlmEvaluationProgrammer_IError is one rejected position of the decision type.
|
|
35
|
+
type LlmEvaluationProgrammer_IError struct {
|
|
36
|
+
// Accessor is the typia validation path of the position, like
|
|
37
|
+
// `$input.refund.requested`.
|
|
38
|
+
Accessor string
|
|
39
|
+
Message string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
var llmEvaluationProgrammer_factory = shimast.NewNodeFactory(shimast.NodeFactoryHooks{})
|
|
43
|
+
|
|
44
|
+
// Write emits `_createLlmEvaluation(plan) as ILlmEvaluation<T>`. The metadata
|
|
45
|
+
// must already have passed Compose without errors.
|
|
46
|
+
func (llmEvaluationProgrammerNamespace) Write(props LlmEvaluationProgrammer_IWriteProps, plan []any) *shimast.Node {
|
|
47
|
+
f := nativecontext.EmitFactoryOf(llmEvaluationProgrammer_factory, props.Context.Emit)
|
|
48
|
+
typeName := "unknown"
|
|
49
|
+
if props.Name != nil {
|
|
50
|
+
typeName = *props.Name
|
|
51
|
+
}
|
|
52
|
+
return f.NewAsExpression(
|
|
53
|
+
f.NewCallExpression(
|
|
54
|
+
llmProgrammer_internal(props.Context, "createLlmEvaluation"),
|
|
55
|
+
nil,
|
|
56
|
+
nil,
|
|
57
|
+
f.NewNodeList([]*shimast.Node{
|
|
58
|
+
nativefactories.LiteralFactory.Write(plan, props.Context.Emit),
|
|
59
|
+
}),
|
|
60
|
+
shimast.NodeFlagsNone,
|
|
61
|
+
),
|
|
62
|
+
llmProgrammer_import_type(props.Context, nativecontext.ImportProgrammer_TypeProps{
|
|
63
|
+
File: "typia",
|
|
64
|
+
Name: "ILlmEvaluation",
|
|
65
|
+
Arguments: []*shimast.TypeNode{llmProgrammer_type_reference(typeName, props.Context.Emit)},
|
|
66
|
+
}),
|
|
67
|
+
)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Compose walks the decision type and returns the evaluation plan, or every
|
|
71
|
+
// position an evaluation model cannot answer.
|
|
72
|
+
func (llmEvaluationProgrammerNamespace) Compose(metadata *schemametadata.MetadataSchema) ([]any, []LlmEvaluationProgrammer_IError) {
|
|
73
|
+
composer := &llmEvaluationComposer{
|
|
74
|
+
plan: []any{},
|
|
75
|
+
visited: map[*schemametadata.MetadataObjectType]bool{},
|
|
76
|
+
}
|
|
77
|
+
object := composer.top(metadata)
|
|
78
|
+
if object != nil {
|
|
79
|
+
composer.object(object, []string{}, "$input")
|
|
80
|
+
if len(composer.errors) == 0 && len(composer.plan) == 0 {
|
|
81
|
+
composer.fail("$input", "LLM evaluation type must have at least one decision property.")
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return composer.plan, composer.errors
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Message renders Compose errors in the transformer's unsupported-type layout.
|
|
88
|
+
func (llmEvaluationProgrammerNamespace) Message(errors []LlmEvaluationProgrammer_IError) string {
|
|
89
|
+
lines := make([]string, 0, len(errors))
|
|
90
|
+
for _, err := range errors {
|
|
91
|
+
lines = append(lines, "- "+err.Accessor+"\n - "+err.Message)
|
|
92
|
+
}
|
|
93
|
+
return "unsupported type detected\n\n" + strings.Join(lines, "\n\n")
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
type llmEvaluationComposer struct {
|
|
97
|
+
plan []any
|
|
98
|
+
errors []LlmEvaluationProgrammer_IError
|
|
99
|
+
visited map[*schemametadata.MetadataObjectType]bool
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
func (c *llmEvaluationComposer) fail(accessor string, message string) {
|
|
103
|
+
c.errors = append(c.errors, LlmEvaluationProgrammer_IError{Accessor: accessor, Message: message})
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
func (c *llmEvaluationComposer) top(metadata *schemametadata.MetadataSchema) *schemametadata.MetadataObjectType {
|
|
107
|
+
metadata = llmEvaluation_unalias(metadata)
|
|
108
|
+
if metadata == nil || len(metadata.Objects) == 0 {
|
|
109
|
+
c.fail("$input", "LLM evaluation type must be an object type.")
|
|
110
|
+
return nil
|
|
111
|
+
}
|
|
112
|
+
if len(metadata.Objects) != 1 || metadata.Size() != 1 {
|
|
113
|
+
c.fail("$input", "LLM evaluation type must be a single object type.")
|
|
114
|
+
return nil
|
|
115
|
+
}
|
|
116
|
+
if metadata.Nullable || metadata.IsRequired() == false {
|
|
117
|
+
c.fail("$input", "LLM evaluation type must be a non-nullable and non-undefined object type.")
|
|
118
|
+
return nil
|
|
119
|
+
}
|
|
120
|
+
if message := llmEvaluation_unsupported_tags(metadata.Objects[0].Tags); message != "" {
|
|
121
|
+
c.fail("$input", message)
|
|
122
|
+
return nil
|
|
123
|
+
}
|
|
124
|
+
return metadata.Objects[0].Type
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
func (c *llmEvaluationComposer) object(object *schemametadata.MetadataObjectType, path []string, accessor string) {
|
|
128
|
+
if c.visited[object] {
|
|
129
|
+
c.fail(accessor, "LLM evaluation does not support recursive types.")
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
c.visited[object] = true
|
|
133
|
+
defer delete(c.visited, object)
|
|
134
|
+
|
|
135
|
+
for _, property := range object.Properties {
|
|
136
|
+
key := property.Key.GetSoleLiteral()
|
|
137
|
+
if key == nil {
|
|
138
|
+
c.fail(accessor, "LLM evaluation does not support dynamic keys.")
|
|
139
|
+
continue
|
|
140
|
+
}
|
|
141
|
+
c.property(property, append(append([]string{}, path...), *key), accessor+nativefactories.IdentifierFactory.PathPostfix(*key))
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
func (c *llmEvaluationComposer) property(property *schemametadata.MetadataProperty, path []string, accessor string) {
|
|
146
|
+
value := llmEvaluation_unalias(property.Value)
|
|
147
|
+
if llmEvaluation_has_tag(property.JsDocTags, "hidden", "ignore", "internal") {
|
|
148
|
+
c.fail(accessor, "LLM evaluation does not support hidden properties, because every property of the result needs an answer.")
|
|
149
|
+
return
|
|
150
|
+
}
|
|
151
|
+
// typia's metadata folds unknown into any, and never and void into an
|
|
152
|
+
// undefined value, so each message names the class the metadata records
|
|
153
|
+
if value == nil || value.Any {
|
|
154
|
+
c.fail(accessor, "LLM evaluation does not support any or unknown types, because an evaluation model answers only closed sets.")
|
|
155
|
+
return
|
|
156
|
+
}
|
|
157
|
+
if value.Optional || value.Required == false {
|
|
158
|
+
c.fail(accessor, "LLM evaluation does not support optional, undefined, void, or never properties, because every property of the result needs an answer.")
|
|
159
|
+
return
|
|
160
|
+
}
|
|
161
|
+
if value.Nullable {
|
|
162
|
+
c.fail(accessor, "LLM evaluation does not support nullable properties.")
|
|
163
|
+
return
|
|
164
|
+
}
|
|
165
|
+
if len(value.Functions) != 0 {
|
|
166
|
+
c.fail(accessor, "LLM evaluation does not support function properties.")
|
|
167
|
+
return
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// nested object: flattened, so the property itself asks nothing
|
|
171
|
+
if len(value.Objects) != 0 {
|
|
172
|
+
if len(value.Objects) != 1 || value.Size() != 1 {
|
|
173
|
+
c.fail(accessor, "LLM evaluation does not support union types.")
|
|
174
|
+
return
|
|
175
|
+
}
|
|
176
|
+
if message := llmEvaluation_unsupported_tags(value.Objects[0].Tags); message != "" {
|
|
177
|
+
c.fail(accessor, message)
|
|
178
|
+
return
|
|
179
|
+
}
|
|
180
|
+
if _, found := llmEvaluation_jsdoc_probability(property.JsDocTags); found {
|
|
181
|
+
c.fail(accessor, "LLM evaluation @probability must be on a boolean, choice, score, or set property, not on an object.")
|
|
182
|
+
}
|
|
183
|
+
plan, errors := len(c.plan), len(c.errors)
|
|
184
|
+
c.object(value.Objects[0].Type, path, accessor)
|
|
185
|
+
if len(c.plan) == plan && len(c.errors) == errors {
|
|
186
|
+
// no answer could ever create the object, so decode() would return a
|
|
187
|
+
// value missing this required property
|
|
188
|
+
c.fail(accessor, "LLM evaluation object must have at least one decision property.")
|
|
189
|
+
}
|
|
190
|
+
return
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
kind := llmEvaluation_kind(value)
|
|
194
|
+
if kind == "" {
|
|
195
|
+
c.fail(accessor, llmEvaluation_unsupported(value))
|
|
196
|
+
return
|
|
197
|
+
}
|
|
198
|
+
instructions := llmEvaluation_description(property.Description, property.JsDocTags)
|
|
199
|
+
if instructions == nil || strings.TrimSpace(*instructions) == "" {
|
|
200
|
+
c.fail(accessor, "LLM evaluation property must have a JSDoc description, because it is the question text.")
|
|
201
|
+
return
|
|
202
|
+
}
|
|
203
|
+
fallback, hasFallback, message := llmEvaluation_property_probability(property.JsDocTags)
|
|
204
|
+
if message != "" {
|
|
205
|
+
c.fail(accessor, message)
|
|
206
|
+
return
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
leaf := map[string]any{
|
|
210
|
+
"kind": kind,
|
|
211
|
+
"path": llmEvaluation_strings(path),
|
|
212
|
+
"instructions": *instructions,
|
|
213
|
+
}
|
|
214
|
+
switch kind {
|
|
215
|
+
case "boolean":
|
|
216
|
+
if message := llmEvaluation_unsupported_tags(value.Atomics[0].Tags); message != "" {
|
|
217
|
+
c.fail(accessor, message)
|
|
218
|
+
return
|
|
219
|
+
}
|
|
220
|
+
threshold := 0.5
|
|
221
|
+
tagged, found, message := llmEvaluation_tag_probability(value.Atomics[0].Tags)
|
|
222
|
+
if message != "" {
|
|
223
|
+
c.fail(accessor, message)
|
|
224
|
+
return
|
|
225
|
+
}
|
|
226
|
+
if found && hasFallback {
|
|
227
|
+
c.fail(accessor, "LLM evaluation boolean has both tags.Probability and @probability; keep only one.")
|
|
228
|
+
return
|
|
229
|
+
}
|
|
230
|
+
if found {
|
|
231
|
+
threshold = tagged
|
|
232
|
+
} else if hasFallback {
|
|
233
|
+
threshold = fallback
|
|
234
|
+
}
|
|
235
|
+
leaf["threshold"] = threshold
|
|
236
|
+
case "choice", "score", "set":
|
|
237
|
+
constant := value.Constants
|
|
238
|
+
if kind == "set" {
|
|
239
|
+
constant = llmEvaluation_unalias(value.Arrays[0].Type.Value).Constants
|
|
240
|
+
}
|
|
241
|
+
entries := constant[0].Values
|
|
242
|
+
if kind == "score" {
|
|
243
|
+
entries = append([]*schemametadata.MetadataConstantValue{}, entries...)
|
|
244
|
+
sort.SliceStable(entries, func(i, j int) bool {
|
|
245
|
+
left, _ := llmEvaluation_number(entries[i].Value)
|
|
246
|
+
right, _ := llmEvaluation_number(entries[j].Value)
|
|
247
|
+
return left < right
|
|
248
|
+
})
|
|
249
|
+
}
|
|
250
|
+
members := make([]any, 0, len(entries))
|
|
251
|
+
hasMemberRequirement := false
|
|
252
|
+
missingRequirements := make([]*schemametadata.MetadataConstantValue, 0)
|
|
253
|
+
for _, entry := range entries {
|
|
254
|
+
if message := llmEvaluation_unsupported_tags(entry.Tags); message != "" {
|
|
255
|
+
c.fail(accessor, fmt.Sprintf("%s (member %s)", message, llmEvaluation_value_text(entry.Value)))
|
|
256
|
+
continue
|
|
257
|
+
}
|
|
258
|
+
member := map[string]any{"value": llmEvaluation_value(entry.Value)}
|
|
259
|
+
// an undocumented score level is described by its value, which the
|
|
260
|
+
// runtime writes with JavaScript's own number formatting
|
|
261
|
+
if description := llmEvaluation_member_description(entry); description != nil {
|
|
262
|
+
member["description"] = *description
|
|
263
|
+
}
|
|
264
|
+
requirement, found, message := llmEvaluation_member_probability(entry)
|
|
265
|
+
if message != "" {
|
|
266
|
+
c.fail(accessor, fmt.Sprintf("%s (member %s)", message, llmEvaluation_value_text(entry.Value)))
|
|
267
|
+
continue
|
|
268
|
+
}
|
|
269
|
+
if found == false && hasFallback {
|
|
270
|
+
requirement, found = fallback, true
|
|
271
|
+
}
|
|
272
|
+
if found {
|
|
273
|
+
hasMemberRequirement = true
|
|
274
|
+
} else {
|
|
275
|
+
missingRequirements = append(missingRequirements, entry)
|
|
276
|
+
}
|
|
277
|
+
if kind == "set" {
|
|
278
|
+
if found == false {
|
|
279
|
+
requirement = 0.5
|
|
280
|
+
}
|
|
281
|
+
member["threshold"] = requirement
|
|
282
|
+
} else if found {
|
|
283
|
+
member["minimum"] = requirement
|
|
284
|
+
}
|
|
285
|
+
members = append(members, member)
|
|
286
|
+
}
|
|
287
|
+
if hasMemberRequirement && len(missingRequirements) != 0 {
|
|
288
|
+
for _, entry := range missingRequirements {
|
|
289
|
+
c.fail(accessor, fmt.Sprintf(
|
|
290
|
+
"LLM evaluation probability requirements must cover every member once one member declares one; add tags.Probability or @probability to member %s, or add a property @probability default.",
|
|
291
|
+
llmEvaluation_value_text(entry.Value),
|
|
292
|
+
))
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
leaf[map[string]string{"choice": "options", "score": "levels", "set": "members"}[kind]] = members
|
|
296
|
+
}
|
|
297
|
+
c.plan = append(c.plan, leaf)
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// A decoded value is trusted as T, so every tag with a validation meaning
|
|
301
|
+
// must either be enforced here or rejected. These kinds only annotate schema
|
|
302
|
+
// documentation or other serializers; probability is enforced by the decoder.
|
|
303
|
+
func llmEvaluation_unsupported_tags(rows [][]schemametadata.IMetadataTypeTag) string {
|
|
304
|
+
for _, row := range rows {
|
|
305
|
+
for _, tag := range row {
|
|
306
|
+
if tag.Validate != "" {
|
|
307
|
+
return fmt.Sprintf("LLM evaluation does not support type tag %q, because decode() cannot enforce its constraint.", tag.Kind)
|
|
308
|
+
}
|
|
309
|
+
switch tag.Kind {
|
|
310
|
+
case "probability", "constant", "default", "example", "examples", "contentMediaType", "sequence":
|
|
311
|
+
continue
|
|
312
|
+
default:
|
|
313
|
+
return fmt.Sprintf("LLM evaluation does not support type tag %q, because decode() cannot enforce its constraint.", tag.Kind)
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return ""
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// llmEvaluation_kind classifies a non-object property value as one question
|
|
321
|
+
// family, or "" when no evaluation question can answer it.
|
|
322
|
+
func llmEvaluation_kind(value *schemametadata.MetadataSchema) string {
|
|
323
|
+
if value.Escaped != nil || value.Rest != nil || len(value.Templates) != 0 ||
|
|
324
|
+
len(value.Tuples) != 0 || len(value.Natives) != 0 || len(value.Sets) != 0 ||
|
|
325
|
+
len(value.Maps) != 0 || len(value.Aliases) != 0 {
|
|
326
|
+
return ""
|
|
327
|
+
}
|
|
328
|
+
if len(value.Atomics) == 1 && value.Atomics[0].Type == "boolean" &&
|
|
329
|
+
len(value.Constants) == 0 && len(value.Arrays) == 0 {
|
|
330
|
+
return "boolean"
|
|
331
|
+
}
|
|
332
|
+
if len(value.Atomics) == 0 && len(value.Arrays) == 0 && len(value.Constants) == 1 &&
|
|
333
|
+
len(value.Constants[0].Values) >= 2 {
|
|
334
|
+
switch value.Constants[0].Type {
|
|
335
|
+
case "string":
|
|
336
|
+
return "choice"
|
|
337
|
+
case "number":
|
|
338
|
+
return "score"
|
|
339
|
+
}
|
|
340
|
+
return ""
|
|
341
|
+
}
|
|
342
|
+
if len(value.Atomics) == 0 && len(value.Constants) == 0 && len(value.Arrays) == 1 &&
|
|
343
|
+
len(value.Arrays[0].Tags) == 0 {
|
|
344
|
+
element := llmEvaluation_unalias(value.Arrays[0].Type.Value)
|
|
345
|
+
if element != nil && element.IsConstant() && element.Nullable == false && element.IsRequired() &&
|
|
346
|
+
len(element.Constants) == 1 && element.Constants[0].Type == "string" &&
|
|
347
|
+
len(element.Constants[0].Values) != 0 {
|
|
348
|
+
return "set"
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
return ""
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
func llmEvaluation_unsupported(value *schemametadata.MetadataSchema) string {
|
|
355
|
+
if len(value.Arrays) == 1 && len(value.Arrays[0].Tags) != 0 {
|
|
356
|
+
return "LLM evaluation does not support array type tags, because the answered members decide the array."
|
|
357
|
+
}
|
|
358
|
+
if len(value.Arrays) != 0 {
|
|
359
|
+
return "LLM evaluation supports only arrays of a string literal union or string enum."
|
|
360
|
+
}
|
|
361
|
+
if len(value.Constants) == 1 && len(value.Constants[0].Values) == 1 && len(value.Atomics) == 0 {
|
|
362
|
+
return "LLM evaluation does not support a single literal value, because it decides nothing."
|
|
363
|
+
}
|
|
364
|
+
if len(value.Constants) > 1 || (len(value.Constants) != 0 && len(value.Atomics) != 0) {
|
|
365
|
+
return "LLM evaluation does not support union types mixing different question kinds."
|
|
366
|
+
}
|
|
367
|
+
if len(value.Constants) == 1 && value.Constants[0].Type != "string" && value.Constants[0].Type != "number" {
|
|
368
|
+
return fmt.Sprintf("LLM evaluation does not support %s literal types.", value.Constants[0].Type)
|
|
369
|
+
}
|
|
370
|
+
return "LLM evaluation supports only boolean, string or numeric literal unions (or enums), arrays of a string literal union (or enum), and objects composed of them, because an evaluation model answers only closed sets."
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
func llmEvaluation_unalias(metadata *schemametadata.MetadataSchema) *schemametadata.MetadataSchema {
|
|
374
|
+
for metadata != nil && len(metadata.Aliases) == 1 && metadata.Size() == 1 &&
|
|
375
|
+
metadata.Aliases[0].Type != nil && metadata.Aliases[0].Type.Value != nil {
|
|
376
|
+
metadata = metadata.Aliases[0].Type.Value
|
|
377
|
+
}
|
|
378
|
+
return metadata
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
func llmEvaluation_description(description *string, tags []schemametadata.IJsDocTagInfo) *string {
|
|
382
|
+
for _, tag := range tags {
|
|
383
|
+
if tag.Name != "description" || len(tag.Text) == 0 {
|
|
384
|
+
continue
|
|
385
|
+
}
|
|
386
|
+
text := strings.ReplaceAll(tag.Text[0].Text, "\r\n", "\n")
|
|
387
|
+
return &text
|
|
388
|
+
}
|
|
389
|
+
if description == nil {
|
|
390
|
+
return nil
|
|
391
|
+
}
|
|
392
|
+
text := strings.ReplaceAll(*description, "\r\n", "\n")
|
|
393
|
+
return &text
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
func llmEvaluation_member_description(entry *schemametadata.MetadataConstantValue) *string {
|
|
397
|
+
if description := llmEvaluation_description(entry.Description, entry.JsDocTags); description != nil && strings.TrimSpace(*description) != "" {
|
|
398
|
+
return description
|
|
399
|
+
}
|
|
400
|
+
for _, row := range entry.Tags {
|
|
401
|
+
for _, tag := range row {
|
|
402
|
+
if tag.Kind != "constant" {
|
|
403
|
+
continue
|
|
404
|
+
}
|
|
405
|
+
schema, ok := tag.Schema.(map[string]any)
|
|
406
|
+
if ok == false {
|
|
407
|
+
continue
|
|
408
|
+
}
|
|
409
|
+
if text, ok := schema["description"].(string); ok && strings.TrimSpace(text) != "" {
|
|
410
|
+
return &text
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return nil
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
func llmEvaluation_has_tag(tags []schemametadata.IJsDocTagInfo, names ...string) bool {
|
|
418
|
+
for _, tag := range tags {
|
|
419
|
+
for _, name := range names {
|
|
420
|
+
if tag.Name == name {
|
|
421
|
+
return true
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return false
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// llmEvaluation_jsdoc_probability reads the raw `@probability` comment tags.
|
|
429
|
+
func llmEvaluation_jsdoc_probability(tags []schemametadata.IJsDocTagInfo) ([]string, bool) {
|
|
430
|
+
texts := []string{}
|
|
431
|
+
for _, tag := range tags {
|
|
432
|
+
if tag.Name != "probability" {
|
|
433
|
+
continue
|
|
434
|
+
}
|
|
435
|
+
parts := make([]string, 0, len(tag.Text))
|
|
436
|
+
for _, part := range tag.Text {
|
|
437
|
+
parts = append(parts, part.Text)
|
|
438
|
+
}
|
|
439
|
+
texts = append(texts, strings.TrimSpace(strings.Join(parts, "")))
|
|
440
|
+
}
|
|
441
|
+
return texts, len(texts) != 0
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
func llmEvaluation_parse_jsdoc_probability(tags []schemametadata.IJsDocTagInfo) (float64, bool, string) {
|
|
445
|
+
texts, found := llmEvaluation_jsdoc_probability(tags)
|
|
446
|
+
if found == false {
|
|
447
|
+
return 0, false, ""
|
|
448
|
+
}
|
|
449
|
+
if len(texts) != 1 {
|
|
450
|
+
return 0, false, "LLM evaluation @probability is declared more than once; keep only one."
|
|
451
|
+
}
|
|
452
|
+
value, err := strconv.ParseFloat(texts[0], 64)
|
|
453
|
+
if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {
|
|
454
|
+
return 0, false, fmt.Sprintf("LLM evaluation @probability must be a number in [0, 1], but got %q.", texts[0])
|
|
455
|
+
}
|
|
456
|
+
if value < 0 || value > 1 {
|
|
457
|
+
return 0, false, fmt.Sprintf("LLM evaluation @probability must be in [0, 1], but got %s.", texts[0])
|
|
458
|
+
}
|
|
459
|
+
return value, true, ""
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
func llmEvaluation_property_probability(tags []schemametadata.IJsDocTagInfo) (float64, bool, string) {
|
|
463
|
+
return llmEvaluation_parse_jsdoc_probability(tags)
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
func llmEvaluation_tag_probability(rows [][]schemametadata.IMetadataTypeTag) (float64, bool, string) {
|
|
467
|
+
values := []any{}
|
|
468
|
+
for _, row := range rows {
|
|
469
|
+
for _, tag := range row {
|
|
470
|
+
if tag.Kind == "probability" {
|
|
471
|
+
values = append(values, tag.Value)
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if len(values) == 0 {
|
|
476
|
+
return 0, false, ""
|
|
477
|
+
}
|
|
478
|
+
if len(values) != 1 {
|
|
479
|
+
return 0, false, "LLM evaluation tags.Probability is declared more than once; keep only one."
|
|
480
|
+
}
|
|
481
|
+
value, ok := llmEvaluation_number(values[0])
|
|
482
|
+
if ok == false || math.IsNaN(value) || math.IsInf(value, 0) {
|
|
483
|
+
return 0, false, "LLM evaluation tags.Probability must have a numeric literal value."
|
|
484
|
+
}
|
|
485
|
+
if value < 0 || value > 1 {
|
|
486
|
+
return 0, false, fmt.Sprintf("LLM evaluation tags.Probability must be in [0, 1], but got %s.", llmEvaluation_number_text(value))
|
|
487
|
+
}
|
|
488
|
+
return value, true, ""
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
func llmEvaluation_member_probability(entry *schemametadata.MetadataConstantValue) (float64, bool, string) {
|
|
492
|
+
tagged, taggedFound, message := llmEvaluation_tag_probability(entry.Tags)
|
|
493
|
+
if message != "" {
|
|
494
|
+
return 0, false, message
|
|
495
|
+
}
|
|
496
|
+
commented, commentedFound, message := llmEvaluation_parse_jsdoc_probability(entry.JsDocTags)
|
|
497
|
+
if message != "" {
|
|
498
|
+
return 0, false, message
|
|
499
|
+
}
|
|
500
|
+
if taggedFound && commentedFound {
|
|
501
|
+
return 0, false, "LLM evaluation member has both tags.Probability and @probability; keep only one."
|
|
502
|
+
}
|
|
503
|
+
if taggedFound {
|
|
504
|
+
return tagged, true, ""
|
|
505
|
+
}
|
|
506
|
+
return commented, commentedFound, ""
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
func llmEvaluation_number(value any) (float64, bool) {
|
|
510
|
+
reflected := reflect.ValueOf(value)
|
|
511
|
+
switch reflected.Kind() {
|
|
512
|
+
case reflect.Float32, reflect.Float64:
|
|
513
|
+
return reflected.Float(), true
|
|
514
|
+
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
|
515
|
+
return float64(reflected.Int()), true
|
|
516
|
+
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
|
517
|
+
return float64(reflected.Uint()), true
|
|
518
|
+
}
|
|
519
|
+
return 0, false
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// llmEvaluation_value normalizes a literal to a plain Go value LiteralFactory
|
|
523
|
+
// writes, so a checker-specific number type never reaches the emitter.
|
|
524
|
+
func llmEvaluation_value(value any) any {
|
|
525
|
+
if str, ok := value.(string); ok {
|
|
526
|
+
return str
|
|
527
|
+
}
|
|
528
|
+
if number, ok := llmEvaluation_number(value); ok {
|
|
529
|
+
return number
|
|
530
|
+
}
|
|
531
|
+
return value
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
func llmEvaluation_number_text(value any) string {
|
|
535
|
+
number, ok := llmEvaluation_number(value)
|
|
536
|
+
if ok == false {
|
|
537
|
+
return fmt.Sprint(value)
|
|
538
|
+
}
|
|
539
|
+
return strconv.FormatFloat(number, 'f', -1, 64)
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
func llmEvaluation_value_text(value any) string {
|
|
543
|
+
if str, ok := value.(string); ok {
|
|
544
|
+
return strconv.Quote(str)
|
|
545
|
+
}
|
|
546
|
+
return llmEvaluation_number_text(value)
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
func llmEvaluation_strings(values []string) []any {
|
|
550
|
+
output := make([]any, 0, len(values))
|
|
551
|
+
for _, value := range values {
|
|
552
|
+
output = append(output, value)
|
|
553
|
+
}
|
|
554
|
+
return output
|
|
555
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
package llm
|
|
2
2
|
|
|
3
3
|
import (
|
|
4
|
-
"strings"
|
|
5
|
-
|
|
6
4
|
shimast "github.com/microsoft/typescript-go/shim/ast"
|
|
7
5
|
nativecontext "github.com/samchon/typia/packages/typia/native/core/context"
|
|
8
6
|
nativefactories "github.com/samchon/typia/packages/typia/native/core/factories"
|
|
@@ -73,11 +71,13 @@ func (llmParametersProgrammerNamespace) WriteParameters(props struct {
|
|
|
73
71
|
return llmParametersProgrammer_convert_parameters(schema, collection.Components, props.Config)
|
|
74
72
|
}
|
|
75
73
|
if ref, ok := schema["$ref"].(string); ok {
|
|
76
|
-
name :=
|
|
74
|
+
name := llmSchemaProgrammer_ref_key(ref)
|
|
77
75
|
if collection.Components != nil && collection.Components.Schemas != nil {
|
|
78
76
|
if target, found := collection.Components.Schemas[name]; found {
|
|
79
77
|
if typ, ok := target["type"].(string); ok && typ == "object" {
|
|
80
|
-
|
|
78
|
+
// the reference itself, so the root description can cascade from
|
|
79
|
+
// its component key
|
|
80
|
+
return llmParametersProgrammer_convert_parameters(schema, collection.Components, props.Config)
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
83
|
}
|
|
@@ -125,6 +125,7 @@ func llmParametersProgrammer_convert_parameters(schema nativeiterate.JsonSchema,
|
|
|
125
125
|
output := llmSchemaProgrammer_convert_schema_config(target, components, defs, config)
|
|
126
126
|
output["additionalProperties"] = false
|
|
127
127
|
output["$defs"] = defs
|
|
128
|
+
llmReferenceDescription_assign(output, schema, components)
|
|
128
129
|
return output
|
|
129
130
|
}
|
|
130
131
|
|
|
@@ -133,7 +134,7 @@ func llmParametersProgrammer_dereference_schema(schema nativeiterate.JsonSchema,
|
|
|
133
134
|
if ok == false || components == nil || components.Schemas == nil {
|
|
134
135
|
return schema
|
|
135
136
|
}
|
|
136
|
-
name :=
|
|
137
|
+
name := llmSchemaProgrammer_ref_key(ref)
|
|
137
138
|
if target, found := components.Schemas[name]; found {
|
|
138
139
|
return target
|
|
139
140
|
}
|