sillyspec 3.34.0 → 3.34.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/deploy/SKILL.md +108 -0
- package/.claude/skills/sillyspec-flow/SKILL.md +1 -1
- package/AGENTS.md +3 -0
- package/assets/command-cards/flow.md +2 -1
- package/eval/knowledge/baseline.json +34 -481
- package/package.json +1 -1
- package/src/archive-baseline-replay.js +94 -0
- package/src/decision-distill.js +111 -5
- package/src/doctor-diagnostics.js +56 -1
- package/src/flow.js +37 -19
- package/src/foreign-declared.js +22 -7
- package/src/hunk-attribution.js +7 -3
- package/src/index.js +10 -2
- package/src/knowledge-classify.js +230 -4
- package/src/knowledge-digest.js +19 -0
- package/src/knowledge-match.js +146 -3
- package/src/knowledge-vector.js +59 -5
- package/src/md-fence.js +64 -0
- package/src/progress.js +34 -2
- package/src/route-hindsight.js +13 -8
- package/src/spec-sync.js +155 -25
- package/src/stages/brainstorm.js +1 -1
- package/src/stages/execute.js +3 -1
- package/src/stages/explore.js +4 -2
- package/src/stages/knowledge.js +110 -125
- package/src/task-tick.js +42 -9
- package/templates/agents-instruction.md +3 -0
|
@@ -3,24 +3,15 @@
|
|
|
3
3
|
"generated_by": "eval/knowledge/run.mjs",
|
|
4
4
|
"snapshot": {
|
|
5
5
|
"sillyspec": {
|
|
6
|
-
"head": "
|
|
7
|
-
"knowledge_hash": "
|
|
8
|
-
"knowledge_files":
|
|
9
|
-
},
|
|
10
|
-
"platform": {
|
|
11
|
-
"head": "954873529837ce253ef611f7aa38df9e512ae12e",
|
|
12
|
-
"knowledge_hash": "36cb77cf9b1e1f50f1aea28c9a1d9ad1b887dec8cd7e6b1b10c71527011ae0e3",
|
|
13
|
-
"knowledge_files": 160
|
|
6
|
+
"head": "c5b1d9a5a6cd83437910253ba0a6ff9ea5dd7904",
|
|
7
|
+
"knowledge_hash": "ef33189ed3662e37e286fb6827eb06e5ea449853459da0a74a99d0305b973665",
|
|
8
|
+
"knowledge_files": 141
|
|
14
9
|
}
|
|
15
10
|
},
|
|
16
11
|
"cases": {
|
|
17
12
|
"sillyspec": {
|
|
18
13
|
"organic": 31,
|
|
19
14
|
"self": 214
|
|
20
|
-
},
|
|
21
|
-
"platform": {
|
|
22
|
-
"organic": 42,
|
|
23
|
-
"self": 330
|
|
24
15
|
}
|
|
25
16
|
},
|
|
26
17
|
"metrics": {
|
|
@@ -272,47 +263,47 @@
|
|
|
272
263
|
},
|
|
273
264
|
"scope": {
|
|
274
265
|
"raw": {
|
|
275
|
-
"top1": 0.
|
|
276
|
-
"top3": 0.
|
|
277
|
-
"mrr": 0.
|
|
266
|
+
"top1": 0.6667,
|
|
267
|
+
"top3": 0.75,
|
|
268
|
+
"mrr": 0.7215,
|
|
278
269
|
"n": 12
|
|
279
270
|
},
|
|
280
271
|
"rendered": {
|
|
281
|
-
"top1": 0.
|
|
282
|
-
"top3": 0.
|
|
283
|
-
"mrr": 0.
|
|
272
|
+
"top1": 0.6667,
|
|
273
|
+
"top3": 0.75,
|
|
274
|
+
"mrr": 0.7215,
|
|
284
275
|
"n": 12
|
|
285
276
|
},
|
|
286
277
|
"truncated": [],
|
|
287
278
|
"details": [
|
|
288
279
|
{
|
|
289
280
|
"id": "s-man-01",
|
|
290
|
-
"raw_rank":
|
|
291
|
-
"rendered_rank":
|
|
281
|
+
"raw_rank": 1,
|
|
282
|
+
"rendered_rank": 1,
|
|
292
283
|
"truncated": false
|
|
293
284
|
},
|
|
294
285
|
{
|
|
295
286
|
"id": "s-man-02",
|
|
296
|
-
"raw_rank":
|
|
297
|
-
"rendered_rank":
|
|
287
|
+
"raw_rank": 8,
|
|
288
|
+
"rendered_rank": 8,
|
|
298
289
|
"truncated": false
|
|
299
290
|
},
|
|
300
291
|
{
|
|
301
292
|
"id": "s-man-03",
|
|
302
|
-
"raw_rank":
|
|
303
|
-
"rendered_rank":
|
|
293
|
+
"raw_rank": 1,
|
|
294
|
+
"rendered_rank": 1,
|
|
304
295
|
"truncated": false
|
|
305
296
|
},
|
|
306
297
|
{
|
|
307
298
|
"id": "s-man-04",
|
|
308
|
-
"raw_rank":
|
|
309
|
-
"rendered_rank":
|
|
299
|
+
"raw_rank": 1,
|
|
300
|
+
"rendered_rank": 1,
|
|
310
301
|
"truncated": false
|
|
311
302
|
},
|
|
312
303
|
{
|
|
313
304
|
"id": "s-man-05",
|
|
314
|
-
"raw_rank":
|
|
315
|
-
"rendered_rank":
|
|
305
|
+
"raw_rank": 1,
|
|
306
|
+
"rendered_rank": 1,
|
|
316
307
|
"truncated": false
|
|
317
308
|
},
|
|
318
309
|
{
|
|
@@ -329,14 +320,14 @@
|
|
|
329
320
|
},
|
|
330
321
|
{
|
|
331
322
|
"id": "s-scope-03",
|
|
332
|
-
"raw_rank":
|
|
333
|
-
"rendered_rank":
|
|
323
|
+
"raw_rank": 3,
|
|
324
|
+
"rendered_rank": 3,
|
|
334
325
|
"truncated": false
|
|
335
326
|
},
|
|
336
327
|
{
|
|
337
328
|
"id": "s-scope-04",
|
|
338
|
-
"raw_rank":
|
|
339
|
-
"rendered_rank":
|
|
329
|
+
"raw_rank": 5,
|
|
330
|
+
"rendered_rank": 5,
|
|
340
331
|
"truncated": false
|
|
341
332
|
},
|
|
342
333
|
{
|
|
@@ -360,18 +351,18 @@
|
|
|
360
351
|
],
|
|
361
352
|
"subsets": {
|
|
362
353
|
"frozen": {
|
|
363
|
-
"top1": 0,
|
|
364
|
-
"top3": 0.
|
|
365
|
-
"mrr": 0.
|
|
354
|
+
"top1": 0.8,
|
|
355
|
+
"top3": 0.8,
|
|
356
|
+
"mrr": 0.825,
|
|
366
357
|
"n": 5,
|
|
367
|
-
"reachable":
|
|
358
|
+
"reachable": 1
|
|
368
359
|
},
|
|
369
360
|
"added": {
|
|
370
|
-
"top1": 0.
|
|
361
|
+
"top1": 0.4,
|
|
371
362
|
"top3": 0.6,
|
|
372
|
-
"mrr": 0.
|
|
363
|
+
"mrr": 0.5067,
|
|
373
364
|
"n": 5,
|
|
374
|
-
"reachable": 0.
|
|
365
|
+
"reachable": 0.8
|
|
375
366
|
},
|
|
376
367
|
"fusion": {
|
|
377
368
|
"top1": 1,
|
|
@@ -382,11 +373,6 @@
|
|
|
382
373
|
}
|
|
383
374
|
},
|
|
384
375
|
"scope_unreachable": [
|
|
385
|
-
"s-man-01",
|
|
386
|
-
"s-man-03",
|
|
387
|
-
"s-man-04",
|
|
388
|
-
"s-man-05",
|
|
389
|
-
"s-scope-04",
|
|
390
376
|
"s-scope-05"
|
|
391
377
|
]
|
|
392
378
|
},
|
|
@@ -394,7 +380,7 @@
|
|
|
394
380
|
"raw": {
|
|
395
381
|
"top1": 0.25,
|
|
396
382
|
"top3": 0.5,
|
|
397
|
-
"mrr": 0.
|
|
383
|
+
"mrr": 0.4167,
|
|
398
384
|
"n": 4
|
|
399
385
|
},
|
|
400
386
|
"rendered": {
|
|
@@ -422,7 +408,7 @@
|
|
|
422
408
|
},
|
|
423
409
|
{
|
|
424
410
|
"id": "s-man-10",
|
|
425
|
-
"raw_rank":
|
|
411
|
+
"raw_rank": 12,
|
|
426
412
|
"rendered_rank": null,
|
|
427
413
|
"truncated": true
|
|
428
414
|
},
|
|
@@ -437,7 +423,7 @@
|
|
|
437
423
|
"approx": {
|
|
438
424
|
"top1": 0,
|
|
439
425
|
"top3": 0.3333,
|
|
440
|
-
"mrr": 0.
|
|
426
|
+
"mrr": 0.2222,
|
|
441
427
|
"n": 3,
|
|
442
428
|
"reachable": 1
|
|
443
429
|
},
|
|
@@ -450,448 +436,15 @@
|
|
|
450
436
|
}
|
|
451
437
|
}
|
|
452
438
|
}
|
|
453
|
-
},
|
|
454
|
-
"platform": {
|
|
455
|
-
"entries": {
|
|
456
|
-
"raw": {
|
|
457
|
-
"top1": 0.5556,
|
|
458
|
-
"top3": 0.8889,
|
|
459
|
-
"mrr": 0.7037,
|
|
460
|
-
"n": 9
|
|
461
|
-
},
|
|
462
|
-
"rendered": {
|
|
463
|
-
"top1": 0.5556,
|
|
464
|
-
"top3": 0.8889,
|
|
465
|
-
"mrr": 0.7037,
|
|
466
|
-
"n": 9
|
|
467
|
-
},
|
|
468
|
-
"truncated": [],
|
|
469
|
-
"details": [
|
|
470
|
-
{
|
|
471
|
-
"id": "p-tel-01",
|
|
472
|
-
"raw_rank": 1,
|
|
473
|
-
"rendered_rank": 1,
|
|
474
|
-
"truncated": false
|
|
475
|
-
},
|
|
476
|
-
{
|
|
477
|
-
"id": "p-tel-02",
|
|
478
|
-
"raw_rank": 2,
|
|
479
|
-
"rendered_rank": 2,
|
|
480
|
-
"truncated": false
|
|
481
|
-
},
|
|
482
|
-
{
|
|
483
|
-
"id": "p-tel-03",
|
|
484
|
-
"raw_rank": 3,
|
|
485
|
-
"rendered_rank": 3,
|
|
486
|
-
"truncated": false
|
|
487
|
-
},
|
|
488
|
-
{
|
|
489
|
-
"id": "p-tel-04",
|
|
490
|
-
"raw_rank": 1,
|
|
491
|
-
"rendered_rank": 1,
|
|
492
|
-
"truncated": false
|
|
493
|
-
},
|
|
494
|
-
{
|
|
495
|
-
"id": "p-tel-05",
|
|
496
|
-
"raw_rank": 2,
|
|
497
|
-
"rendered_rank": 2,
|
|
498
|
-
"truncated": false
|
|
499
|
-
},
|
|
500
|
-
{
|
|
501
|
-
"id": "p-tel-06",
|
|
502
|
-
"raw_rank": 1,
|
|
503
|
-
"rendered_rank": 1,
|
|
504
|
-
"truncated": false
|
|
505
|
-
},
|
|
506
|
-
{
|
|
507
|
-
"id": "p-tel-07",
|
|
508
|
-
"raw_rank": null,
|
|
509
|
-
"rendered_rank": null,
|
|
510
|
-
"truncated": false
|
|
511
|
-
},
|
|
512
|
-
{
|
|
513
|
-
"id": "p-tel-09",
|
|
514
|
-
"raw_rank": 1,
|
|
515
|
-
"rendered_rank": 1,
|
|
516
|
-
"truncated": false
|
|
517
|
-
},
|
|
518
|
-
{
|
|
519
|
-
"id": "p-tel-13",
|
|
520
|
-
"raw_rank": 1,
|
|
521
|
-
"rendered_rank": 1,
|
|
522
|
-
"truncated": false
|
|
523
|
-
}
|
|
524
|
-
]
|
|
525
|
-
},
|
|
526
|
-
"decisions": {
|
|
527
|
-
"raw": {
|
|
528
|
-
"top1": 0.8571,
|
|
529
|
-
"top3": 0.9048,
|
|
530
|
-
"mrr": 0.9008,
|
|
531
|
-
"n": 21
|
|
532
|
-
},
|
|
533
|
-
"rendered": {
|
|
534
|
-
"top1": 0.8571,
|
|
535
|
-
"top3": 0.9048,
|
|
536
|
-
"mrr": 0.8929,
|
|
537
|
-
"n": 21
|
|
538
|
-
},
|
|
539
|
-
"truncated": [
|
|
540
|
-
"auto-platform-04"
|
|
541
|
-
],
|
|
542
|
-
"details": [
|
|
543
|
-
{
|
|
544
|
-
"id": "p-tel-08",
|
|
545
|
-
"raw_rank": 1,
|
|
546
|
-
"rendered_rank": 1,
|
|
547
|
-
"truncated": false
|
|
548
|
-
},
|
|
549
|
-
{
|
|
550
|
-
"id": "p-dec-01",
|
|
551
|
-
"raw_rank": 1,
|
|
552
|
-
"rendered_rank": 1,
|
|
553
|
-
"truncated": false
|
|
554
|
-
},
|
|
555
|
-
{
|
|
556
|
-
"id": "p-dec-02",
|
|
557
|
-
"raw_rank": 1,
|
|
558
|
-
"rendered_rank": 1,
|
|
559
|
-
"truncated": false
|
|
560
|
-
},
|
|
561
|
-
{
|
|
562
|
-
"id": "p-dec-03",
|
|
563
|
-
"raw_rank": 1,
|
|
564
|
-
"rendered_rank": 1,
|
|
565
|
-
"truncated": false
|
|
566
|
-
},
|
|
567
|
-
{
|
|
568
|
-
"id": "p-dec-04",
|
|
569
|
-
"raw_rank": 1,
|
|
570
|
-
"rendered_rank": 1,
|
|
571
|
-
"truncated": false
|
|
572
|
-
},
|
|
573
|
-
{
|
|
574
|
-
"id": "p-dec-05",
|
|
575
|
-
"raw_rank": 1,
|
|
576
|
-
"rendered_rank": 1,
|
|
577
|
-
"truncated": false
|
|
578
|
-
},
|
|
579
|
-
{
|
|
580
|
-
"id": "p-dec-06",
|
|
581
|
-
"raw_rank": 1,
|
|
582
|
-
"rendered_rank": 1,
|
|
583
|
-
"truncated": false
|
|
584
|
-
},
|
|
585
|
-
{
|
|
586
|
-
"id": "p-dec-07",
|
|
587
|
-
"raw_rank": 1,
|
|
588
|
-
"rendered_rank": 1,
|
|
589
|
-
"truncated": false
|
|
590
|
-
},
|
|
591
|
-
{
|
|
592
|
-
"id": "p-dec-08",
|
|
593
|
-
"raw_rank": 1,
|
|
594
|
-
"rendered_rank": 1,
|
|
595
|
-
"truncated": false
|
|
596
|
-
},
|
|
597
|
-
{
|
|
598
|
-
"id": "auto-platform-01",
|
|
599
|
-
"raw_rank": 1,
|
|
600
|
-
"rendered_rank": 1,
|
|
601
|
-
"truncated": false
|
|
602
|
-
},
|
|
603
|
-
{
|
|
604
|
-
"id": "auto-platform-02",
|
|
605
|
-
"raw_rank": 1,
|
|
606
|
-
"rendered_rank": 1,
|
|
607
|
-
"truncated": false
|
|
608
|
-
},
|
|
609
|
-
{
|
|
610
|
-
"id": "auto-platform-03",
|
|
611
|
-
"raw_rank": 2,
|
|
612
|
-
"rendered_rank": 2,
|
|
613
|
-
"truncated": false
|
|
614
|
-
},
|
|
615
|
-
{
|
|
616
|
-
"id": "auto-platform-04",
|
|
617
|
-
"raw_rank": 6,
|
|
618
|
-
"rendered_rank": null,
|
|
619
|
-
"truncated": true
|
|
620
|
-
},
|
|
621
|
-
{
|
|
622
|
-
"id": "auto-platform-05",
|
|
623
|
-
"raw_rank": 1,
|
|
624
|
-
"rendered_rank": 1,
|
|
625
|
-
"truncated": false
|
|
626
|
-
},
|
|
627
|
-
{
|
|
628
|
-
"id": "auto-platform-06",
|
|
629
|
-
"raw_rank": 1,
|
|
630
|
-
"rendered_rank": 1,
|
|
631
|
-
"truncated": false
|
|
632
|
-
},
|
|
633
|
-
{
|
|
634
|
-
"id": "auto-platform-07",
|
|
635
|
-
"raw_rank": 1,
|
|
636
|
-
"rendered_rank": 1,
|
|
637
|
-
"truncated": false
|
|
638
|
-
},
|
|
639
|
-
{
|
|
640
|
-
"id": "auto-platform-08",
|
|
641
|
-
"raw_rank": 1,
|
|
642
|
-
"rendered_rank": 1,
|
|
643
|
-
"truncated": false
|
|
644
|
-
},
|
|
645
|
-
{
|
|
646
|
-
"id": "auto-platform-09",
|
|
647
|
-
"raw_rank": 1,
|
|
648
|
-
"rendered_rank": 1,
|
|
649
|
-
"truncated": false
|
|
650
|
-
},
|
|
651
|
-
{
|
|
652
|
-
"id": "auto-platform-10",
|
|
653
|
-
"raw_rank": 1,
|
|
654
|
-
"rendered_rank": 1,
|
|
655
|
-
"truncated": false
|
|
656
|
-
},
|
|
657
|
-
{
|
|
658
|
-
"id": "auto-platform-11",
|
|
659
|
-
"raw_rank": 4,
|
|
660
|
-
"rendered_rank": 4,
|
|
661
|
-
"truncated": false
|
|
662
|
-
},
|
|
663
|
-
{
|
|
664
|
-
"id": "auto-platform-12",
|
|
665
|
-
"raw_rank": 1,
|
|
666
|
-
"rendered_rank": 1,
|
|
667
|
-
"truncated": false
|
|
668
|
-
}
|
|
669
|
-
]
|
|
670
|
-
},
|
|
671
|
-
"search": {
|
|
672
|
-
"raw": {
|
|
673
|
-
"top1": 0.5556,
|
|
674
|
-
"top3": 0.8889,
|
|
675
|
-
"mrr": 0.7037,
|
|
676
|
-
"n": 9
|
|
677
|
-
},
|
|
678
|
-
"rendered": {
|
|
679
|
-
"top1": 0.5556,
|
|
680
|
-
"top3": 0.8889,
|
|
681
|
-
"mrr": 0.7037,
|
|
682
|
-
"n": 9
|
|
683
|
-
},
|
|
684
|
-
"truncated": [],
|
|
685
|
-
"details": [
|
|
686
|
-
{
|
|
687
|
-
"id": "p-tel-01",
|
|
688
|
-
"raw_rank": 1,
|
|
689
|
-
"rendered_rank": 1,
|
|
690
|
-
"truncated": false
|
|
691
|
-
},
|
|
692
|
-
{
|
|
693
|
-
"id": "p-tel-02",
|
|
694
|
-
"raw_rank": 2,
|
|
695
|
-
"rendered_rank": 2,
|
|
696
|
-
"truncated": false
|
|
697
|
-
},
|
|
698
|
-
{
|
|
699
|
-
"id": "p-tel-03",
|
|
700
|
-
"raw_rank": 3,
|
|
701
|
-
"rendered_rank": 3,
|
|
702
|
-
"truncated": false
|
|
703
|
-
},
|
|
704
|
-
{
|
|
705
|
-
"id": "p-tel-04",
|
|
706
|
-
"raw_rank": 1,
|
|
707
|
-
"rendered_rank": 1,
|
|
708
|
-
"truncated": false
|
|
709
|
-
},
|
|
710
|
-
{
|
|
711
|
-
"id": "p-tel-05",
|
|
712
|
-
"raw_rank": 2,
|
|
713
|
-
"rendered_rank": 2,
|
|
714
|
-
"truncated": false
|
|
715
|
-
},
|
|
716
|
-
{
|
|
717
|
-
"id": "p-tel-06",
|
|
718
|
-
"raw_rank": 1,
|
|
719
|
-
"rendered_rank": 1,
|
|
720
|
-
"truncated": false
|
|
721
|
-
},
|
|
722
|
-
{
|
|
723
|
-
"id": "p-tel-07",
|
|
724
|
-
"raw_rank": null,
|
|
725
|
-
"rendered_rank": null,
|
|
726
|
-
"truncated": false
|
|
727
|
-
},
|
|
728
|
-
{
|
|
729
|
-
"id": "p-tel-09",
|
|
730
|
-
"raw_rank": 1,
|
|
731
|
-
"rendered_rank": 1,
|
|
732
|
-
"truncated": false
|
|
733
|
-
},
|
|
734
|
-
{
|
|
735
|
-
"id": "p-tel-13",
|
|
736
|
-
"raw_rank": 1,
|
|
737
|
-
"rendered_rank": 1,
|
|
738
|
-
"truncated": false
|
|
739
|
-
}
|
|
740
|
-
]
|
|
741
|
-
},
|
|
742
|
-
"scope": {
|
|
743
|
-
"raw": {
|
|
744
|
-
"top1": 0.8333,
|
|
745
|
-
"top3": 1,
|
|
746
|
-
"mrr": 0.9167,
|
|
747
|
-
"n": 6
|
|
748
|
-
},
|
|
749
|
-
"rendered": {
|
|
750
|
-
"top1": 0.8333,
|
|
751
|
-
"top3": 1,
|
|
752
|
-
"mrr": 0.9167,
|
|
753
|
-
"n": 6
|
|
754
|
-
},
|
|
755
|
-
"truncated": [],
|
|
756
|
-
"details": [
|
|
757
|
-
{
|
|
758
|
-
"id": "p-scope-01",
|
|
759
|
-
"raw_rank": 1,
|
|
760
|
-
"rendered_rank": 1,
|
|
761
|
-
"truncated": false
|
|
762
|
-
},
|
|
763
|
-
{
|
|
764
|
-
"id": "p-scope-02",
|
|
765
|
-
"raw_rank": 2,
|
|
766
|
-
"rendered_rank": 2,
|
|
767
|
-
"truncated": false
|
|
768
|
-
},
|
|
769
|
-
{
|
|
770
|
-
"id": "p-scope-03",
|
|
771
|
-
"raw_rank": 1,
|
|
772
|
-
"rendered_rank": 1,
|
|
773
|
-
"truncated": false
|
|
774
|
-
},
|
|
775
|
-
{
|
|
776
|
-
"id": "p-scope-04",
|
|
777
|
-
"raw_rank": 1,
|
|
778
|
-
"rendered_rank": 1,
|
|
779
|
-
"truncated": false
|
|
780
|
-
},
|
|
781
|
-
{
|
|
782
|
-
"id": "p-scope-05",
|
|
783
|
-
"raw_rank": 1,
|
|
784
|
-
"rendered_rank": 1,
|
|
785
|
-
"truncated": false
|
|
786
|
-
},
|
|
787
|
-
{
|
|
788
|
-
"id": "p-scope-06",
|
|
789
|
-
"raw_rank": 1,
|
|
790
|
-
"rendered_rank": 1,
|
|
791
|
-
"truncated": false
|
|
792
|
-
}
|
|
793
|
-
],
|
|
794
|
-
"subsets": {
|
|
795
|
-
"added": {
|
|
796
|
-
"top1": 0.8333,
|
|
797
|
-
"top3": 1,
|
|
798
|
-
"mrr": 0.9167,
|
|
799
|
-
"n": 6,
|
|
800
|
-
"reachable": 1
|
|
801
|
-
}
|
|
802
|
-
},
|
|
803
|
-
"scope_unreachable": []
|
|
804
|
-
},
|
|
805
|
-
"fr": {
|
|
806
|
-
"raw": {
|
|
807
|
-
"top1": 0.1667,
|
|
808
|
-
"top3": 0.3333,
|
|
809
|
-
"mrr": 0.2732,
|
|
810
|
-
"n": 6
|
|
811
|
-
},
|
|
812
|
-
"rendered": {
|
|
813
|
-
"top1": 0.1667,
|
|
814
|
-
"top3": 0.3333,
|
|
815
|
-
"mrr": 0.25,
|
|
816
|
-
"n": 6
|
|
817
|
-
},
|
|
818
|
-
"truncated": [
|
|
819
|
-
"p-tel-10",
|
|
820
|
-
"p-tel-11",
|
|
821
|
-
"p-tel-12",
|
|
822
|
-
"p-tel-14"
|
|
823
|
-
],
|
|
824
|
-
"details": [
|
|
825
|
-
{
|
|
826
|
-
"id": "p-tel-10",
|
|
827
|
-
"raw_rank": 34,
|
|
828
|
-
"rendered_rank": null,
|
|
829
|
-
"truncated": true
|
|
830
|
-
},
|
|
831
|
-
{
|
|
832
|
-
"id": "p-tel-11",
|
|
833
|
-
"raw_rank": 32,
|
|
834
|
-
"rendered_rank": null,
|
|
835
|
-
"truncated": true
|
|
836
|
-
},
|
|
837
|
-
{
|
|
838
|
-
"id": "p-tel-12",
|
|
839
|
-
"raw_rank": 35,
|
|
840
|
-
"rendered_rank": null,
|
|
841
|
-
"truncated": true
|
|
842
|
-
},
|
|
843
|
-
{
|
|
844
|
-
"id": "p-tel-14",
|
|
845
|
-
"raw_rank": 20,
|
|
846
|
-
"rendered_rank": null,
|
|
847
|
-
"truncated": true
|
|
848
|
-
},
|
|
849
|
-
{
|
|
850
|
-
"id": "p-tel-15",
|
|
851
|
-
"raw_rank": 2,
|
|
852
|
-
"rendered_rank": 2,
|
|
853
|
-
"truncated": false
|
|
854
|
-
},
|
|
855
|
-
{
|
|
856
|
-
"id": "p-fr-prod-01",
|
|
857
|
-
"raw_rank": 1,
|
|
858
|
-
"rendered_rank": 1,
|
|
859
|
-
"truncated": false
|
|
860
|
-
}
|
|
861
|
-
],
|
|
862
|
-
"subsets": {
|
|
863
|
-
"approx": {
|
|
864
|
-
"top1": 0,
|
|
865
|
-
"top3": 0.2,
|
|
866
|
-
"mrr": 0.1278,
|
|
867
|
-
"n": 5,
|
|
868
|
-
"reachable": 1
|
|
869
|
-
},
|
|
870
|
-
"prod": {
|
|
871
|
-
"top1": 1,
|
|
872
|
-
"top3": 1,
|
|
873
|
-
"mrr": 1,
|
|
874
|
-
"n": 1,
|
|
875
|
-
"reachable": 1
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
}
|
|
879
439
|
}
|
|
880
440
|
},
|
|
881
441
|
"self_smoke": {
|
|
882
442
|
"sillyspec": {
|
|
883
443
|
"cases": 214,
|
|
884
|
-
"skipped":
|
|
444
|
+
"skipped": 618,
|
|
885
445
|
"own_hit_top3": 0.0187,
|
|
886
446
|
"name_only_top3": 0.028,
|
|
887
447
|
"leak_gap": -0.0093
|
|
888
|
-
},
|
|
889
|
-
"platform": {
|
|
890
|
-
"cases": 330,
|
|
891
|
-
"skipped": 488,
|
|
892
|
-
"own_hit_top3": 0.0121,
|
|
893
|
-
"name_only_top3": 0.003,
|
|
894
|
-
"leak_gap": 0.0091
|
|
895
448
|
}
|
|
896
449
|
},
|
|
897
450
|
"probes": {
|