rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
@font-face{font-family:Mona Sans Variable;font-style:normal;font-display:swap;font-weight:200 900;src:url(/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2)format("woff2-variations");unicode-range:U+102-103,U+110-111,U+128-129,U+168-169,U+1A0-1A1,U+1AF-1B0,U+300-301,U+303-304,U+308-309,U+323,U+329,U+1EA0-1EF9,U+20AB}@font-face{font-family:Mona Sans Variable;font-style:normal;font-display:swap;font-weight:200 900;src:url(/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2)format("woff2-variations");unicode-range:U+100-2BA,U+2BD-2C5,U+2C7-2CC,U+2CE-2D7,U+2DD-2FF,U+304,U+308,U+329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:Mona Sans Variable;font-style:normal;font-display:swap;font-weight:200 900;src:url(/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2)format("woff2-variations");unicode-range:U+??,U+131,U+152-153,U+2BB-2BC,U+2C6,U+2DA,U+2DC,U+304,U+308,U+329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}:root{--ground:#faf9f4;--ink:#141413;--body:#54534e;--muted:#696862;--rule:#c7c5be;--hairline:#14141324;--wash:#1414130d;--wash-strong:#14141317;--accent:var(--ink);--sans:"Mona Sans Variable", system-ui, -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;--display:var(--sans);--mono:ui-monospace, "SF Mono", Menlo, Consolas, "Courier New", monospace;color:var(--ink);background:var(--ground);font-family:var(--sans);font-synthesis:none;font-size:16px;font-weight:400;line-height:1.55}*{box-sizing:border-box}[hidden]{display:none!important}body{background:var(--ground);margin:0}body.modal-open{overflow:hidden}button,input,select{font:inherit;color:inherit}button{cursor:pointer}button:disabled{cursor:default}code,pre{font-family:var(--mono);font-variant-ligatures:none}code{overflow-wrap:anywhere;font-size:.875em}:focus-visible{outline:2px solid var(--accent);outline-offset:2px;border-radius:3px}h1,h2,h3{font-family:var(--display);text-wrap:pretty;margin:0;font-weight:500}p{margin:0}.link{text-align:left;text-decoration:underline;-webkit-text-decoration-color:var(--rule);text-decoration-color:var(--rule);text-underline-offset:3px;background:0 0;border:0;padding:0;text-decoration-thickness:1px}.link:hover{text-decoration-color:currentColor}.link:disabled{color:var(--muted);text-decoration:none}.link-row{flex-wrap:wrap;align-items:center;gap:8px 18px;margin-top:16px;display:flex}.pill{background:var(--ink);min-height:38px;color:var(--ground);border:0;border-radius:19px;align-items:center;padding:0 18px;font-size:15px;display:inline-flex}.pill:hover{background:#30302e}.pill:disabled{background:var(--wash-strong);color:var(--muted)}.pill.quiet{background:var(--wash-strong);color:var(--ink)}.filter,.tabs button{color:var(--muted);background:0 0;border:0;border-bottom:1.5px solid #0000;padding:0 0 3px;font-size:13.5px}.filter:hover,.tabs button:hover{color:var(--ink)}.filter[aria-pressed=true],.tabs button[aria-selected=true]{color:var(--ink);border-bottom-color:var(--accent)}.field-line{border:0;border-bottom:1px solid var(--rule);background:0 0;border-radius:0;width:100%;min-height:36px;padding:0;font-size:14.5px}.field-line:focus{border-bottom-color:var(--ink);outline:0}.field-line::placeholder{color:var(--muted)}select{border:0;border-bottom:1px solid var(--rule);background:0 0;border-radius:0;min-height:34px;padding:0 2px;font-size:14.5px}.visually-labelled{display:block}.visually-labelled>span{clip-path:inset(50%);white-space:nowrap;width:1px;height:1px;position:absolute;overflow:hidden}.checkbox{color:var(--body);align-items:center;gap:8px;font-size:13.5px;display:flex}.checkbox input{accent-color:var(--accent);margin:0}.masthead{flex-wrap:wrap;align-items:baseline;gap:10px 36px;max-width:1360px;margin:0 auto;padding:24px 56px 0;display:flex}.brand{font-family:var(--display);font-size:19px;font-weight:600}.tabs{flex-wrap:wrap;gap:4px 24px;display:flex}.tabs button{font-size:15px}.run-controls{align-items:baseline;gap:24px;margin-left:auto;font-size:15px;display:flex}.run-menu{position:relative}.popover{z-index:5;border:1px solid var(--hairline);width:400px;max-width:calc(100vw - 40px);color:var(--ink);background:#fffefb;border-radius:12px;gap:10px;padding:20px;font-size:14.5px;display:grid;position:absolute;top:calc(100% + 10px);right:0;box-shadow:0 8px 28px #14141314}.popover-field{color:var(--body);gap:2px;font-size:13px;display:grid}.literal-line{overflow-wrap:anywhere;margin-top:6px;font-size:13px}.page{max-width:1360px;margin:0 auto;padding:48px 56px 88px}.blank{max-width:640px;margin:40px auto 0}.progress{background:var(--hairline);height:2px;margin-top:24px;position:relative;overflow:hidden}.progress i{background:var(--ink);width:30%;animation:1.6s ease-in-out infinite alternate progress;position:absolute;top:0;bottom:0;left:0}@keyframes progress{to{left:70%}}@media (prefers-reduced-motion:reduce){.progress i{opacity:.4;width:100%;animation:none}}.kicker{color:var(--muted);flex-wrap:wrap;justify-content:space-between;gap:4px 16px;margin-bottom:8px;font-size:14.5px;display:flex}.kicker .link{color:var(--ink)}.headline{letter-spacing:-.012em;overflow-wrap:anywhere;font-size:36px;font-weight:500;line-height:1.2}.headline code{letter-spacing:0;font-size:.82em;font-weight:400}.dek{color:var(--body);margin-top:12px}.also{color:var(--body);flex-wrap:wrap;align-items:baseline;gap:4px 12px;margin-top:6px;font-size:14.5px;display:flex}.also .link{color:var(--ink)}.section{margin-top:48px}.section h2{font-size:23px;line-height:1.3}.section-heading{flex-wrap:wrap;justify-content:space-between;align-items:baseline;gap:6px 16px;margin-bottom:12px;display:flex}.section-note{color:var(--body);font-size:14.5px}.section>.section-note{margin:6px 0 16px}.quiet-note{color:var(--muted);margin-top:12px;font-size:12.5px;line-height:1.4}.loading{color:var(--muted);margin:12px 0;font-size:14.5px}.unavailable,.muted{color:var(--muted);font-style:italic}.notice{border-left:2px solid var(--ink);flex-wrap:wrap;gap:4px 12px;margin:12px 0;padding-left:12px;font-size:14.5px;display:flex}.more{border-top:1px solid var(--hairline);color:var(--body);margin-top:14px;padding-top:12px;font-size:13.5px}.more summary{cursor:pointer;width:fit-content}.more[open]>summary{color:var(--ink);margin-bottom:10px}.facts{margin:0}.facts>div{border-top:1px solid var(--hairline);grid-template-columns:180px minmax(0,1fr);gap:2px 24px;padding:11px 0;font-size:14.5px;display:grid}.facts dd{margin:0}.fields{grid-template-columns:repeat(auto-fill,minmax(190px,1fr));gap:10px 20px;margin:10px 0;display:grid}.fields dt{color:var(--muted);font-size:12.5px}.fields dd{overflow-wrap:anywhere;color:var(--ink);margin:0;font-size:14px}.table-scroll{margin:12px 0;overflow-x:auto}.benchmark table,.trajectory-summary table{white-space:nowrap}table{border-collapse:collapse;width:100%;font-size:13.5px;line-height:1.45}caption{color:var(--ink);text-align:left;padding-bottom:6px;font-weight:500}th,td{border-top:1px solid var(--hairline);text-align:left;vertical-align:top;font-variant-numeric:tabular-nums;padding:8px 16px 8px 0;font-weight:400}thead th{color:var(--muted);border-top:0;font-size:12px}th:last-child,td:last-child{padding-right:0}.scrim{z-index:10;background:var(--ground);position:fixed;inset:0;overflow-y:auto}.modal{max-width:1160px;min-height:100%;margin:0 auto;padding:40px 56px 88px}.modal:focus,[role=tabpanel]:focus{outline:0}.modal-heading{flex-wrap:wrap;justify-content:space-between;align-items:baseline;gap:8px 24px;margin-bottom:12px;display:flex}.sensitivity{grid-column:1/-1;min-width:0}.sensitivity .section-note{margin:12px 0}.sensitivity code{overflow-wrap:anywhere}.sensitivity table{white-space:nowrap}.events{grid-template-columns:minmax(0,320px) minmax(0,820px);justify-content:space-between;align-items:start;gap:48px 72px;display:grid}.filters{flex-wrap:wrap;gap:4px 16px;margin:14px 0 10px;display:flex}.event-list,.feature-list,.responses{margin:0;padding:0;list-style:none}.event-list{margin:0 -10px}.feature-list{max-height:320px;margin:8px -10px 0;overflow-y:auto}.event-row{width:100%;color:var(--ink);text-align:left;background:0 0;border:0;border-radius:8px;grid-template-columns:26px minmax(0,1fr) auto;align-items:baseline;gap:0 8px;padding:7px 10px;display:grid}.event-row:hover{background:var(--wash)}.event-row[aria-pressed=true]{background:var(--wash-strong)}.event-rank,.event-change{color:var(--muted);font-variant-numeric:tabular-nums;white-space:nowrap;font-size:12.5px}.event-feature{text-overflow:ellipsis;white-space:nowrap;font-size:13px;overflow:hidden}.event-row.behavioral .event-feature{font-weight:600}.event-row.contextual .event-feature{color:var(--body)}.event-row.unavailable{font-style:normal}.event-row.unavailable .event-feature{color:var(--muted)}.pagination{color:var(--body);flex-wrap:wrap;justify-content:space-between;gap:4px 12px;margin-top:10px;font-size:13px;display:flex}.filter-fields{gap:10px;display:grid}.filter-fields label{gap:2px;font-size:13px;display:grid}.event-detail{min-width:0}.responses li{border-top:1px solid var(--hairline)}.response-row{width:100%;color:var(--ink);text-align:left;background:0 0;border:0;grid-template-columns:minmax(0,1fr) minmax(0,.6fr) minmax(0,.6fr) minmax(0,1.4fr) auto;align-items:baseline;gap:0 16px;padding:11px 8px;font-size:14.5px;display:grid}.response-row:hover{background:var(--wash)}.response-row span{color:var(--body)}.response-row .read{color:var(--ink);text-decoration:underline;-webkit-text-decoration-color:var(--rule);text-decoration-color:var(--rule);text-underline-offset:3px}.response-missing{padding:11px 8px;font-size:14.5px}.figure{margin:36px 0 0}.chart{max-width:100%;height:auto;display:block;overflow:visible}.chart text{fill:var(--muted);font-family:var(--sans);font-variant-numeric:tabular-nums;font-size:12px}.chart .marker-label{fill:var(--body)}.chart .marker-label.strong,.chart .missing-mark{fill:var(--ink)}.gridline{stroke:var(--hairline);stroke-width:1px}.gap-band{fill:var(--wash)}.onset-range{fill:var(--wash-strong)}.error-band{fill:#1414131c}.error-bar{fill:none;stroke:var(--muted);stroke-width:1px}.mean-line{fill:none;stroke:var(--ink);stroke-width:1.75px;stroke-linejoin:round}.mean-point,.retained,.coverage-point{fill:var(--ink)}.coverage-point{stroke:var(--ink);stroke-width:.75px}.uncertain-point{fill:var(--ground);stroke:var(--ink);stroke-width:1.25px}.unretained{fill:none;stroke:var(--ink);stroke-width:1.25px}.track-missing{fill:var(--rule)}.marker{stroke:var(--ink);stroke-width:1px}.marker.onset{stroke:var(--muted);stroke-dasharray:3 3}figcaption{color:var(--body);margin-top:10px;font-size:13.5px;line-height:1.45}figcaption strong{color:var(--ink);font-weight:600}figcaption .link{color:var(--ink)}@media (width<=960px){.masthead,.page,.modal{padding-left:20px;padding-right:20px}.events{grid-template-columns:minmax(0,1fr)}.event-list{max-height:300px;overflow-y:auto}.headline{font-size:29px}.run-controls{margin-left:0}.popover{width:auto;max-width:none;max-height:calc(100dvh - 24px);position:fixed;top:12px;left:20px;right:20px;overflow-y:auto}.link,.filter,.tabs button,.event-row,.response-row,.pill{min-height:44px}.facts>div{grid-template-columns:minmax(0,1fr)}.response-row{grid-template-columns:minmax(0,1fr) auto}.response-row span:not(.read){display:none}}.reader-top{overflow-wrap:anywhere;align-items:baseline}.reader-paging{flex-wrap:wrap;align-items:baseline;gap:4px 16px;display:flex}.reader .kicker .link:disabled{color:var(--muted)}.reader .headline{margin-top:20px}.reader .dek{max-width:760px}.reader-columns{grid-template-columns:repeat(2,minmax(0,1fr));gap:40px 48px;margin-top:40px;display:grid}.reader-columns.single{grid-template-columns:minmax(0,1fr);max-width:820px}.reader-column-heading{border-bottom:1px solid var(--rule);flex-wrap:wrap;justify-content:space-between;align-items:baseline;gap:4px 12px;padding:0 8px 10px;display:flex}.reader-column-heading h2{overflow-wrap:anywhere;font-size:20px;line-height:1.3}.reader-column-heading h2 code{font-size:.85em;font-weight:400}.reader-identity{color:var(--body);font-size:13px}.reader-aside{margin:16px 8px 0}pre.response{margin:12px 0 0;font-size:13.5px;line-height:1.65}.reader-line{white-space:pre-wrap;overflow-wrap:anywhere;border-radius:3px;padding:0 8px;display:block}.reader-line.differs{background:var(--wash-strong)}.reader-line.differs:empty{min-height:1.65em}.reader-provenance{border-top:1px solid var(--hairline);color:var(--body);margin-top:48px;padding-top:14px;font-size:13px;line-height:1.5}.reader-provenance-line{flex-wrap:wrap;justify-content:space-between;gap:12px 40px;display:flex}.reader-provenance-line p{flex:480px}.reader-provenance-line .checkbox{align-self:flex-start}.reader .more h3{color:var(--ink);margin-top:20px;font-size:14.5px}.reader .more .section-note{font-size:13.5px}.reader .sources{margin:8px 0 0;padding:0;list-style:none}.reader .sources li{border-top:1px solid var(--hairline);gap:2px;padding:8px 0;display:grid}@media (width<=960px){.reader-columns{grid-template-columns:minmax(0,1fr)}.reader-paging{align-items:center}.reader-provenance-line .checkbox{min-height:44px}.reader .more summary{padding-block:12px}}.compare{max-width:980px;margin:0 auto}.headline.compare-title{line-height:1.35}.compare-step{border:0;border-bottom:1px solid var(--rule);max-width:100%;font:inherit;font-variant-numeric:tabular-nums;letter-spacing:inherit;text-align:center;background:0 0;border-radius:0;padding:0}.compare-step:focus{border-bottom-color:var(--ink);outline:0}.compare-timeline{margin-top:16px}.compare-timeline svg{cursor:crosshair;touch-action:none;-webkit-user-select:none;user-select:none;display:block;overflow:visible}.compare-observed{stroke:var(--rule);stroke-width:2px;stroke-linecap:round}.compare-window line{stroke-width:5px;stroke-linecap:round}.compare-window.a line{stroke:var(--muted)}.compare-window.b line{stroke:var(--ink)}.compare-window text{fill:var(--body);font-family:var(--sans);font-size:12px}.compare-window.b text{fill:var(--ink)}.compare-axis{color:var(--muted);font-variant-numeric:tabular-nums;overflow-wrap:anywhere;grid-template-columns:minmax(0,1fr) auto minmax(0,1fr);gap:12px;font-size:12px;display:grid}.compare-axis span:last-child{text-align:right}.compare-controls{color:var(--body);flex-wrap:wrap;align-items:baseline;gap:10px 20px;margin-top:14px;font-size:14px;display:flex}.compare-drag,.compare-from{align-items:baseline;gap:8px;min-width:0;max-width:100%;display:flex}.compare-from select{min-width:0;max-width:100%;color:var(--ink)}.compare-controls .pill{margin-left:auto}.compare-stale{color:var(--body);text-align:right;margin-top:10px;font-size:13.5px;font-style:italic}.compare-results.stale{opacity:.4}.compare-overlap{max-width:720px;margin-top:36px}.compare-overlap strong{font-weight:500}.compare-overlap-note{max-width:720px;margin-top:6px}.compare-results>.section{margin-top:36px}.compare .compare-search{width:220px;max-width:100%}.figure.compare-figure{margin-top:8px}.link-row.compare-limit{color:var(--body);margin-top:4px;font-size:13.5px}.compare-limit .link{color:var(--ink)}.compare-table{table-layout:fixed;min-width:720px}.compare-table caption{color:var(--muted);font-size:12.5px;font-weight:400}.compare-table thead th:first-child{width:30%}.compare-table thead th:nth-child(2),.compare-table thead th:nth-child(3),.compare-table thead th:nth-child(4){width:10%}.compare-table thead th:nth-child(6){width:16%}.compare-no-interval{color:var(--muted);font-size:12px;font-style:italic;line-height:1.3;display:block}.compare-table thead th{vertical-align:bottom}.compare-table tbody th,.compare-table tbody td{vertical-align:middle}.compare-table .num{text-align:right}.compare-table .link code{font-size:13px}.compare-scale{justify-content:space-between;gap:8px;display:flex}.compare-plot{display:block;overflow:visible}.compare-zero{stroke:var(--rule);stroke-width:1px}.compare-plot .error-bar{stroke-width:1.5px}.compare-table .compare-detail{vertical-align:top;padding:4px 0 24px}.compare-detail>.quiet-note{margin:4px 0 12px}.compare-detail h3{margin-bottom:4px;font-size:13.5px;line-height:1.45}.compare-estimates,.compare-citations{grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:16px 28px;display:grid}.compare-estimates{margin-bottom:14px}.compare-citations{grid-template-columns:repeat(auto-fit,minmax(240px,1fr));margin-top:14px}.compare-estimate dl{margin:0}.compare-estimate dl>div{justify-content:space-between;gap:12px;display:flex}.compare-estimate dt{color:var(--muted)}.compare-estimate dd{overflow-wrap:anywhere;text-align:right;margin:0}.compare-reads{margin:6px 0 0;padding:0;list-style:none}.compare-reads li{flex-wrap:wrap;align-items:baseline;gap:2px 12px;margin-top:4px;display:flex}.compare-reads li span{color:var(--muted)}@media (width<=960px){.compare-step,.compare-from select,.compare .compare-search{min-height:44px}.compare-controls,.compare-drag,.compare-from{align-items:center}.compare-controls .pill{margin-left:0}.compare-stale{text-align:left}.compare .more summary{padding:12px 0}}.saved{max-width:880px;margin:0 auto}.saved .section>h2{margin-bottom:6px}.saved h3{font-size:15px;line-height:1.4}.saved td,.saved tbody th{overflow-wrap:anywhere}.saved-hypothesis{white-space:pre-wrap;overflow-wrap:anywhere;max-width:720px;font-size:20px;line-height:1.45}.saved-conditions th[scope=row]{width:170px;min-width:130px}.saved-conditions thead th:not(:first-child){min-width:170px}.saved-aside{color:var(--muted);margin-top:2px;font-size:12.5px;display:block}.saved-recorded{margin-top:14px}.saved-run{margin-top:20px}.saved-run .section-heading{margin-bottom:4px}.saved-edit{grid-template-columns:repeat(auto-fit,minmax(min(300px,100%),1fr));gap:24px 40px;margin-top:16px;display:grid}.saved-edit h3{margin-bottom:8px}.saved pre.response{color:var(--ink);white-space:pre-wrap;overflow-wrap:anywhere;margin:0;font-size:13px;line-height:1.65}.picker{max-width:680px;margin:0 auto}.picker .dek{margin-top:0}.picker-path{color:var(--body);flex-wrap:wrap;align-items:baseline;gap:4px 16px;margin-top:24px;font-size:14.5px;display:flex}.picker-list{max-height:40vh;margin:12px 0 0;padding:0;list-style:none;overflow-y:auto}.picker-list li{border-top:1px solid var(--hairline)}.picker-list li.selected{background:var(--wash-strong);border-top-color:#0000;border-radius:8px}.picker-entry,.picker-row{flex-wrap:wrap;justify-content:space-between;align-items:baseline;gap:2px 16px;width:100%;padding:11px 10px;font-size:14px;display:flex}.picker-entry{color:var(--ink);text-align:left;background:0 0;border:0;border-radius:8px}.picker-entry:hover:not(:disabled){background:var(--wash)}.picker-entry:disabled{color:var(--muted)}.picker-entry code,.picker-row code{min-width:0}.picker-side{color:var(--muted);flex-wrap:wrap;align-items:baseline;gap:2px 14px;font-size:13px;display:flex}.picker-side .link{color:var(--body)}.picker-form{margin-top:28px}.picker-fields{flex-wrap:wrap;gap:20px 32px;margin-top:20px;display:flex}.picker-field{min-width:0;color:var(--body);flex-direction:column;flex:200px;gap:2px;font-size:13px;display:flex}.picker-field.wide{flex-basis:100%}.picker-field input,.picker-field select{width:100%;color:var(--ink)}.picker-field input:disabled,.picker-field select:disabled{color:var(--muted)}.picker-field input[type=checkbox]{width:auto}.picker-actions{flex-wrap:wrap;justify-content:flex-end;align-items:center;gap:8px 20px;margin-top:24px;display:flex}@media (width<=960px){.saved .more summary{padding:11px 0}.picker-entry,.picker-row,.picker-field .field-line,.picker-field select{min-height:44px}}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Copyright 2022 The Mona Sans Project Authors (https://github.com/github/mona-sans), with Reserved Font Name "Mona" MonaSans-Italic[wdth,wght].ttf: Copyright 2022 The Mona Sans Project Authors (https://github.com/github/mona-sans), with Reserved Font Name "Mona"
|
|
2
|
+
|
|
3
|
+
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
|
4
|
+
This license is copied below, and is also available with a FAQ at:
|
|
5
|
+
http://scripts.sil.org/OFL
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
-----------------------------------------------------------
|
|
9
|
+
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
|
10
|
+
-----------------------------------------------------------
|
|
11
|
+
|
|
12
|
+
PREAMBLE
|
|
13
|
+
The goals of the Open Font License (OFL) are to stimulate worldwide
|
|
14
|
+
development of collaborative font projects, to support the font creation
|
|
15
|
+
efforts of academic and linguistic communities, and to provide a free and
|
|
16
|
+
open framework in which fonts may be shared and improved in partnership
|
|
17
|
+
with others.
|
|
18
|
+
|
|
19
|
+
The OFL allows the licensed fonts to be used, studied, modified and
|
|
20
|
+
redistributed freely as long as they are not sold by themselves. The
|
|
21
|
+
fonts, including any derivative works, can be bundled, embedded,
|
|
22
|
+
redistributed and/or sold with any software provided that any reserved
|
|
23
|
+
names are not used by derivative works. The fonts and derivatives,
|
|
24
|
+
however, cannot be released under any other type of license. The
|
|
25
|
+
requirement for fonts to remain under this license does not apply
|
|
26
|
+
to any document created using the fonts or their derivatives.
|
|
27
|
+
|
|
28
|
+
DEFINITIONS
|
|
29
|
+
"Font Software" refers to the set of files released by the Copyright
|
|
30
|
+
Holder(s) under this license and clearly marked as such. This may
|
|
31
|
+
include source files, build scripts and documentation.
|
|
32
|
+
|
|
33
|
+
"Reserved Font Name" refers to any names specified as such after the
|
|
34
|
+
copyright statement(s).
|
|
35
|
+
|
|
36
|
+
"Original Version" refers to the collection of Font Software components as
|
|
37
|
+
distributed by the Copyright Holder(s).
|
|
38
|
+
|
|
39
|
+
"Modified Version" refers to any derivative made by adding to, deleting,
|
|
40
|
+
or substituting -- in part or in whole -- any of the components of the
|
|
41
|
+
Original Version, by changing formats or by porting the Font Software to a
|
|
42
|
+
new environment.
|
|
43
|
+
|
|
44
|
+
"Author" refers to any designer, engineer, programmer, technical
|
|
45
|
+
writer or other person who contributed to the Font Software.
|
|
46
|
+
|
|
47
|
+
PERMISSION & CONDITIONS
|
|
48
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
49
|
+
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
|
50
|
+
redistribute, and sell modified and unmodified copies of the Font
|
|
51
|
+
Software, subject to the following conditions:
|
|
52
|
+
|
|
53
|
+
1) Neither the Font Software nor any of its individual components,
|
|
54
|
+
in Original or Modified Versions, may be sold by itself.
|
|
55
|
+
|
|
56
|
+
2) Original or Modified Versions of the Font Software may be bundled,
|
|
57
|
+
redistributed and/or sold with any software, provided that each copy
|
|
58
|
+
contains the above copyright notice and this license. These can be
|
|
59
|
+
included either as stand-alone text files, human-readable headers or
|
|
60
|
+
in the appropriate machine-readable metadata fields within text or
|
|
61
|
+
binary files as long as those fields can be easily viewed by the user.
|
|
62
|
+
|
|
63
|
+
3) No Modified Version of the Font Software may use the Reserved Font
|
|
64
|
+
Name(s) unless explicit written permission is granted by the corresponding
|
|
65
|
+
Copyright Holder. This restriction only applies to the primary font name as
|
|
66
|
+
presented to the users.
|
|
67
|
+
|
|
68
|
+
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
|
69
|
+
Software shall not be used to promote, endorse or advertise any
|
|
70
|
+
Modified Version, except to acknowledge the contribution(s) of the
|
|
71
|
+
Copyright Holder(s) and the Author(s) or with their explicit written
|
|
72
|
+
permission.
|
|
73
|
+
|
|
74
|
+
5) The Font Software, modified or unmodified, in part or in whole,
|
|
75
|
+
must be distributed entirely under this license, and must not be
|
|
76
|
+
distributed under any other license. The requirement for fonts to
|
|
77
|
+
remain under this license does not apply to any document created
|
|
78
|
+
using the Font Software.
|
|
79
|
+
|
|
80
|
+
TERMINATION
|
|
81
|
+
This license becomes null and void if any of the above conditions are
|
|
82
|
+
not met.
|
|
83
|
+
|
|
84
|
+
DISCLAIMER
|
|
85
|
+
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
86
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
|
87
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
|
88
|
+
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
|
89
|
+
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
90
|
+
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
|
91
|
+
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
92
|
+
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
|
93
|
+
OTHER DEALINGS IN THE FONT SOFTWARE.
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
Third-party components in the Rollout Diff web application
|
|
2
|
+
|
|
3
|
+
react 19.3.1, react-dom 19.3.1 and scheduler 0.28.0
|
|
4
|
+
|
|
5
|
+
MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
|
|
27
|
+
vite 8.3.6 (module preload polyfill)
|
|
28
|
+
|
|
29
|
+
MIT License
|
|
30
|
+
|
|
31
|
+
Copyright (c) 2019-present, VoidZero Inc. and Vite contributors
|
|
32
|
+
|
|
33
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
34
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
35
|
+
in the Software without restriction, including without limitation the rights
|
|
36
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
37
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
38
|
+
furnished to do so, subject to the following conditions:
|
|
39
|
+
|
|
40
|
+
The above copyright notice and this permission notice shall be included in all
|
|
41
|
+
copies or substantial portions of the Software.
|
|
42
|
+
|
|
43
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
44
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
45
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
46
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
47
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
48
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
49
|
+
SOFTWARE.
|
|
50
|
+
|
|
51
|
+
rolldown 1.2.16 (CommonJS runtime helper)
|
|
52
|
+
|
|
53
|
+
MIT License
|
|
54
|
+
|
|
55
|
+
Copyright (c) 2024-present VoidZero Inc. & Contributors
|
|
56
|
+
|
|
57
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
58
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
59
|
+
in the Software without restriction, including without limitation the rights
|
|
60
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
61
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
62
|
+
furnished to do so, subject to the following conditions:
|
|
63
|
+
|
|
64
|
+
The above copyright notice and this permission notice shall be included in all
|
|
65
|
+
copies or substantial portions of the Software.
|
|
66
|
+
|
|
67
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
68
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
69
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
70
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
71
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
72
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
73
|
+
SOFTWARE.
|
|
74
|
+
|
|
75
|
+
Additional notices provided by Rolldown for code derived from Rollup and esbuild
|
|
76
|
+
|
|
77
|
+
The MIT License (MIT)
|
|
78
|
+
|
|
79
|
+
Copyright (c) 2017 [these people](https://github.com/rollup/rollup/graphs/contributors)
|
|
80
|
+
|
|
81
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
82
|
+
|
|
83
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
84
|
+
|
|
85
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
MIT License
|
|
90
|
+
|
|
91
|
+
Copyright (c) 2020 Evan Wallace
|
|
92
|
+
|
|
93
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
94
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
95
|
+
in the Software without restriction, including without limitation the rights
|
|
96
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
97
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
98
|
+
furnished to do so, subject to the following conditions:
|
|
99
|
+
|
|
100
|
+
The above copyright notice and this permission notice shall be included in all
|
|
101
|
+
copies or substantial portions of the Software.
|
|
102
|
+
|
|
103
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
104
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
105
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
106
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
107
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
108
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
109
|
+
SOFTWARE.
|
|
110
|
+
|
|
111
|
+
Mona Sans font files and stylesheet (@fontsource-variable/mona-sans 5.3.0)
|
|
112
|
+
The complete copyright and SIL Open Font License are in mona-sans-OFL.txt.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
6
|
+
<meta name="color-scheme" content="light" />
|
|
7
|
+
<title>Rollout Diff | Investigation</title>
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-D8uGEQ9w.js"></script>
|
|
9
|
+
<link rel="stylesheet" crossorigin href="/assets/index-Dvo9nx9b.css">
|
|
10
|
+
</head>
|
|
11
|
+
<body>
|
|
12
|
+
<div id="root"></div>
|
|
13
|
+
</body>
|
|
14
|
+
</html>
|
rolloutdiff/adapters.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Text-only TRL/verl adapters and an explicit mapping for tabular input.
|
|
2
|
+
|
|
3
|
+
Layout references:
|
|
4
|
+
https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py (log)
|
|
5
|
+
https://github.com/verl-project/verl/blob/v0.9.1/verl/trainer/ppo/ray_trainer.py (_write_generations)
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import gzip
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
import zlib
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from .schema import CANONICAL_COLUMNS, validate_rollouts
|
|
23
|
+
|
|
24
|
+
_TRL_FILE = re.compile(r"completions_(\d+)\.parquet\Z")
|
|
25
|
+
_VERL_FILE = re.compile(r"(?:step_)?(\d+)\.jsonl(?:\.gz)?\Z")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _plain(value: Any) -> Any:
|
|
29
|
+
"""Convert Arrow's nested numpy arrays to ordinary JSON-shaped containers."""
|
|
30
|
+
if isinstance(value, np.ndarray):
|
|
31
|
+
return [_plain(x) for x in value.tolist()]
|
|
32
|
+
if isinstance(value, list):
|
|
33
|
+
return [_plain(x) for x in value]
|
|
34
|
+
if isinstance(value, dict):
|
|
35
|
+
return {k: _plain(v) for k, v in value.items()}
|
|
36
|
+
if isinstance(value, np.generic):
|
|
37
|
+
return value.item()
|
|
38
|
+
return value
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _text(value: Any) -> tuple[str, list[dict] | None]:
|
|
42
|
+
value = _plain(value)
|
|
43
|
+
if isinstance(value, str):
|
|
44
|
+
return value, None
|
|
45
|
+
if not isinstance(value, list):
|
|
46
|
+
raise ValueError("content must be text or a list of chat messages")
|
|
47
|
+
parts = []
|
|
48
|
+
for message in value:
|
|
49
|
+
if not isinstance(message, dict) or not isinstance(message.get("role"), str):
|
|
50
|
+
raise ValueError("chat content requires messages with a role")
|
|
51
|
+
content = message.get("content")
|
|
52
|
+
if content is None and message.get("tool_calls"):
|
|
53
|
+
content = ""
|
|
54
|
+
if isinstance(content, str):
|
|
55
|
+
parts.append(content)
|
|
56
|
+
elif isinstance(content, list):
|
|
57
|
+
for block in content:
|
|
58
|
+
if (
|
|
59
|
+
not isinstance(block, dict)
|
|
60
|
+
or block.get("type") != "text"
|
|
61
|
+
or not isinstance(block.get("text"), str)
|
|
62
|
+
):
|
|
63
|
+
raise ValueError("only text content blocks are supported")
|
|
64
|
+
parts.append(block["text"])
|
|
65
|
+
else:
|
|
66
|
+
raise ValueError("chat content must be text")
|
|
67
|
+
return "\n".join(parts), value
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _identity(value: Any) -> str:
|
|
71
|
+
value = _plain(value)
|
|
72
|
+
if isinstance(value, str) and value.strip():
|
|
73
|
+
return value
|
|
74
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
75
|
+
return str(value)
|
|
76
|
+
raise ValueError("identity column must contain nonempty strings or integers")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def from_table(
|
|
80
|
+
table: pd.DataFrame,
|
|
81
|
+
*,
|
|
82
|
+
run: str,
|
|
83
|
+
columns: Mapping[str, str] | None = None,
|
|
84
|
+
source: str = "table",
|
|
85
|
+
) -> pd.DataFrame:
|
|
86
|
+
"""Map canonical names to input columns; discard other input columns.
|
|
87
|
+
|
|
88
|
+
`step`, `prompt`, `response`, `reward` default to their canonical names.
|
|
89
|
+
Optional `sample_id`, `prompt_id`, and `tool_calls` columns are used when
|
|
90
|
+
present or explicitly mapped. Generated sample IDs are source:row-position.
|
|
91
|
+
"""
|
|
92
|
+
if not table.columns.is_unique:
|
|
93
|
+
raise ValueError("duplicate input column names")
|
|
94
|
+
allowed = {
|
|
95
|
+
"step",
|
|
96
|
+
"prompt",
|
|
97
|
+
"response",
|
|
98
|
+
"reward",
|
|
99
|
+
"sample_id",
|
|
100
|
+
"prompt_id",
|
|
101
|
+
"tool_calls",
|
|
102
|
+
}
|
|
103
|
+
mapping = dict(columns or {})
|
|
104
|
+
if set(mapping) - allowed:
|
|
105
|
+
raise ValueError("unsupported canonical column mapping")
|
|
106
|
+
for name in ("step", "prompt", "response", "reward"):
|
|
107
|
+
mapping.setdefault(name, name)
|
|
108
|
+
for name in ("sample_id", "prompt_id", "tool_calls"):
|
|
109
|
+
if name in table:
|
|
110
|
+
mapping.setdefault(name, name)
|
|
111
|
+
missing = set(mapping.values()) - set(table.columns)
|
|
112
|
+
if missing:
|
|
113
|
+
raise ValueError(f"missing input columns: {sorted(missing)}")
|
|
114
|
+
rows = []
|
|
115
|
+
for position, raw in enumerate(table.to_dict("records")):
|
|
116
|
+
try:
|
|
117
|
+
prompt, prompt_messages = _text(raw[mapping["prompt"]])
|
|
118
|
+
response, response_messages = _text(raw[mapping["response"]])
|
|
119
|
+
identity_source = (
|
|
120
|
+
["chat", prompt_messages]
|
|
121
|
+
if prompt_messages is not None
|
|
122
|
+
else ["text", prompt]
|
|
123
|
+
)
|
|
124
|
+
digest = hashlib.sha256(
|
|
125
|
+
json.dumps(
|
|
126
|
+
identity_source,
|
|
127
|
+
ensure_ascii=False,
|
|
128
|
+
sort_keys=True,
|
|
129
|
+
separators=(",", ":"),
|
|
130
|
+
).encode("utf-8")
|
|
131
|
+
).hexdigest()
|
|
132
|
+
tool_calls = (
|
|
133
|
+
_plain(raw[mapping["tool_calls"]]) if "tool_calls" in mapping else None
|
|
134
|
+
)
|
|
135
|
+
if not isinstance(tool_calls, (list, dict)) and pd.isna(tool_calls):
|
|
136
|
+
tool_calls = None
|
|
137
|
+
if tool_calls is None and response_messages is not None:
|
|
138
|
+
tool_calls = [
|
|
139
|
+
call
|
|
140
|
+
for message in response_messages
|
|
141
|
+
for call in (message.get("tool_calls") or [])
|
|
142
|
+
] or None
|
|
143
|
+
rows.append(
|
|
144
|
+
{
|
|
145
|
+
"run": run,
|
|
146
|
+
"step": _plain(raw[mapping["step"]]),
|
|
147
|
+
"sample_id": _identity(raw[mapping["sample_id"]])
|
|
148
|
+
if "sample_id" in mapping
|
|
149
|
+
else f"{source}:{position}",
|
|
150
|
+
"prompt_id": _identity(raw[mapping["prompt_id"]])
|
|
151
|
+
if "prompt_id" in mapping
|
|
152
|
+
else digest,
|
|
153
|
+
"prompt": prompt,
|
|
154
|
+
"response": response,
|
|
155
|
+
"reward": _plain(raw[mapping["reward"]]),
|
|
156
|
+
"prompt_messages": prompt_messages,
|
|
157
|
+
"response_messages": response_messages,
|
|
158
|
+
"tool_calls": tool_calls,
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
except (ValueError, TypeError) as exc:
|
|
162
|
+
raise ValueError(f"{source}:{position + 1}: {exc}") from None
|
|
163
|
+
try:
|
|
164
|
+
return validate_rollouts(pd.DataFrame(rows, columns=CANONICAL_COLUMNS))
|
|
165
|
+
except ValueError as exc:
|
|
166
|
+
raise ValueError(f"{source}: {exc}") from None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _files(path: str | Path, pattern: re.Pattern, glob: str) -> list[Path]:
|
|
170
|
+
path = Path(path)
|
|
171
|
+
if not path.exists():
|
|
172
|
+
raise FileNotFoundError(path)
|
|
173
|
+
candidates = [path] if path.is_file() else list(path.glob(glob))
|
|
174
|
+
files = [p for p in candidates if p.is_file() and pattern.fullmatch(p.name)]
|
|
175
|
+
if not files:
|
|
176
|
+
raise ValueError(f"No matching rollout files in {path}")
|
|
177
|
+
return sorted(
|
|
178
|
+
files, key=lambda p: (int(pattern.fullmatch(p.name).group(1)), p.name)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _check_steps(table: pd.DataFrame, file: Path, pattern: re.Pattern) -> None:
|
|
183
|
+
expected = int(pattern.fullmatch(file.name).group(1))
|
|
184
|
+
if "step" not in table or not table.step.eq(expected).all():
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f"{file.name}: step column must agree with filename step {expected}"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def load_trl(
|
|
191
|
+
path: str | Path, *, run: str, reward_column: str | None = None
|
|
192
|
+
) -> pd.DataFrame:
|
|
193
|
+
"""Read completions_<step>.parquet, or a trainer output's completions/.
|
|
194
|
+
|
|
195
|
+
Only `reward` is selected automatically; specify reward_column for other columns.
|
|
196
|
+
"""
|
|
197
|
+
path = Path(path)
|
|
198
|
+
if path.is_dir() and (path / "completions").is_dir():
|
|
199
|
+
path = path / "completions"
|
|
200
|
+
parts = []
|
|
201
|
+
for file in _files(path, _TRL_FILE, "completions_*.parquet"):
|
|
202
|
+
table = pd.read_parquet(file)
|
|
203
|
+
selected_reward = reward_column or "reward"
|
|
204
|
+
if selected_reward not in table:
|
|
205
|
+
raise ValueError(
|
|
206
|
+
f"{file.name}: select an available reward_column explicitly"
|
|
207
|
+
)
|
|
208
|
+
_check_steps(table, file, _TRL_FILE)
|
|
209
|
+
parts.append(
|
|
210
|
+
from_table(
|
|
211
|
+
table,
|
|
212
|
+
run=run,
|
|
213
|
+
columns={"response": "completion", "reward": selected_reward},
|
|
214
|
+
source=file.name,
|
|
215
|
+
)
|
|
216
|
+
)
|
|
217
|
+
return (
|
|
218
|
+
validate_rollouts(pd.concat(parts, ignore_index=True))
|
|
219
|
+
.sort_values(["step", "sample_id"], kind="stable")
|
|
220
|
+
.reset_index(drop=True)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def load_verl(path: str | Path, *, run: str) -> pd.DataFrame:
|
|
225
|
+
"""Read verl rollout_data_dir/<step>.jsonl with input/output/score/step.
|
|
226
|
+
|
|
227
|
+
Also reads `step_<step>` names and gzip `.jsonl.gz` shards; steps may be
|
|
228
|
+
zero-padded.
|
|
229
|
+
"""
|
|
230
|
+
parts = []
|
|
231
|
+
for file in _files(path, _VERL_FILE, "*.jsonl*"):
|
|
232
|
+
rows = []
|
|
233
|
+
open_text = gzip.open if file.suffix == ".gz" else open
|
|
234
|
+
try:
|
|
235
|
+
with open_text(file, "rt", encoding="utf-8") as stream:
|
|
236
|
+
for line_number, line in enumerate(stream, 1):
|
|
237
|
+
try:
|
|
238
|
+
row = json.loads(line)
|
|
239
|
+
if not isinstance(row, dict):
|
|
240
|
+
raise ValueError
|
|
241
|
+
except (ValueError, TypeError):
|
|
242
|
+
raise ValueError(
|
|
243
|
+
f"{file.name}:{line_number}: invalid JSON object"
|
|
244
|
+
) from None
|
|
245
|
+
rows.append(row)
|
|
246
|
+
except (EOFError, gzip.BadGzipFile, zlib.error):
|
|
247
|
+
raise ValueError(f"{file.name}: invalid gzip data") from None
|
|
248
|
+
table = pd.DataFrame(
|
|
249
|
+
rows, columns=None if rows else ["input", "output", "score", "step"]
|
|
250
|
+
)
|
|
251
|
+
_check_steps(table, file, _VERL_FILE)
|
|
252
|
+
parts.append(
|
|
253
|
+
from_table(
|
|
254
|
+
table,
|
|
255
|
+
run=run,
|
|
256
|
+
columns={"prompt": "input", "response": "output", "reward": "score"},
|
|
257
|
+
source=file.name,
|
|
258
|
+
)
|
|
259
|
+
)
|
|
260
|
+
return (
|
|
261
|
+
validate_rollouts(pd.concat(parts, ignore_index=True))
|
|
262
|
+
.sort_values(["step", "sample_id"], kind="stable")
|
|
263
|
+
.reset_index(drop=True)
|
|
264
|
+
)
|