parsearena 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. parsearena/__init__.py +3 -0
  2. parsearena/cli.py +288 -0
  3. parsearena/demo/pdfs/NOTICE.md +36 -0
  4. parsearena/demo/pdfs/census-income-2023.pdf +0 -0
  5. parsearena/demo/pdfs/dpo-arxiv-2305.18290.pdf +0 -0
  6. parsearena/demo/pdfs/naca-report-1135-scan.pdf +0 -0
  7. parsearena/demo/questions.yaml +104 -0
  8. parsearena/demo/recorded.json +304 -0
  9. parsearena/errors.py +14 -0
  10. parsearena/evaluate.py +130 -0
  11. parsearena/llm.py +154 -0
  12. parsearena/parsers.py +197 -0
  13. parsearena/pipeline.py +276 -0
  14. parsearena/questions.py +125 -0
  15. parsearena/render.py +40 -0
  16. parsearena/report.py +57 -0
  17. parsearena/report_template.html +450 -0
  18. parsearena/retrieval.py +197 -0
  19. parsearena/runners/_common.py +148 -0
  20. parsearena/runners/run_docling.py +25 -0
  21. parsearena/runners/run_marker.py +54 -0
  22. parsearena/runners/run_mineru.py +59 -0
  23. parsearena/runners/run_pypdf.py +22 -0
  24. parsearena/runners/run_unstructured.py +38 -0
  25. parsearena/text.py +90 -0
  26. parsearena-0.1.3.dist-info/METADATA +267 -0
  27. parsearena-0.1.3.dist-info/RECORD +30 -0
  28. parsearena-0.1.3.dist-info/WHEEL +4 -0
  29. parsearena-0.1.3.dist-info/entry_points.txt +2 -0
  30. parsearena-0.1.3.dist-info/licenses/LICENSE +21 -0
parsearena/demo/recorded.json ADDED
@@ -0,0 +1,304 @@
1
+ {
2
+ "meta": {
3
+ "tool": "parsearena",
4
+ "tool_version": "0.1.0",
5
+ "created": "2026-11-03T16:53:52Z",
6
+ "hardware": "Apple M4, 16 GB RAM, macOS 27.2",
7
+ "device_requested": "auto"
8
+ },
9
+ "parsers": [
10
+ {
11
+ "name": "docling",
12
+ "label": "Docling",
13
+ "version": "2.146.0",
14
+ "packages": [
15
+ "docling==2.146.0"
16
+ ],
17
+ "license": "MIT",
18
+ "url": "https://github.com/docling-project/docling"
19
+ },
20
+ {
21
+ "name": "marker",
22
+ "label": "Marker",
23
+ "version": "2.1.0",
24
+ "packages": [
25
+ "marker-pdf==2.1.0"
26
+ ],
27
+ "license": "Code Apache-2.0; model weights modified OpenRAIL-M",
28
+ "url": "https://github.com/datalab-to/marker"
29
+ },
30
+ {
31
+ "name": "mineru",
32
+ "label": "MinerU",
33
+ "version": "4.1.6",
34
+ "packages": [
35
+ "mineru==4.1.6"
36
+ ],
37
+ "license": "Apache-2.0 with additional terms",
38
+ "url": "https://github.com/opendatalab/MinerU"
39
+ },
40
+ {
41
+ "name": "unstructured",
42
+ "label": "Unstructured",
43
+ "version": "0.27.36",
44
+ "packages": [
45
+ "unstructured[pdf]==0.27.36"
46
+ ],
47
+ "license": "Apache-2.0",
48
+ "url": "https://github.com/Unstructured-IO/unstructured"
49
+ },
50
+ {
51
+ "name": "pypdf",
52
+ "label": "pypdf (baseline)",
53
+ "version": "6.21.4",
54
+ "packages": [
55
+ "pypdf==6.21.4"
56
+ ],
57
+ "license": "BSD-3-Clause",
58
+ "url": "https://github.com/py-pdf/pypdf"
59
+ }
60
+ ],
61
+ "docs": [
62
+ {
63
+ "id": "census-income-2023",
64
+ "file": "census-income-2023.pdf",
65
+ "num_pages": 3,
66
+ "title": "U.S. Census Bureau, Income in the United States: 2023 (P60-282), pp. 1, 2, 15",
67
+ "license": "Public domain (U.S. government work)",
68
+ "source": "https://www.census.gov/library/publications/2024/demo/p60-282.html"
69
+ },
70
+ {
71
+ "id": "dpo-arxiv-2305.18290",
72
+ "file": "dpo-arxiv-2305.18290.pdf",
73
+ "num_pages": 4,
74
+ "title": "Rafailov et al., Direct Preference Optimization (arXiv:2305.18290v3), pp. 1, 4, 5, 9",
75
+ "license": "CC BY 4.0",
76
+ "source": "https://arxiv.org/abs/2305.18290"
77
+ },
78
+ {
79
+ "id": "naca-report-1135-scan",
80
+ "file": "naca-report-1135-scan.pdf",
81
+ "num_pages": 1,
82
+ "title": "NACA Report 1135 (1953), p. 613, image-only scan",
83
+ "license": "Public domain (U.S. government work)",
84
+ "source": "https://ntrs.nasa.gov/citations/19930091059"
85
+ }
86
+ ],
87
+ "outputs": {
88
+ "docling": {
89
+ "census-income-2023": {
90
+ "pages": [
91
+ "## Income in the United States: 2023\n\n## INTRODUCTION\n\nThe U.S. Census Bureau collects data and publishes estimates on income, earnings, and inequality in order to evaluate national economic trends and to understand their effect on the wellbeing of households, families, and individuals.\n\nThe estimates in this report are based on data collected in the 2024 and earlier Current Population Survey Annual Social and Economic Supplements (CPS ASEC) conducted by the Census Bureau.* This report presents estimates on income in the United States for calendar year 2023. Historical estimates are expressed in real or 2023 dollars to account for the change in the cost of living over time. 1 The current method for inflation-adjustment is based on the Chained Consumer Price Index for all Urban Consumers (C-CPI-U) between 2000 and 2023 and the Consumer Price Index for all Urban Consumers Retroactive Series (R-CPI-U-RS) prior to 2000. 2 The C-CPI-U measured a 4.0 percent increase in consumer prices between 2022 and 2023, down from a 7.7 percent increase between 2021 and 2022.\n\n* The Census Bureau reviewed this data product to ensure appropriate access, use, and disclosure avoidance protection of the confidential source data used to produce this product (Data Management System [DMS] number: P-7534374, Disclosure Review Board [DRB] approval number: CBDRB-FY24-0434). To further protect respondent privacy, all estimates in this report have undergone additional rounding. As a result, this year's estimates may differ from previous publications and details may not sum to totals. All comparative statements have undergone statistical testing and are statistically significant at the 90 percent confidence level unless otherwise noted.\n\nThis report begins with a section discussing median household income, highlighting year-to-year comparisons by characteristics such as race and Hispanic origin, nativity, region, and education. This is followed by sections on income inequality, workers, and median earnings. The income estimates in the main sections of this report are based on the concept of money income, which is pretax and does not account for the value of in-kind transfers. Estimates of post-tax income and inequality are included in Appendix B.\n\nThis report is released alongside two other reports focused on poverty estimates and health insurance coverage in the United States. For poverty and health insurance estimates, refer to \"Poverty in the United States: 2023\" and \"Health Insurance Coverage in the United States: 2023.\" 3\n\n## Highlights\n\n- Real median household income was $80,610 in 2023, a 4.0 percent increase from the 2022 estimate of $77,540 (Figure 1 and Table A-1). This is the first statistically significant annual increase in real median household income since 2019. 4\n- Real median household incomes increased by 5.4 percent for White households and by 5.7 percent for non-Hispanic White households between 2022 and 2023. There was no significant change in median incomes for Black, Asian, and Hispanic households (Figure 2 and Table A-1).\n- Household income rose throughout the income distribution, increasing 6.7 percent at the 10th percentile and 4.6 percent at the 90th percentile (Table A-3).\n- Income inequality as measured by the Gini index and income percentile ratios was not significantly different between 2022 and 2023 (Figure 3 and Table A-3).\n- Between 2022 and 2023, the number of total workers increased by 2.2 million, or 1.3 percent. The change in the number of full-time, year-round workers was not statistically significant (Figure 5 and Table A-6).\n- Real median earnings for men who worked full-time, year-round increased by 3.0 percent, and real median earnings increased 1.5 percent for women who worked full-time, year-round (Figure 4 and Table A-6).\n- For full-time, year-round workers, the female-to-male earnings ratio in 2023 fell to 82.7 percent from 84.0 percent in 2022 (Figure 6 and Table A-6). This is the first statistically significant annual decrease in the female-to-male earnings ratio since 2003.\n\n## HOUSEHOLD INCOME BY SELECTED CHARACTERISTICS\n\nThis section focuses on median household income by selected characteristics of the householder, including race and Hispanic origin, nativity, region, and education. The householder is the person (or",
92
+ "Figure 1. Median Household Income and Percent Change by Selected Characteristics\n\n<!-- image -->\n\n1 Householders aged 25 and older. In 2023, the median household income for this group was $82,010.\n\nNote: Households as of March of the following year. Percent change estimate may be different due to rounded components. Statistically significant indicates the change is statistically different from zero at the 90 percent confidence level. Margins of error and other related estimates and notes are available in Table A-1. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at &lt;https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf&gt;.\n\nSource: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC).\n\nLorem ipsu\n\ndolor sit",
93
+ "## Table A-1.\n\n## Income Summary Measures by Selected Characteristics: 2022 and 2023\n\n(Income in 2023 dollars, adjusted using the C-CPI-U. Households as of March of the following year. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at &lt;https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf&gt;)\n\n| Characteristic | 2022 - Number (thousands) | 2022 - Median income (dollars) - Estimate | 2022 - Median income (dollars) - Margin of error 1 (±) | 2023 - Number (thou- - sands) | 2023 - Median income (dollars) - Estimate | 2023 - Median income (dollars) - Margin of error 1 (±) | Percent change in - median income (2023 less 2022)* , 2 - real Estimate | Percent change in - median income (2023 less 2022)* , 2 - Margin of error 1 (±) |\n|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|---------------------------------------------|----------------------------------------------------------|---------------------------------|---------------------------------------------|----------------------------------------------------------|---------------------------------------------------------------------------|-----------------------------------------------------------------------------------|\n| HOUSEHOLDS | | | | | | | | |\n| All households. . . . . . . . . . . . . . . . . . . . . | . . . . 131,400 | 77,540 | 1,006 | 132,200 | 80,610 | 634 | *4.0 | 1.40 |\n| Type of Household | | | | | | | | |\n| Family households. . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 84,330 | 99,250 | 996 | 84,680 | 102,800 | 1,200 | *3.6 | 1.41 |\n| Married-couple. . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 62,180 | 115,200 | 1,166 | 62,300 | 119,400 | 1,576 | *3.6 | 1.55 |\n| Female householder, no spouse present . . . | . . . . . 15,030 | 58,260 | 1,258 | 15,180 | 59,470 | 1,774 | 2.1 | 3.72 |\n| Male householder, no spouse present. . . . . . | . . . . . 7,128 | 76,550 | 2,824 | 7,208 | 81,890 | 2,059 | *7.0 | 4.10 |\n| Nonfamily households. . . . . . . . . . . . . . . . . . . . . . | . . . . . 47,100 | 47,250 | 861 | 47,530 | 49,600 | 895 | *5.0 | 2.53 |\n| Female householder . . . . . . . . . . . . . . . . . . . . . | . . . . . 24,360 | 41,790 | 812 | 24,680 | 42,140 | 947 | 0.8 | 2.74 |\n| Male householder. . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 22,740 | 53,990 | 1,029 | 22,850 | 57,200 | 1,190 | *5.9 | 2.81 |\n| Race 3 and Hispanic Origin of Householder | | | | | | | | |\n| White. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 101,400 | 80,320 | 905 | 101,900 | 84,630 | 1,182 | *5.4 | 1.60 |\n| White, not Hispanic. . . . . . . . . . . . . . . . . . . . . . | . . . . . 84,490 | 84,280 | 997 | 84,440 | 89,050 | 1,234 | *5.7 | 1.63 |\n| Black . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 18,080 | 54,960 | 1,528 | 18,040 | 56,490 | 1,328 | 2.8 | 3.65 |\n| Asian. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 7,609 | 113,100 | 4,040 | 7,655 | 112,800 | 4,187 | -0.2 | 4.69 |\n| Hispanic (any race). . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 19,320 | 65,300 | 1,659 | 19,860 | 65,540 | 1,259 | 0.4 | 2.77 |\n| Age of Householder | | | | | | | | |\n| Under 65 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 94,300 | 89,270 | 985 | 94,590 | 92,470 | 1,088 | *3.6 | 1.46 |\n| 15 to 24 years. . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 6,136 | 54,540 | 3,591 | 5,881 | 54,930 | 2,870 | 0.7 | 7.95 |\n| 25 to 34 years. . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 20,720 | 83,420 | 1,566 | 20,910 | 85,780 | 1,190 | *2.8 | 2.30 |\n| 35 to 44 years. . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 22,530 | 100,500 | 1,870 | 23,060 | 101,300 | 1,234 | 0.8 | 2.00 |\n| 45 to 54 years. . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 21,500 | 105,600 | 1,440 | 21,660 | 110,700 | 1,887 | *4.9 | 1.95 |\n| 55 to 64 years. . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 23,410 | 84,470 | 1,899 | 23,080 | 90,640 | 1,773 | *7.3 | 3.11 |\n| 65 years and older. . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 37,130 | 52,290 | 1,014 | 37,630 | 54,710 | 1,018 | *4.6 | 2.39 |\n| Nativity of Householder | | | | | | | | |\n| Native-born. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 110,300 | 78,200 | 821 | 110,300 | 81,700 | 683 | *4.5 | 1.22 |\n| . Foreign-born. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 21,140 | 74,180 | 1,384 | 21,920 | 73,360 | 2,546 | -1.1 | 3.32 |\n| Naturalized citizen. . . . . . . . . . . . . . . . . . . . . . . | . . . . . 11,770 | 83,970 | 2,237 | 12,220 | 86,060 | 2,773 | 2.5 | 3.75 |\n| Not a citizen. . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 9,375 | 64,490 | 1,826 | 9,700 | 61,440 | 1,109 | *-4.7 | 3.01 |\n| Region | | | | | | | | |\n| Northeast. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 22,630 | 83,550 | 2,008 | 22,590 | 86,250 | 1,816 | *3.2 | 2.80 |\n| Midwest. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 28,280 | 75,980 | 2,009 | 28,410 | 81,020 | 1,319 | *6.6 | 3.01 |\n| South. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 51,080 | 70,940 | 1,533 | 51,550 | 73,280 | 1,750 | *3.3 | 2.84 |\n| West. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | . . . . . 29,440 | 86,190 | 2,068 | 29,670 | 88,290 | 2,212 | 2.4 | 3.01 |\n| Residence 4 | | | | | | | | |\n| Inside metropolitan statistical areas . . . . . . . . . | . . . . . 113,500 | 80,580 | 1,011 | 114,300 | 83,590 | 1,140 | *3.7 | 1.67 |\n| Inside principal cities. . . . . . . . . . . . . . . . . . . . . | . . . . . 43,710 | 72,720 | 1,436 | 43,910 | 73,540 | 1,547 | 1.1 | 2.66 |\n| Outside principal cities. . . . . . . . . . . . . . . . . . . | . . . . . 69,770 | 86,530 | 1,542 | 70,360 | 90,140 | 1,087 | *4.2 | 1.94 |\n| Outside metropolitan statistical areas. . . . . . . . | . . . . . 17,950 | 58,180 | 1,393 | 17,950 | 62,520 | 1,723 | *7.5 | 2.82 |\n| Educational Attainment of Householder Total, aged 25 and older. . . . . . . . . . . . . . . . . . | . . . . . 125,300 | 79,000 | 708 | 126,300 | 82,010 | 633 | *3.8 | 1.06 |\n| . . | 9,632 | 36,230 | | 9,546 | | | 1.1 | |\n| No high school diploma . . . . . . . . . . . . . . . . . . | . . . . . | | 1,553 | | 36,620 | 1,162 | | 5.29 |\n| High school, no college. . . . . . . . . . . . . . . . . . . . . Some college. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Bachelor's degree or higher. . . . . . . . . . . . . . . . . | . . . . . 31,830 . . . . . 33,650 50,180 | 53,510 71,420 123,000 | 801 1,389 1,900 | 31,810 33,830 51,150 | 55,810 73,610 126,800 | 988 1,540 1,462 | *4.3 *3.1 *3.1 | 2.15 2.69 1.84 |\n\n* An asterisk preceding an estimate indicates change is statistically different from zero at the 90 percent confidence level.\n\n1 A margin of error (MOE) is a measure of an estimate's variability. The larger the MOE in relation to the size of the estimate, the less reliable the estimate. This number, when added to and subtracted from the estimate, forms the 90 percent confidence interval. MOEs shown in this table are based on standard errors calculated using replicate weights.\n\n2 Calculated estimate may be different due to rounded components.\n\n3 Federal surveys give respondents the option of reporting more than one race. Therefore, two basic ways of defining a race group are possible. A group, such as Asian, may be defined as those who reported Asian and no other race (the race-alone or single-race concept) or as those who reported Asian regardless of whether they also reported another race (the race-alone-or-in-combination concept). This table shows data using the first approach (race alone). The use of the single-race population does not imply that it is the preferred method of presenting or analyzing data. The Census Bureau uses a variety of approaches. Data for American Indians and Alaska Natives, Native Hawaiians and Other Pacific Islanders, and those reporting Two or More Races are not shown separately.\n\n4 Information on metropolitan statistical areas and principal cities is available at &lt;www.census.gov/programs-surveys/metro-micro/about/glossary.html&gt;.\n\nSource: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC)."
94
+ ],
95
+ "seconds": 13.70306066700141,
96
+ "error": null,
97
+ "device": "mps",
98
+ "warmup_seconds": 38.477670707998186,
99
+ "extra": {
100
+ "version": "2.146.0"
101
+ }
102
+ },
103
+ "dpo-arxiv-2305.18290": {
104
+ "pages": [
105
+ "## Direct Preference Optimization:\n\n## Your Language Model is Secretly a Reward Model\n\nRafael Rafailov∗†\n\nArchit Sharma∗†\n\nEric Mitchell∗†\n\nStefano Ermon†‡\n\nChristopher D. Manning†\n\nChelsea Finn†\n\n†Stanford University ‡ CZ Biohub\n\n{rafailov,architsh,eric.mitchell}@cs.stanford.edu\n\n## Abstract\n\nWhile large-scale unsupervised language models (LMs) learn broad world knowledge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these preferences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call Direct Preference Optimization (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sentiment of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.\n\n## 1 Introduction\n\nLarge unsupervised language models (LMs) trained on very large datasets acquire surprising capabilities [11, 7, 42, 8]. However, these models are trained on data generated by humans with a wide variety of goals, priorities, and skillsets. Some of these goals and skillsets may not be desirable to imitate; for example, while we may want our AI coding assistant to understand common programming mistakes in order to correct them, nevertheless, when generating code, we would like to bias our model toward the (potentially rare) high-quality coding ability present in its training data. Similarly, we might want our language model to be aware of a common misconception believed by 50% of people, but we certainly do not want the model to claim this misconception to be true in 50% of queries about it! In other words, selecting the model's desired responses and behavior from its very wide knowledge and abilities is crucial to building AI systems that are safe, performant, and controllable [28]. While existing methods typically steer LMs to match human preferences using reinforcement learning (RL), where β is a parameter controlling the deviation from the base reference policy πref, namely the initial SFT model πSFT . In practice, the language model policy πθ is also initialized to πSFT . The added constraint is important, as it prevents the model from deviating too far from the distribution on which the reward model is accurate, as well as maintaining the generation diversity and preventing mode-collapse to single high-reward answers. Due to the discrete nature of language generation, this objective is not differentiable and is typically optimized with reinforcement learning. The standard approach [51, 40, 1, 28] has been to construct the reward function r(x, y) = rϕ(x, y) - β(log πθ(y | x) - log πref(y | x)), and maximize using PPO [39].\n\n∗Equal contribution; more junior authors listed earlier.",
106
+ "## 4 Direct Preference Optimization\n\nMotivated by the challenges of applying reinforcement learning algorithms on large-scale problems such as fine-tuning language models, our goal is to derive a simple approach for policy optimization using preferences directly. Unlike prior RLHF methods, which learn a reward and then optimize it via RL, our approach leverages a particular choice of reward model parameterization that enables extraction of its optimal policy in closed form, without an RL training loop. As we will describe next in detail, our key insight is to leverage an analytical mapping from reward functions to optimal policies, which enables us to transform a loss function over reward functions into a loss function over policies. This change-of-variables approach avoids fitting an explicit, standalone reward model, while still optimizing under existing models of human preferences, such as the Bradley-Terry model. In essence, the policy network represents both the language model and the (implicit) reward.\n\nDeriving the DPO objective. We start with the same RL objective as prior work, Eq. 3, under a general reward function r. Following prior work [31, 30, 19, 15], it is straightforward to show that the optimal solution to the KL-constrained reward maximization objective in Eq. 3 takes the form:\n\n<!-- formula-not-decoded -->\n\nwhere Z(x) = ∑y πref(y | x) exp ( 1β r(x, y)) is the partition function. See Appendix A.1 for a complete derivation. Even if we use the MLE estimate rϕ of the ground-truth reward function r∗ , it is still expensive to estimate the partition function Z(x) [19, 15], which makes this representation hard to utilize in practice. However, we can rearrange Eq. 4 to express the reward function in terms of its corresponding optimal policy πr, the reference policy πref, and the unknown partition function Z(·). Specifically, we first take the logarithm of both sides of Eq. 4 and then with some algebra we obtain:\n\n<!-- formula-not-decoded -->\n\nWe can apply this reparameterization to the ground-truth reward r∗ and corresponding optimal model π∗ . Fortunately, the Bradley-Terry model depends only on the difference of rewards between two completions, i.e., p∗ (y1 ≻ y2 | x) = σ(r∗ (x, y1) - r∗ (x, y2)). Substituting the reparameterization in Eq. 5 for r∗ (x, y) into the preference model Eq. 1, the partition function cancels, and we can express the human preference probability in terms of only the optimal policy π∗ and reference policy πref. Thus, the optimal RLHF policy π∗ under the Bradley-Terry model satisfies the preference model:\n\n<!-- formula-not-decoded -->\n\nThe derivation is in Appendix A.2. While Eq. 6 uses the Bradley-Terry model, we can similarly derive expressions under the more general Plackett-Luce models [32, 23], shown in Appendix A.3.\n\nNow that we have the probability of human preference data in terms of the optimal policy rather than the reward model, we can formulate a maximum likelihood objective for a parametrized policy πθ. Analogous to the reward modeling approach (i.e. Eq. 2), our policy objective becomes:\n\n<!-- formula-not-decoded -->\n\nThis way, we fit an implicit reward using an alternative parameterization, whose optimal policy is simply πθ. Moreover, since our procedure is equivalent to fitting a reparametrized Bradley-Terry model, it enjoys certain theoretical properties, such as consistencies under suitable assumption of the preference data distribution [4]. In Section 5, we further discuss theoretical properties of DPO in relation to other works.",
107
+ "What does the DPO update do? For a mechanistic understanding of DPO, it is useful to analyze the gradient of the loss function LDPO. The gradient with respect to the parameters θ can be written as:\n\n<!-- formula-not-decoded -->\n\nwhere ˆ rθ(x, y) = β log πθ(y|x) πref(y|x) is the reward implicitly defined by the language model πθ and reference model πref (more in Section 5). Intuitively, the gradient of the loss function LDPO increases the likelihood of the preferred completions yw and decreases the likelihood of dispreferred completions yl. Importantly, the examples are weighed by howmuch higher the implicit reward model ˆ rθ rates the dispreferred completions, scaled by β, i.e, how incorrectly the implicit reward model orders the completions, accounting for the strength of the KL constraint. Our experiments suggest the importance of this weighting, as a naïve version of this method without the weighting coefficient can cause the language model to degenerate (Appendix Table 3).\n\nDPO outline. The general DPO pipeline is as follows: 1) Sample completions y1, y2 ∼ πref(· | x) for every prompt x, label with human preferences to construct the offline dataset of preferences D = {x(i) , y(i) w , yl)(i)}Ni=1 and 2) optimize the language model πθ to minimize LDPO for the given πref and D and desired β. In practice, one would like to reuse preference datasets publicly available, rather than generating samples and gathering human preferences. Since the preference datasets are sampled using πSFT , we initialize πref = πSFT whenever available. However, when πSFT is not available, we initialize πref by maximizing likelihood of preferred completions (x, yw), that is, πref = argmaxπ Ex,yw∼D [logπ(yw | x)]. This procedure helps mitigate the distribution shift between the true reference distribution which is unavailable, and πref used by DPO. Further details related to the implementation and hyperparameters can be found in Appendix B.\n\n## 5 Theoretical Analysis of DPO\n\nIn this section, we give further interpretation of the DPO method, provide theoretical backing, and relate advantages of DPO to issues with actor critic algorithms used for RLHF (such as PPO [39]).\n\n## 5.1 Your Language Model Is Secretly a RewardModel\n\nDPO is able to bypass both fitting an explicit reward and performing RL to learn the policy using a single maximum likelihood objective. Note the optimization objective Eq. 5 is equivalent to a Bradley-Terry model with a reward parameterization r∗ (x, y) = β log π∗ θ (y|x) πref(y|x) and we optimize our parametric model πθ, equivalently to the reward model optimization in Eq. 2 under the change of variables. In this section we will build the theory behind this reparameterization, show that it does not constrain the class of learned reward models, and allows for the exact recovery of the optimal policy. We begin with by defining an equivalence relation between reward functions.\n\nDefinition 1. We say that two reward functions r(x, y) and r′ (x, y) are equivalent iff r(x, y) - r′ (x, y) = f(x) for some function f .\n\nIt is easy to see that this is indeed an equivalence relation, which partitions the set of reward functions into classes. We can state the following two lemmas:\n\nLemma 1. Under the Plackett-Luce, and in particular the Bradley-Terry, preference framework, two reward functions from the same class induce the same preference distribution.\n\nLemma 2. Two reward functions from the same equivalence class induce the same optimal policy under the constrained RL problem.\n\nThe proofs are straightforward and we defer them to Appendix A.5. The first lemma is a well-known under-specification issue with the Plackett-Luce family of models [32]. Due to this under-specification, DPO's reward/KL tradeoff strictly dominates PPO. Second, DPO achieves a better frontier than PPO, even when PPO can access ground truth rewards (PPO-GT).",
108
+ "## 6.2 Can DPO scale to real preference datasets?\n\nNext, we evaluate fine-tuning performance of DPO on summarization and single-turn dialogue. For summarization, automatic evaluation metrics such as ROUGE can be poorly correlated with human preferences [40], and prior work has found that fine-tuning LMs using PPO on human preferences to provide more effective summaries. We evaluate different methods by sampling completions on the test split of TL;DR summarization dataset, and computing the average win rate against reference completions in the test set. The completions for all methods are sampled at temperatures varying from 0.0 to 1.0, and the win rates are shown in Figure 2 (right). DPO, PPO and Preferred-FT all fine-tune the same GPT-J SFT model 4 . We find that DPO has a win rate of approximately 61% at a temperature of 0.0, exceeding the performance of PPO at 57% at its optimal sampling temperature of 0.0. DPO also achieves a higher maximum win rate compared to the best of N baseline. We note that we did not meaningfully tune DPO's β hyperparameter, so these results may underestimate DPO's potential. Moreover, we find DPO to be much more robust to the sampling temperature than PPO, the performance of which can degrade to that of the base GPT-J model at high temperatures. Preferred-FT does not improve significantly over the SFT model. We also compare DPO and PPO head-to-head in human evaluations in Section 6.4, where DPO samples at temperature 0.25 were preferred 58% times over PPO samples at temperature 0.\n\nOn single-turn dialogue, we evaluate the different methods on the subset of the test split of the Anthropic HH dataset [1] with one step of human-assistant interaction. GPT-4 evaluations use the preferred completions on the test as the reference to compute the win rate for different methods. As there is no standard SFT model for this task, we start with a pre-trained Pythia-2.8B, use Preferred-FT to train a reference model on the chosen completions such that completions are within distribution of the model, and then train using DPO. We also compare against the best of 128 Preferred-FT completions (we found the Best of N baseline plateaus at 128 completions for this task; see Appendix Figure 4) and a 2-shot prompted version of the Pythia-2.8B base model, finding DPO performs as well or better for the best-performing temperatures for each method. We also evaluate an RLHF model trained with PPO on the Anthropic HH dataset 5 from a well-known source 6 , but are unable to find a prompt or sampling temperature that gives performance better than the base Pythia-2.8B model. Based on our results from TL;DR and the fact that both methods optimize the same reward function, we consider Best of 128 a rough proxy for PPO-level performance. Overall, DPO is the only computationally efficient method that improves over the preferred completions in the Anthropic HH dataset, and provides similar or better performance to the computationally demanding Best of 128 baseline. Finally, Figure 3 shows that DPO converges to its best performance relatively quickly.\n\n## 6.3 Generalization to a new input distribution\n\nTo further compare the performance of PPO and DPO under distribution shifts, we evaluate the PPO and DPO policies from our Reddit TL;DR summarization experiment on a different distribution, news articles in the test split of the CNN/DailyMail dataset [26], using the best sampling temperatures from TL;DR (0 and 0.25). The results are presented in Table 1. We computed the GPT-4 win rate against the ground-truth summaries in the datasets, using the same GPT-\n\nTable 1: GPT-4 win rates vs. ground truth summaries for out-of-distribution CNN/DailyMail input articles.\n\n| Alg. | Win rate vs. Temp 0 | ground truth Temp 0.25 |\n|--------|-----------------------|--------------------------|\n| DPO | 0.36 | 0.31 |\n| PPO | 0.26 | 0.23 |\n\n4 (C) prompt we used for Reddit TL;DR, but replacing the words \"forum post\" with \"news article\". For this new distribution, DPO continues to outperform the PPO policy by a significant margin. This experiment provides initial evidence that DPO policies can generalize similarly well to PPO policies, even though DPO does not use the additional unlabeled Reddit TL;DR prompts that PPO uses.\n\n[4 https://huggingface.co/CarperAI/openai\\_summarize\\_tldr\\_sft](https://huggingface.co/CarperAI/openai_summarize_tldr_sft)\n\n[5 https://huggingface.co/reciprocate/ppo\\_hh\\_pythia-6B](https://huggingface.co/reciprocate/ppo_hh_pythia-6B)\n\n[6 https://github.com/CarperAI/trlx/tree/main/examples/hh](https://github.com/CarperAI/trlx/tree/main/examples/hh)"
109
+ ],
110
+ "seconds": 3.6249257500021486,
111
+ "error": null,
112
+ "device": "mps",
113
+ "warmup_seconds": 38.477670707998186,
114
+ "extra": {
115
+ "version": "2.146.0"
116
+ }
117
+ },
118
+ "naca-report-1135-scan": {
119
+ "pages": [
120
+ "## REPORT 1135\n\n## EQUATIONS,TABLES, ANDCHARTSFORCOMPRESSIBLEFLOW1\n\nBy AMES RESEARCH STAFF\n\n| CN q∞S0 | normal force normal-force coefficient for cones, |\n|------------|---------------------------------------------------------------------------------------------------------------------------------------|\n| Cp c, h 1 | specific heat at constant pressure specific heat at constant volume enthalpy per unit mass, u+pv characteristic reference length |\n| M | V Mach number, a |\n| p q | pressure 2 dynamic pressure, ρV²/2 |\n| q R | heat added per unit mass gas constant ρVl |\n| R | Reynolds number, μ |\n| So s | base area of cone entropy per unit mass absolute temperature |\n| T | |\n| u | internal energy per unit mass 1 |\n| v | specific volume, ρ |\n| u,v | velocity components parallel and perpendicular respectively, to free-stream flow direction velocity components normal and tangential, |\n| ū,ō | respectively, to oblique shock wave speed of flow maximum speedobtainable by expanding to zero absolute temperature |\n| V Vm w | external work performed per unit mass angle of attack |\n| α β γ | √&#124;M2-1&#124; Cp ratio of specific heats, C. |\n| δ θ | angle of flow deflection across an oblique shock wave shock-wave angle measured from upstream flow |\n| θ | direction molecular vibrational-energy constant |\n| | 1 Mach angle, sin-¹ |\n| μ | absolute viscosity |\n| | Prandtl-Meyer angle (angle through which a |\n| ν | supersonic stream is turned to expand from |\n| μ | |\n| M=1 toM>1) | M=1 toM>1) |\n\na\n\nA\n\n## SUMMARY\n\nThis report, which is a revision and extension of NACA TN 1428, presents a compilation of equations, tables, and charts useful in the analysis of high-speed flow of a compressible fluid. The equations provide relations for continuous one-dimensional flow, normal and oblique shock waves, and Prandtl-Meyer expansions for both perfect and imperfect gases. The tables presentusefuldimensionlessratiosfor continuousone-dimensional flow and for normal shock waves as functions of Mach number for air considered as a perfect gas. One series of charts presents the characteristics of the flow of air (considered a perfect gas) for oblique shock waves and for cones in a supersonic air stream. A second series shows the effects of caloric imperfectionsoncontinuous one-dimensionalflowandontheflow through normal and oblique shock waves.\n\n## INTRODUCTION\n\nThe practical analysis of compressible flow involves frequent application of a few basic results. A convenient compilation of equations, tables, and charts embodying these results is therefore of great assistance in both research and design. The present report makes one of the first such compilations (ref. 1) more readily available in a revised and extended form. The revisions include a complete rewriting of the lists of equations, as well as the correction of certain typographical errors which appeared in the earlier work. The extensions are primarily in the directions dictated by increasing flight speeds, that is, to higher Mach numbers and to higher temperatures with the accompanying gaseous imperfections.\n\nCompilations similar to those of reference 1 have been given in other publications, as, for example, references 2 through 6. These references have been utilized in extending the tables and charts to higher values of the Mach number. The extension to imperfect gases is based on the relations presented in references 7 and 8.\n\n## SYMBOLS AND NOTATION\n\n## PRIMARY SYMBOLS\n\nspeed of sound cross-sectionalarea of stream tube or channel\n\nSupersedes NACA TN 1428, 'Notes and Tables for Use in the Analysis of Supersonic Flow\" by the Staff of the Ames 1- by 3-foot Supersonic Wind-Tunnel Section, 1947.\n\nWhen used without subscripts, p, p, and T denote static pressure, static density, and static temperature, respectively."
121
+ ],
122
+ "seconds": 76.8602088330008,
123
+ "error": null,
124
+ "device": "mps",
125
+ "warmup_seconds": 38.477670707998186,
126
+ "extra": {
127
+ "version": "2.146.0"
128
+ }
129
+ }
130
+ },
131
+ "marker": {
132
+ "census-income-2023": {
133
+ "pages": [
134
+ "# Income in the United States: 2023\n\n## **INTRODUCTION**\n\nThe U.S. Census Bureau collects data and publishes estimates on income, earnings, and inequality in order to evaluate national economic trends and to understand their effect on the wellbeing of households, families, and individuals.\n\nThe estimates in this report are based on data collected in the 2024 and earlier Current Population Survey Annual Social and Economic Supplements (CPS ASEC) conducted by the Census Bureau.\\* This report presents estimates on income in the United States for calendar year 2023. Historical estimates are expressed in real or 2023 dollars to account for the change in the cost of living over time.1 The current method for inflation-adjustment is based on the Chained Consumer Price Index for all Urban Consumers (C-CPI-U) between 2000 and 2023 and the Consumer Price Index for all Urban Consumers Retroactive Series (R-CPI-U-RS) prior to 2000.2 The C-CPI-U measured a 4.0 percent increase in consumer prices between 2022 and 2023, down from a 7.7 percent increase between 2021 and 2022.\n\nThis report begins with a section discussing median household income, highlighting year-to-year comparisons by characteristics such as race and Hispanic origin, nativity, region, and education. This is followed by sections on income inequality, workers, and median earnings. The income estimates in the main sections of this report are based on the concept of money income, which is pretax and does not account for the value of in-kind transfers. Estimates of post-tax income and inequality are included in Appendix B.\n\nThis report is released alongside two other reports focused on poverty estimates and health insurance coverage in the United States. For poverty and health insurance estimates, refer to \"Poverty in the United States: 2023\" and \"Health Insurance Coverage in the United States: 2023.\"3\n\n## **Highlights**\n\n- Real median household income was \\$80,610 in 2023, a 4.0 percent increase from the 2022 estimate of \\$77,540 (Figure 1 and Table A-1). This is the first statistically significant annual increase in real median household income since 2019.4\n- Real median household incomes increased by 5.4 percent for White households and by 5.7 percent for non-Hispanic White households between 2022 and 2023. There was no significant change in median incomes for Black, Asian, and Hispanic households (Figure 2 and Table A-1).\n\n- Household income rose throughout the income distribution, increasing 6.7 percent at the 10th percentile and 4.6 percent at the 90th percentile (Table A-3).\n- Income inequality as measured by the Gini index and income percentile ratios was not significantly different between 2022 and 2023 (Figure 3 and Table A-3).\n- Between 2022 and 2023, the number of total workers increased by 2.2 million, or 1.3 percent. The change in the number of full-time, year-round workers was not statistically significant (Figure 5 and Table A-6).\n- Real median earnings for men who worked full-time, year-round increased by 3.0 percent, and real median earnings increased 1.5 percent for women who worked full-time, year-round (Figure 4 and Table A-6).\n- For full-time, year-round workers, the female-to-male earnings ratio in 2023 fell to 82.7 percent from 84.0 percent in 2022 (Figure 6 and Table A-6). This is the first statistically significant annual decrease in the female-to-male earnings ratio since 2003.\n\n#### **HOUSEHOLD INCOME BY SELECTED CHARACTERISTICS**\n\nThis section focuses on median household income by selected characteristics of the householder, including race and Hispanic origin, nativity, region, and education. The householder is the person (or\n\n<sup>\\*</sup> The Census Bureau reviewed this data product to ensure appropriate access, use, and disclosure avoidance protection of the confidential source data used to produce this product (Data Management System [DMS] number: P-7534374, Disclosure Review Board [DRB] approval number: CBDRB-FY24-0434). To further protect respondent privacy, all estimates in this report have undergone additional rounding. As a result, this year's estimates may differ from previous publications and details may not sum to totals. All comparative statements have undergone statistical testing and are statistically significant at the 90 percent confidence level unless otherwise noted.",
135
+ "Figure 1.\n\n### **Median Household Income and Percent Change by Selected Characteristics**\n\nNote: Households as of March of the following year. Percent change estimate may be dierent due to rounded components. Statistically significant indicates the change is statistically dierent from zero at the 90 percent confidence level. Margins of error and other related estimates and notes are available in Table A-1. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at <https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf>. Source: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC).\n\nHouseholders aged 25 and older. In 2023, the median household income for this group was \\$82,010.",
136
+ "Table A-1.\n\n# **Income Summary Measures by Selected Characteristics: 2022 and 2023**\n\n(Income in 2023 dollars, adjusted using the C-CPI-U. Households as of March of the following year. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at <[https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf>](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf))\n\n| | | 2022 | | | 2023 | | Percent change in real median income (2023 less 2022)*, 2 | |\n|-------------------------------------------------------------------------------------------------------------------|---------|-----------|-------|---------|----------|-------|-----------------------------------------------------------|------|\n| Characteristic | | (dollars) | | Number | | | | |\n| | | | (±) | | Estimate | | | |\n| HOUSEHOLDS | | | | | | (±) | Estimate | |\n| All households . . . . . . . . . . . . . . . . . . . . . . . . Type of Household | 131,400 | 77,540 | 1,006 | 132,200 | 80,610 | 634 | *4.0 | 1.40 |\n| Family households . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 84,330 | 99,250 | 996 | 84,680 | 102,800 | 1,200 | *3.6 | 1.41 |\n| Married-couple . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 62,180 | 115,200 | 1,166 | 62,300 | 119,400 | 1,576 | *3.6 | 1.55 |\n| Female householder, no spouse present . . . . . . . | 15,030 | 58,260 | 1,258 | 15,180 | 59,470 | 1,774 | 2.1 | 3.72 |\n| Male householder, no spouse present . . . . . . . . . . | 7,128 | 76,550 | 2,824 | 7,208 | 81,890 | 2,059 | *7.0 | 4.10 |\n| Nonfamily households . . . . . . . . . . . . . . . . . . . . . . . . . . | 47,100 | 47,250 | 861 | 47,530 | 49,600 | 895 | *5.0 | 2.53 |\n| Female householder . . . . . . . . . . . . . . . . . . . . . . . . . | 24,360 | 41,790 | 812 | 24,680 | 42,140 | 947 | 0.8 | 2.74 |\n| Male householder . . . . . . . . . . . . . . . . . . . . . . . . . . . . Race3 and Hispanic Origin of Householder | 22,740 | 53,990 | 1,029 | 22,850 | 57,200 | 1,190 | *5.9 | 2.81 |\n| White . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 101,400 | 80,320 | 905 | 101,900 | 84,630 | 1,182 | *5.4 | 1.60 |\n| White, not Hispanic . . . . . . . . . . . . . . . . . . . . . . . . . . | 84,490 | 84,280 | 997 | 84,440 | 89,050 | 1,234 | *5.7 | 1.63 |\n| Black . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 18,080 | 54,960 | 1,528 | 18,040 | 56,490 | 1,328 | 2.8 | 3.65 |\n| Asian . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 7,609 | 113,100 | 4,040 | 7,655 | 112,800 | 4,187 | –0.2 | 4.69 |\n| Hispanic (any race) . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Age of Householder | 19,320 | 65,300 | 1,659 | 19,860 | 65,540 | 1,259 | 0.4 | 2.77 |\n| Under 65 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 94,300 | 89,270 | 985 | 94,590 | 92,470 | 1,088 | *3.6 | 1.46 |\n| 15 to 24 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 6,136 | 54,540 | 3,591 | 5,881 | 54,930 | 2,870 | 0.7 | 7.95 |\n| 25 to 34 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 20,720 | 83,420 | 1,566 | 20,910 | 85,780 | 1,190 | *2.8 | 2.30 |\n| 35 to 44 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 22,530 | 100,500 | 1,870 | 23,060 | 101,300 | 1,234 | 0.8 | 2.00 |\n| 45 to 54 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 21,500 | 105,600 | 1,440 | 21,660 | 110,700 | 1,887 | *4.9 | 1.95 |\n| 55 to 64 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 23,410 | 84,470 | 1,899 | 23,080 | 90,640 | 1,773 | *7.3 | 3.11 |\n| 65 years and older . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Nativity of Householder | 37,130 | 52,290 | 1,014 | 37,630 | 54,710 | 1,018 | *4.6 | 2.39 |\n| Native-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 110,300 | 78,200 | 821 | 110,300 | 81,700 | 683 | *4.5 | 1.22 |\n| Foreign-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 21,140 | 74,180 | 1,384 | 21,920 | 73,360 | 2,546 | –1.1 | 3.32 |\n| Naturalized citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . | 11,770 | 83,970 | 2,237 | 12,220 | 86,060 | 2,773 | 2.5 | 3.75 |\n| Not a citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Region | 9,375 | 64,490 | 1,826 | 9,700 | 61,440 | 1,109 | *–4.7 | 3.01 |\n| Northeast . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 22,630 | 83,550 | 2,008 | 22,590 | 86,250 | 1,816 | *3.2 | 2.80 |\n| Midwest . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 28,280 | 75,980 | 2,009 | 28,410 | 81,020 | 1,319 | *6.6 | 3.01 |\n| South . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 51,080 | 70,940 | 1,533 | 51,550 | 73,280 | 1,750 | *3.3 | 2.84 |\n| West . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Residence4 | 29,440 | 86,190 | 2,068 | 29,670 | 88,290 | 2,212 | 2.4 | 3.01 |\n| Inside metropolitan statistical areas . . . . . . . . . . . . . | 113,500 | 80,580 | 1,011 | 114,300 | 83,590 | 1,140 | *3.7 | 1.67 |\n| Inside principal cities . . . . . . . . . . . . . . . . . . . . . . . . . | 43,710 | 72,720 | 1,436 | 43,910 | 73,540 | 1,547 | 1.1 | 2.66 |\n| Outside principal cities . . . . . . . . . . . . . . . . . . . . . . . | 69,770 | 86,530 | 1,542 | 70,360 | 90,140 | 1,087 | *4.2 | 1.94 |\n| Outside metropolitan statistical areas . . . . . . . . . . . . | 17,950 | 58,180 | 1,393 | 17,950 | 62,520 | 1,723 | *7.5 | 2.82 |\n| Total, aged 25 and older . . . . . . . . . . . . . . . . . . . . . . | 125,300 | 79,000 | 708 | 126,300 | 82,010 | 633 | *3.8 | 1.06 |\n| No high school diploma . . . . . . . . . . . . . . . . . . . . . . . . | 9,632 | 36,230 | 1,553 | 9,546 | 36,620 | 1,162 | 1.1 | 5.29 |\n| High school, no college . . . . . . . . . . . . . . . . . . . . . . . . . | 31,830 | 53,510 | 801 | 31,810 | 55,810 | 988 | *4.3 | 2.15 |\n| Some college . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . | 33,650 | 71,420 | 1,389 | 33,830 | 73,610 | 1,540 | *3.1 | 2.69 |\n| Bachelor’s degree or higher . . . . . . . . . . . . . . . . . . . . . | 50,180 | 123,000 | 1,900 | 51,150 | 126,800 | 1,462 | *3.1 | 1.84 |\n\n\\* An asterisk preceding an estimate indicates change is statistically different from zero at the 90 percent confidence level.\n\n A margin of error (MOE) is a measure of an estimate's variability. The larger the MOE in relation to the size of the estimate, the less reliable the estimate. This number, when added to and subtracted from the estimate, forms the 90 percent confidence interval. MOEs shown in this table are based on standard errors calculated using replicate weights.\n\n2 Calculated estimate may be different due to rounded components.\n\n3 Federal surveys give respondents the option of reporting more than one race. Therefore, two basic ways of defining a race group are possible. A group, such as Asian, may be defined as those who reported Asian and no other race (the race-alone or single-race concept) or as those who reported Asian regardless of whether they also reported another race (the race-alone-or-in-combination concept). This table shows data using the first approach (race alone). The use of the single-race population does not imply that it is the preferred method of presenting or analyzing data. The Census Bureau uses a variety of approaches. Data for American Indians and Alaska Natives, Native Hawaiians and Other Pacific Islanders, and those reporting Two or More Races are not shown separately.\n\n4 Information on metropolitan statistical areas and principal cities is available at [<www.census.gov/programs-surveys/metro-micro/about/glossary.html>](http://www.census.gov/programs-surveys/metro-micro/about/glossary.html)."
137
+ ],
138
+ "seconds": 3.1382098750000296,
139
+ "error": null,
140
+ "device": "mps",
141
+ "warmup_seconds": 98.96880958399925,
142
+ "extra": {
143
+ "version": "2.1.0"
144
+ }
145
+ },
146
+ "dpo-arxiv-2305.18290": {
147
+ "pages": [
148
+ "# Direct Preference Optimization: Your Language Model is Secretly a Reward Model\n\nRafael Rafailov∗† Archit Sharma∗† Eric Mitchell∗†\n\nStefano Ermon†‡ Christopher D. Manning† Chelsea Finn†\n\n†Stanford University ‡CZ Biohub\n\n{rafailov,architsh,eric.mitchell}@cs.stanford.edu\n\n## Abstract\n\nWhile large-scale unsupervised language models (LMs) learn broad world knowledge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these preferences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call *Direct Preference Optimization* (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sentiment of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.\n\n# 1 Introduction\n\nLarge unsupervised language models (LMs) trained on very large datasets acquire surprising capabilities [11, 7, 42, 8]. However, these models are trained on data generated by humans with a wide variety of goals, priorities, and skillsets. Some of these goals and skillsets may not be desirable to imitate; for example, while we may want our AI coding assistant to *understand* common programming mistakes in order to correct them, nevertheless, when generating code, we would like to bias our model toward the (potentially rare) high-quality coding ability present in its training data. Similarly, we might want our language model to be *aware* of a common misconception believed by 50% of people, but we certainly do not want the model to claim this misconception to be true in 50% of queries about it! In other words, selecting the model's *desired responses and behavior* from its very wide *knowledge and abilities* is crucial to building AI systems that are safe, performant, and controllable [28]. While existing methods typically steer LMs to match human preferences using reinforcement learning (RL),\n\n<sup>∗</sup>Equal contribution; more junior authors listed earlier.",
149
+ "<span id=\"page-1-0\"></span>where β is a parameter controlling the deviation from the base reference policy πref, namely the initial SFT model π SFT. In practice, the language model policy π<sup>θ</sup> is also initialized to π SFT. The added constraint is important, as it prevents the model from deviating too far from the distribution on which the reward model is accurate, as well as maintaining the generation diversity and preventing mode-collapse to single high-reward answers. Due to the discrete nature of language generation, this objective is not differentiable and is typically optimized with reinforcement learning. The standard approach [51, 40, 1, 28] has been to construct the reward function r(x, y) = rϕ(x, y) − β(log πθ(y | x) − log πref(y | x)), and maximize using PPO [39].\n\n#### 4 Direct Preference Optimization\n\nMotivated by the challenges of applying reinforcement learning algorithms on large-scale problems such as fine-tuning language models, our goal is to derive a simple approach for policy optimization using preferences directly. Unlike prior RLHF methods, which learn a reward and then optimize it via RL, our approach leverages a particular choice of reward model parameterization that enables extraction of its optimal policy in closed form, without an RL training loop. As we will describe next in detail, our key insight is to leverage an analytical mapping from reward functions to optimal policies, which enables us to transform a loss function over reward functions into a loss function over policies. This change-of-variables approach avoids fitting an explicit, standalone reward model, while still optimizing under existing models of human preferences, such as the Bradley-Terry model. In essence, the policy network represents both the language model and the (implicit) reward.\n\nDeriving the DPO objective. We start with the same RL objective as prior work, Eq. 3, under a general reward function r. Following prior work [31, 30, 19, 15], it is straightforward to show that the optimal solution to the KL-constrained reward maximization objective in Eq. 3 takes the form:\n\n$$\\pi_r(y \\mid x) = \\frac{1}{Z(x)} \\pi_{\\text{ref}}(y \\mid x) \\exp\\left(\\frac{1}{\\beta} r(x, y)\\right), \\quad (4)$$\n\nwhere Z(x) = P y <sup>π</sup>ref(<sup>y</sup> <sup>|</sup> <sup>x</sup>) exp 1 β r(x, y) is the partition function. See Appendix A.1 for a complete derivation. Even if we use the MLE estimate r<sup>ϕ</sup> of the ground-truth reward function r ∗ , it is still expensive to estimate the partition function Z(x) [19, 15], which makes this representation hard to utilize in practice. However, we can rearrange Eq. 4 to express the reward function in terms of its corresponding optimal policy πr, the reference policy πref, and the unknown partition function Z(·). Specifically, we first take the logarithm of both sides of Eq. 4 and then with some algebra we obtain:\n\n$$r(x, y) = \\beta \\log \\frac{\\pi_r(y | x)}{\\pi_{\\text{ref}}(y | x)} + \\beta \\log Z(x). \\quad (5)$$\n\nWe can apply this reparameterization to the ground-truth reward r ∗ and corresponding optimal model π ∗ . Fortunately, the Bradley-Terry model depends only on the difference of rewards between two completions, i.e., p ∗ (y<sup>1</sup> ≻ y<sup>2</sup> | x) = σ(r ∗ (x, y1) − r ∗ (x, y2)). Substituting the reparameterization in Eq. 5 for r ∗ (x, y) into the preference model Eq. 1, the partition function cancels, and we can express the human preference probability in terms of only the optimal policy π ∗ and reference policy πref. Thus, the optimal RLHF policy π <sup>∗</sup> under the Bradley-Terry model satisfies the preference model:\n\n$$p^*(y_1 \\succ y_2 \\mid x) = \\frac{1}{1 + \\exp\\left(\\beta \\log \\frac{\\pi^*(y_2|x)}{\\pi_{\\text{ref}}(y_2|x)} - \\beta \\log \\frac{\\pi^*(y_1|x)}{\\pi_{\\text{ref}}(y_1|x)}\\right)} \\quad (6)$$\n\nThe derivation is in Appendix A.2. While Eq. 6 uses the Bradley-Terry model, we can similarly derive expressions under the more general Plackett-Luce models [32, 23], shown in Appendix A.3.\n\nNow that we have the probability of human preference data in terms of the optimal policy rather than the reward model, we can formulate a maximum likelihood objective for a parametrized policy πθ. Analogous to the reward modeling approach (i.e. Eq. 2), our policy objective becomes:\n\n$$\\mathcal{L}_{\\text{DPO}}(\\pi_\\theta; \\pi_{\\text{ref}}) = -\\mathbb{E}_{(x, y_w, y_l) \\sim \\mathcal{D}} \\left[ \\log \\sigma \\left( \\beta \\log \\frac{\\pi_\\theta(y_w | x)}{\\pi_{\\text{ref}}(y_w | x)} - \\beta \\log \\frac{\\pi_\\theta(y_l | x)}{\\pi_{\\text{ref}}(y_l | x)} \\right) \\right]. \\quad (7)$$\n\nThis way, we fit an implicit reward using an alternative parameterization, whose optimal policy is simply πθ. Moreover, since our procedure is equivalent to fitting a reparametrized Bradley-Terry",
150
+ "model, it enjoys certain theoretical properties, such as consistencies under suitable assumption of the preference data distribution [4]. In Section 5, we further discuss theoretical properties of DPO in relation to other works.\n\nWhat does the DPO update do? For a mechanistic understanding of DPO, it is useful to analyze the gradient of the loss function LDPO. The gradient with respect to the parameters θ can be written as:\n\n$$\\begin{aligned} \\nabla_{\\theta} \\mathcal{L}_{\\text{DPO}}(\\pi_{\\theta}; \\pi_{\\text{ref}}) = & \\\\ - \\beta \\mathbb{E}_{(x, y_w, y_l) \\sim \\mathcal{D}} \\left[ \\underbrace{\\sigma(\\hat{r}_{\\theta}(x, y_l) - \\hat{r}_{\\theta}(x, y_w))}_{\\text{higher weight when reward estimate is wrong}} \\left[ \\underbrace{\\nabla_{\\theta} \\log \\pi(y_w | x)}_{\\text{increase likelihood of } y_w} - \\underbrace{\\nabla_{\\theta} \\log \\pi(y_l | x)}_{\\text{decrease likelihood of } y_l} \\right] \\right], \\end{aligned}$$\n\nwhere <sup>r</sup>ˆθ(x, y) = <sup>β</sup> log <sup>π</sup>θ(y|x) πref(y|x) is the reward implicitly defined by the language model π<sup>θ</sup> and reference model πref (more in Section 5). Intuitively, the gradient of the loss function LDPO increases the likelihood of the preferred completions y<sup>w</sup> and decreases the likelihood of dispreferred completions yl . Importantly, the examples are weighed by how much higher the implicit reward model rˆ<sup>θ</sup> rates the dispreferred completions, scaled by β, i.e, how incorrectly the implicit reward model orders the completions, accounting for the strength of the KL constraint. Our experiments suggest the importance of this weighting, as a naïve version of this method without the weighting coefficient can cause the language model to degenerate (Appendix Table 3).\n\nDPO outline. The general DPO pipeline is as follows: 1) Sample completions y1, y<sup>2</sup> ∼ πref(· | x) for every prompt x, label with human preferences to construct the offline dataset of preferences D = {x (i) , y (i) <sup>w</sup> , yl) (i)} N <sup>i</sup>=1 and 2) optimize the language model π<sup>θ</sup> to minimize LDPO for the given πref and D and desired β. In practice, one would like to reuse preference datasets publicly available, rather than generating samples and gathering human preferences. Since the preference datasets are sampled using π SFT, we initialize πref = π SFT whenever available. However, when π SFT is not available, we initialize πref by maximizing likelihood of preferred completions (x, yw), that is, πref = arg max<sup>π</sup> <sup>E</sup>x,yw∼D [log π(y<sup>w</sup> | x)]. This procedure helps mitigate the distribution shift between the true reference distribution which is unavailable, and πref used by DPO. Further details related to the implementation and hyperparameters can be found in Appendix B.\n\n# 5 Theoretical Analysis of DPO\n\nIn this section, we give further interpretation of the DPO method, provide theoretical backing, and relate advantages of DPO to issues with actor critic algorithms used for RLHF (such as PPO [39]).\n\n### 5.1 Your Language Model Is Secretly a Reward Model\n\nDPO is able to bypass both fitting an explicit reward and performing RL to learn the policy using a single maximum likelihood objective. Note the optimization objective Eq. [5](#page-1-0) is equivalent to a Bradley-Terry model with a reward parameterization r ∗ (x, y) = β log <sup>π</sup> ∗ θ (y|x) πref(y|x) and we optimize our parametric model πθ, equivalently to the reward model optimization in Eq. 2 under the change of variables. In this section we will build the theory behind this reparameterization, show that it does not constrain the class of learned reward models, and allows for the exact recovery of the optimal policy. We begin with by defining an equivalence relation between reward functions.\n\nDefinition 1. *We say that two reward functions* r(x, y) *and* r ′ (x, y) *are equivalent iff* r(x, y) − r ′ (x, y) = f(x) *for some function* f*.*\n\nIt is easy to see that this is indeed an equivalence relation, which partitions the set of reward functions into classes. We can state the following two lemmas:\n\nLemma 1. *Under the Plackett-Luce, and in particular the Bradley-Terry, preference framework, two reward functions from the same class induce the same preference distribution.*\n\nLemma 2. *Two reward functions from the same equivalence class induce the same optimal policy under the constrained RL problem.*\n\nThe proofs are straightforward and we defer them to Appendix A.5. The first lemma is a well-known under-specification issue with the Plackett-Luce family of models [32]. Due to this under-specification,",
151
+ "DPO's reward/KL tradeoff strictly dominates PPO. Second, DPO achieves a better frontier than PPO, *even when PPO can access ground truth rewards* (PPO-GT).\n\n#### 6.2 Can DPO scale to real preference datasets?\n\nNext, we evaluate fine-tuning performance of DPO on summarization and single-turn dialogue. For summarization, automatic evaluation metrics such as ROUGE can be poorly correlated with human preferences [40], and prior work has found that fine-tuning LMs using PPO on human preferences to provide more effective summaries. We evaluate different methods by sampling completions on the test split of TL;DR summarization dataset, and computing the average win rate against reference completions in the test set. The completions for all methods are sampled at temperatures varying from 0.0 to 1.0, and the win rates are shown in Figure 2 (right). DPO, PPO and Preferred-FT all fine-tune the same GPT-J SFT model<sup>4</sup> . We find that DPO has a win rate of approximately 61% at a temperature of 0.0, exceeding the performance of PPO at 57% at its optimal sampling temperature of 0.0. DPO also achieves a higher maximum win rate compared to the best of N baseline. We note that we did not meaningfully tune DPO's β hyperparameter, so these results may underestimate DPO's potential. Moreover, we find DPO to be much more robust to the sampling temperature than PPO, the performance of which can degrade to that of the base GPT-J model at high temperatures. Preferred-FT does not improve significantly over the SFT model. We also compare DPO and PPO head-to-head in human evaluations in Section 6.4, where DPO samples at temperature 0.25 were preferred 58% times over PPO samples at temperature 0.\n\nOn single-turn dialogue, we evaluate the different methods on the subset of the test split of the Anthropic HH dataset [1] with one step of human-assistant interaction. GPT-4 evaluations use the preferred completions on the test as the reference to compute the win rate for different methods. As there is no standard SFT model for this task, we start with a pre-trained Pythia-2.8B, use Preferred-FT to train a reference model on the chosen completions such that completions are within distribution of the model, and then train using DPO. We also compare against the best of 128 Preferred-FT completions (we found the Best of N baseline plateaus at 128 completions for this task; see Appendix Figure 4) and a 2-shot prompted version of the Pythia-2.8B base model, finding DPO performs as well or better for the best-performing temperatures for each method. We also evaluate an RLHF model trained with PPO on the Anthropic HH dataset <sup>5</sup> from a well-known source <sup>6</sup> , but are unable to find a prompt or sampling temperature that gives performance better than the base Pythia-2.8B model. Based on our results from TL;DR and the fact that both methods optimize the same reward function, we consider Best of 128 a rough proxy for PPO-level performance. Overall, DPO is the only computationally efficient method that improves over the preferred completions in the Anthropic HH dataset, and provides similar or better performance to the computationally demanding Best of 128 baseline. Finally, Figure 3 shows that DPO converges to its best performance relatively quickly.\n\n#### 6.3 Generalization to a new input distribution\n\n| Alg. | Win rate vs. ground truth | |\n|------|---------------------------|-----------|\n| | Temp 0 | Temp 0.25 |\n| DPO | 0.36 | 0.31 |\n| PPO | 0.26 | 0.23 |\n\nTable 1: GPT-4 win rates vs. ground truth summaries for out-of-distribution CNN/DailyMail input articles. To further compare the performance of PPO and DPO under distribution shifts, we evaluate the PPO and DPO policies from our Reddit TL;DR summarization experiment on a different distribution, news articles in the test split of the CNN/DailyMail dataset [26], using the best sampling temperatures from TL;DR (0 and 0.25). The results are presented in Table 1. We computed the GPT-4 win rate against the ground-truth summaries in the datasets, using the same GPT-4 (C) prompt we used for Reddit TL;DR, but replacing the words \"forum post\" with \"news article\". For this new distribution, DPO continues to outperform the PPO policy by a significant margin. This experiment provides initial evidence that DPO policies can generalize similarly well to PPO policies, even though DPO does not use the additional unlabeled Reddit TL;DR prompts that PPO uses.\n\n<sup>4</sup> [https://huggingface.co/CarperAI/openai\\\\_summarize\\\\_tldr\\\\_sft](https://huggingface.co/CarperAI/openai_summarize_tldr_sft)\n\n<sup>5</sup> [https://huggingface.co/reciprocate/ppo\\\\_hh\\\\_pythia-6B](https://huggingface.co/reciprocate/ppo_hh_pythia-6B)\n\n<sup>6</sup> <https://github.com/CarperAI/trlx/tree/main/examples/hh>"
152
+ ],
153
+ "seconds": 58.175671500001044,
154
+ "error": null,
155
+ "device": "mps",
156
+ "warmup_seconds": 98.96880958399925,
157
+ "extra": {
158
+ "version": "2.1.0"
159
+ }
160
+ },
161
+ "naca-report-1135-scan": {
162
+ "pages": [
163
+ "# REPORT 1135\n\n## EQUATIONS, TABLES, AND CHARTS FOR COMPRESSIBLE FLOW <sup>1</sup>\n\nBy AMES RESEARCH STAFF\n\n### SUMMARY\n\n*This report, which is a revision and extension of NACA TN 1428, presents a compilation of equations, tables, and charts useful in the analysis of high-speed flow of a compressible fluid. The equations provide relations for continuous one-dimensional flow, normal and oblique shock waves, and Prandtl-Meyer expansions for both perfect and imperfect gases. The tables present useful dimensionless ratios for continuous one-dimensional flow and for normal shock waves as functions of Mach number for air considered as a perfect gas. One series of charts presents the characteristics of the flow of air (considered a perfect gas) for oblique shock waves and for cones in a supersonic air stream. A second series shows the effects of caloric imperfections on continuous one-dimensional flow and on the flow through normal and oblique shock waves.*\n\n### INTRODUCTION\n\nThe practical analysis of compressible flow involves frequent application of a few basic results. A convenient compilation of equations, tables, and charts embodying these results is therefore of great assistance in both research and design. The present report makes one of the first such compilations (ref. 1) more readily available in a revised and extended form. The revisions include a complete rewriting of the lists of equations, as well as the correction of certain typographical errors which appeared in the earlier work. The extensions are primarily in the directions dictated by increasing flight speeds, that is, to higher Mach numbers and to higher temperatures with the accompanying gaseous imperfections.\n\nCompilations similar to those of reference 1 have been given in other publications, as, for example, references 2 through 6. These references have been utilized in extending the tables and charts to higher values of the Mach number. The extension to imperfect gases is based on the relations presented in references 7 and 8.\n\n### SYMBOLS AND NOTATION\n\n#### PRIMARY SYMBOLS\n\n| <i>a</i> | speed of sound |\n|----------|------------------------------------------------|\n| <i>A</i> | cross-sectional area of stream tube or channel |\n\n| $C_N$ | normal-force coefficient for cones, $\\frac{\\text{normal force}}{q \\infty S_b}$ |\n|------------------------|------------------------------------------------------------------------------------------------------------|\n| $c_p$ | specific heat at constant pressure |\n| $c_v$ | specific heat at constant volume |\n| $h$ | enthalpy per unit mass, $u + pv$ |\n| $l$ | characteristic reference length |\n| $M$ | Mach number, $\\frac{V}{a}$ |\n| $p$ | pressure <sup>2</sup> |\n| $q$ | dynamic pressure, $\\rho V^2/2$ |\n| $q$ | heat added per unit mass |\n| $R$ | gas constant |\n| $R$ | Reynolds number, $\\frac{\\rho Vl}{\\mu}$ |\n| $S_b$ | base area of cone |\n| $s$ | entropy per unit mass |\n| $T$ | absolute temperature <sup>2</sup> |\n| $u$ | internal energy per unit mass |\n| $v$ | specific volume, $\\frac{1}{\\rho}$ |\n| $u, v$ | velocity components parallel and perpendicular respectively, to free-stream flow direction |\n| $\\tilde{u}, \\tilde{v}$ | velocity components normal and tangential, respectively, to oblique shock wave |\n| $V$ | speed of flow |\n| $V_m$ | maximum speed obtainable by expanding to zero absolute temperature |\n| $w$ | external work performed per unit mass |\n| $\\alpha$ | angle of attack |\n| $\\beta$ | $\\sqrt{ M^2 - 1 }$ |\n| $\\gamma$ | ratio of specific heats, $\\frac{c_p}{c_v}$ |\n| $\\delta$ | angle of flow deflection across an oblique shock wave |\n| $\\theta$ | shock-wave angle measured from upstream flow direction |\n| $\\Theta$ | molecular vibrational-energy constant |\n| $\\mu$ | Mach angle, $\\sin^{-1} \\frac{1}{M}$ |\n| $\\mu$ | absolute viscosity |\n| $\\nu$ | Prandtl-Meyer angle (angle through which a supersonic stream is turned to expand from $M = 1$ to $M > 1$ ) |\n\n<sup>1</sup> Supersedes NACA TN 1428, \"Notes and Tables for Use in the Analysis of Supersonic Flow\" by the Staff of the Ames 1- by 3-foot Supersonic Wind-Tunnel Section, 1947.\n\n<sup>2</sup> When used without subscripts, *p*, $\\rho$ , and *T* denote static pressure, static density, and static temperature, respectively."
164
+ ],
165
+ "seconds": 106.83621570799733,
166
+ "error": null,
167
+ "device": "mps",
168
+ "warmup_seconds": 98.96880958399925,
169
+ "extra": {
170
+ "version": "2.1.0"
171
+ }
172
+ }
173
+ },
174
+ "mineru": {
175
+ "census-income-2023": {
176
+ "pages": [
177
+ "## Income in the United States: 2023\n\n### INTRODUCTION\n\nThe U.S. Census Bureau collects data and publishes estimates on income, earnings, and inequality in order to evaluate national economic trends and to understand their effect on the wellbeing of households, families, and individuals.\n\nThe estimates in this report are based on data collected in the 2024 and earlier Current Population Survey Annual Social and Economic Supplements (CPS ASEC) conducted by the Census Bureau.\\* This report presents estimates on income in the United States for calendar year 2023. Historical estimates are expressed in real or 2023 dollars to account for the change in the cost of living over time.<sup>1</sup> The current method for inflation-adjustment is based on the Chained Consumer Price Index for all Urban Consumers (C-CPI-U) between 2000 and 2023 and the Consumer Price Index for all Urban Consumers Retroactive Series (R-CPI-U-RS) prior to 2000.<sup>2</sup> The C-CPI-U measured a 4.0 percent increase in consumer prices between 2022 and 2023, down from a 7.7 percent increase between 2021 and 2022.\n\nThis report begins with a section discussing median household income, highlighting year-to-year comparisons by characteristics such as race and Hispanic origin, nativity, region, and education. This is followed by sections on income inequality, workers, and median earnings. The income estimates in the main sections of this report are based on the concept of money income, which is pretax and does not account for the value of in-kind transfers. Estimates of post-tax income and inequality are included in Appendix B.\n\nThis report is released alongside two other reports focused on poverty estimates and health insurance coverage in the United States. For poverty and health insurance estimates, refer to “Poverty in the United States: 2023” and “Health Insurance Coverage in the United States: 2023.”<sup>3</sup>\n\n### Highlights\n\nReal median household income was \\$80,610 in 2023, a 4.0 percent increase from the 2022 estimate of \\$77,540 (Figure 1 and Table A-1). This is the first statistically significant annual increase in real median household income since 2019.<sup>4</sup>\n\nReal median household incomes increased by 5.4 percent for White households and by 5.7 percent for non-Hispanic White households between 2022 and 2023. There was no significant change in median incomes for Black, Asian, and Hispanic households (Figure 2 and Table A-1).\n\n• Household income rose throughout the income distribution, increasing 6.7 percent at the 10th percentile and 4.6 percent at the 90th percentile (Table A-3).\n\n• Income inequality as measured by the Gini index and income percentile ratios was not significantly different between 2022 and 2023 (Figure 3 and Table A-3).\n\nBetween 2022 and 2023, the number of total workers increased by 2.2 million, or 1.3 percent. The change in the number of full-time, year-round workers was not statistically significant (Figure 5 and Table A-6).\n\nReal median earnings for men who worked full-time, year-round increased by 3.0 percent, and real median earnings increased 1.5 percent for women who worked full-time, year-round (Figure 4 and Table A-6).\n\nFor full-time, year-round workers, the female-to-male earnings ratio in 2023 fell to 82.7 percent from 84.0 percent in 2022 (Figure 6 and Table A-6). This is the first statistically significant annual decrease in the female-to-male earnings ratio since 2003.\n\n\\* The Census Bureau reviewed this data product to ensure appropriate access, use, and disclosure avoidance protection of the confidential source data used to produce this product (Data Management System [DMS] number: P-7534374, Disclosure Review Board [DRB] approval number: CBDRB-FY24-0434). To further protect respondent privacy, all estimates in this report have undergone additional rounding. As a result, this year’s estimates may differ from previous publications and details may not sum to totals. All comparative statements have undergone statistical testing and are statistically significant at the 90 percent confidence level unless otherwise noted.\n\n### HOUSEHOLD INCOME BY SELECTED CHARACTERISTICS\n\nThis section focuses on median household income by selected characteristics of the householder, including race and Hispanic origin, nativity, region, and education. The householder is the person (or\n\nU.S. Census Bureau\n\nIncome in the United States: 2023 1",
178
+ "Median Household Income and Percent Change by Selected Characteristics\n\n2 Income in the United States: 2023\n\nU.S. Census Bureau",
179
+ "Table A-1.\n\n### Income Summary Measures by Selected Characteristics: 2022 and 2023\n\n(Income in 2023 dollars, adjusted using the C-CPI-U. Households as of March of the following year. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at <[https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf)>)\n\n<table><tbody><tr><td rowspan=\"3\">Characteristic</td><td colspan=\"3\">2022</td><td colspan=\"3\">2023</td><td rowspan=\"2\" colspan=\"2\">Percent change in real median income (2023 less 2022)*, <sup>2</sup></td></tr><tr><td rowspan=\"2\">Number (thousands)</td><td colspan=\"2\">Median income (dollars)</td><td rowspan=\"2\">Number (thou-sands)</td><td colspan=\"2\">Median income (dollars)</td></tr><tr><td>Estimate</td><td>Margin of error<sup>1</sup> (±)</td><td>Estimate</td><td>Margin of error<sup>1</sup> (±)</td><td>Estimate</td><td>Margin of error<sup>1</sup> (±)</td></tr><tr><td>HOUSEHOLDS All households . . . . . . . . . . . . . . . . . . . . . . . . Type of Household Family households . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Married-couple . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Female householder, no spouse present . . . . . . . Male householder, no spouse present . . . . . . . . . . Nonfamily households . . . . . . . . . . . . . . . . . . . . . . . . . . Female householder . . . . . . . . . . . . . . . . . . . . . . . . . Male householder . . . . . . . . . . . . . . . . . . . . . . . . . . . . Race<sup>3</sup> and Hispanic Origin of Householder White . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . White, not Hispanic . . . . . . . . . . . . . . . . . . . . . . . . . . Black . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Asian . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Hispanic (any race) . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Age of Householder Under 65 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 15 to 24 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 25 to 34 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 35 to 44 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 45 to 54 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 55 to 64 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 65 years and older . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Nativity of Householder Native-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Foreign-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Naturalized citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . Not a citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Region Northeast . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Midwest . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . South . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . West . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Residence<sup>4</sup> Inside metropolitan statistical areas . . . . . . . . . . . . . Inside principal cities . . . . . . . . . . . . . . . . . . . . . . . . . Outside principal cities . . . . . . . . . . . . . . . . . . . . . . . Outside metropolitan statistical areas . . . . . . . . . . . . Educational Attainment of Householder Total, aged 25 and older . . . . . . . . . . . . . . . . . . . . . . No high school diploma . . . . . . . . . . . . . . . . . . . . . . . . High school, no college . . . . . . . . . . . . . . . . . . . . . . . . . Some college . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Bachelor's degree or higher . . . . . . . . . . . . . . . . . . . . .</td><td>131,40084,33062,18015,0307,12847,10024,36022,740101,40084,49018,0807,60919,32094,3006,13620,72022,53021,50023,41037,130110,30021,14011,7709,37522,63028,28051,08029,440113,50043,71069,77017,950125,3009,63231,83033,65050,180</td><td>77,54099,250115,20058,26076,55047,25041,79053,99080,32084,28054,960113,10065,30089,27054,54083,420100,500105,60084,47052,29078,20074,18083,97064,49083,55075,98070,94086,19080,58072,72086,53058,18079,00036,23053,51071,420123,000</td><td>1,0069961,1661,2582,8248618121,0299059971,5284,0401,6599853,5911,5661,8701,4401,8991,0148211,3842,2371,8262,0082,0091,5332,0681,0111,4361,5421,3937081,5538011,3891,900</td><td>132,20084,68062,30015,1807,20847,53024,68022,850101,90084,44018,0407,65519,86094,5905,88120,91023,06021,66023,08037,630110,30021,92012,2209,70022,59028,41051,55029,670114,30043,91070,36017,950126,3009,54631,81033,83051,150</td><td>80,610102,800119,40059,47081,89049,60042,14057,20084,63089,05056,490112,80065,54092,47054,93085,780101,300110,70090,64054,71081,70073,36086,06061,44086,25081,02073,28088,29083,59073,54090,14062,52082,01036,62055,81073,610126,800</td><td>6341,2001,5761,7742,0598959471,1901,1821,2341,3284,1871,2591,0882,8701,1901,2341,8871,7731,0186832,5462,7731,1091,8161,3191,7502,2121,1401,5471,0871,7236331,1629881,5401,462</td><td>*4.0*3.6*3.62.1*7.0*5.00.8*5.9*5.4*5.72.8-0.20.4*3.60.7*2.80.8*4.9*7.3*4.6*4.5-1.12.5*-4.7*3.2*6.6*3.32.4*3.71.1*4.2*7.5*3.81.1*4.3*3.1*3.1</td><td>1.401.411.553.724.102.532.742.811.601.633.654.692.771.467.952.302.001.953.112.391.223.323.753.012.803.012.843.011.672.661.942.821.065.292.152.691.84</td></tr></tbody></table>\n\n\\* An asterisk preceding an estimate indicates change is statistically different from zero at the 90 percent confidence level.\n\n1 A margin of error (MOE) is a measure of an estimate’s variability. The larger the MOE in relation to the size of the estimate, the less reliable the estimate. This number, when added to and subtracted from the estimate, forms the 90 percent confidence interval. MOEs shown in this table are based on standard errors calculated using replicate weights.\n\n2 Calculated estimate may be different due to rounded components.\n\n3 Federal surveys give respondents the option of reporting more than one race. Therefore, two basic ways of defining a race group are possible. A group, such as Asian, may be defined as those who reported Asian and no other race (the race-alone or single-race concept) or as those who reported Asian regardless of whether they also reported another race (the race-alone-or-in-combination concept). This table shows data using the first approach (race alone). The use of the single-race population does not imply that it is the preferred method of presenting or analyzing data. The Census Bureau uses a variety of approaches. Data for American Indians and Alaska Natives, Native Hawaiians and Other Pacific Islanders, and those reporting Two or More Races are not shown separately.\n\n4 Information on metropolitan statistical areas and principal cities is available at <[www.census.gov/programs-surveys/metro-micro/about/glossary.html](http://www.census.gov/programs-surveys/metro-micro/about/glossary.html)>.\n\nSource: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC).\n\nU.S. Census Bureau\n\nIncome in the United States: 2023 15"
180
+ ],
181
+ "seconds": 6.236607874998299,
182
+ "error": null,
183
+ "device": "mps",
184
+ "warmup_seconds": 20.455139541998506,
185
+ "extra": {
186
+ "version": "4.1.6"
187
+ }
188
+ },
189
+ "dpo-arxiv-2305.18290": {
190
+ "pages": [
191
+ "arXiv:2305.18290v3 [cs.LG] 29 Jul 2024\n\n## Direct Preference Optimization: Your Language Model is Secretly a Reward Model\n\n**Rafael Rafailov**<sup>∗†</sup>\n\n**Archit Sharma**<sup>∗†</sup>\n\n**Eric Mitchell**∗†\n\n**Stefano Ermon**†‡\n\n**Christopher D. Manning**†\n\n**Chelsea Finn**†\n\n†Stanford University ‡CZ Biohub {rafailov,architsh,eric.mitchell}@cs.stanford.edu\n\n### Abstract\n\nWhile large-scale unsupervised language models (LMs) learn broad world knowledge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these preferences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call Direct Preference Optimization (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sentiment of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.\n\n### 1 Introduction\n\nLarge unsupervised language models (LMs) trained on very large datasets acquire surprising capabilities [11, 7, 42, 8]. However, these models are trained on data generated by humans with a wide variety of goals, priorities, and skillsets. Some of these goals and skillsets may not be desirable to imitate; for example, while we may want our AI coding assistant to understand common programming mistakes in order to correct them, nevertheless, when generating code, we would like to bias our model toward the (potentially rare) high-quality coding ability present in its training data. Similarly, we might want our language model to be aware of a common misconception believed by 50% of people, but we certainly do not want the model to claim this misconception to be true in 50% of queries about it! In other words, selecting the model’s desired responses and behavior from its very wide knowledge and abilities is crucial to building AI systems that are safe, performant, and controllable [28]. While existing methods typically steer LMs to match human preferences using reinforcement learning (RL),\n\n<sup>∗</sup>Equal contribution; more junior authors listed earlier.\n\n37th Conference on Neural Information Processing Systems (NeurIPS 2023).",
192
+ "where β is a parameter controlling the deviation from the base reference policy $\\pi _ { \\mathrm { r e f } } ,$ , namely the initial SFT model $\\pi ^ { \\mathrm { S F T } }$ . In practice, the language model policy π<sub>θ</sub> is also initialized to $\\pi ^ { \\mathrm { \\check { S } F T } }$ . The added constraint is important, as it prevents the model from deviating too far from the distribution on which the reward model is accurate, as well as maintaining the generation diversity and preventing mode-collapse to single high-reward answers. Due to the discrete nature of language generation, this objective is not differentiable and is typically optimized with reinforcement learning. The standard approach [51, 40, 1, 28] has been to construct the reward function $r ( x , y ) = r _ { \\phi } ( x , y ) - \\beta ( \\operatorname { l o g } \\pi _ { \\theta } ( y \\mid x ) - \\operatorname { l o g } \\pi _ { \\operatorname { r e f } } ( y \\mid x ) )$ , and maximize using PPO [39].\n\n### 4 Direct Preference Optimization\n\nMotivated by the challenges of applying reinforcement learning algorithms on large-scale problems such as fine-tuning language models, our goal is to derive a simple approach for policy optimization using preferences directly. Unlike prior RLHF methods, which learn a reward and then optimize it via $\\mathrm { R L }$ , our approach leverages a particular choice of reward model parameterization that enables extraction of its optimal policy in closed form, without an RL training loop. As we will describe next in detail, our key insight is to leverage an analytical mapping from reward functions to optimal policies, which enables us to transform a loss function over reward functions into a loss function over policies. This change-of-variables approach avoids fitting an explicit, standalone reward model, while still optimizing under existing models of human preferences, such as the Bradley-Terry model. In essence, the policy network represents both the language model and the (implicit) reward.\n\n**Deriving the DPO objective.** We start with the same RL objective as prior work, Eq. 3, under a general reward function r. Following prior work [31, 30, 19, 15], it is straightforward to show that the optimal solution to the KL-constrained reward maximization objective in Eq. 3 takes the form:\n\n\\pi _ { r } ( y \\mid x ) = \\frac { 1 } { Z ( x ) } \\pi _ { \\mathsf { r e f } } ( y \\mid x ) \\operatorname { e x p } \\left( \\frac { 1 } { \\beta } r ( x , y ) \\right) ,\\tag{4}\n\nwhere $\\begin{array} { r } { Z ( x ) = \\sum _ { y } \\pi _ { \\mathrm { r e f } } ( y \\mid x ) \\exp \\left( \\frac { 1 } { \\beta } r ( x , y ) \\right) } \\end{array}$ is the partition function. See Appendix A.1 for a complete derivation. Even if we use the MLE estimate $r _ { \\phi }$ of the ground-truth reward function $r ^ { * }$ , it is still expensive to estimate the partition function $Z ( x ) \\: [ 1 \\dot { 9 } , 1 5 ]$ , which makes this representation hard to utilize in practice. However, we can rearrange Eq. 4 to express the reward function in terms of its corresponding optimal policy $\\pi _ { r } ,$ , the reference policy $\\pi _ { \\mathrm { r e f } } ,$ and the unknown partition function $Z ( \\cdot )$ Specifically, we first take the logarithm of both sides of Eq. 4 and then with some algebra we obtain:\n\nr ( x , y ) = \\beta \\operatorname { l o g } \\frac { \\pi _ { r } ( y \\mid x ) } { \\pi _ { \\mathsf { r e f } } ( y \\mid x ) } + \\beta \\operatorname { l o g } Z ( x ) .\\tag{5}\n\nWe can apply this reparameterization to the ground-truth reward $r ^ { * }$ and corresponding optimal model $\\pi ^ { * }$ . Fortunately, the Bradley-Terry model depends only on the difference of rewards between two completions, $\\dot{\\mathbf{i}.\\mathbf{e}.}, p^*(y_1 \\succ \\dot{y}_2 \\mid x) = \\sigma(r^*(x,y_1) - r^*(x,y_2))$ . Substituting the reparameterization in Eq. 5 for $r ^ { * } ( x , y )$ into the preference model Eq. 1, the partition function cancels, and we can express the human preference probability in terms of only the optimal policy $\\pi ^ { * }$ and reference policy $\\pi _ { \\mathrm { r e f } }$ . Thus, the optimal RLHF policy $\\pi ^ { * }$ under the Bradley-Terry model satisfies the preference model:\n\np ^ { * } ( y _ { 1 } \\succ y _ { 2 } \\mid x ) = \\frac { 1 } { 1 + \\operatorname { e x p } \\left( \\beta \\operatorname { l o g } \\frac { \\pi ^ { * } ( y _ { 2 } \\mid x ) } { \\pi _ { \\operatorname { r e f } } ( y _ { 2 } \\mid x ) } - \\beta \\operatorname { l o g } \\frac { \\pi ^ { * } ( y _ { 1 } \\mid x ) } { \\pi _ { \\operatorname { r e f } } ( y _ { 1 } \\mid x ) } \\right) }\\tag{6}\n\nThe derivation is in Appendix A.2. While Eq. 6 uses the Bradley-Terry model, we can similarly derive expressions under the more general Plackett-Luce models [32, 23], shown in Appendix $\\mathbf { A } . 3 .$\n\nNow that we have the probability of human preference data in terms of the optimal policy rather than the reward model, we can formulate a maximum likelihood objective for a parametrized policy π<sub>θ</sub>. Analogous to the reward modeling approach (i.e. Eq. 2), our policy objective becomes:\n\n\\mathcal { L } _ { \\mathsf { D P O } } ( \\pi _ { \\theta } ; \\pi _ { \\mathsf { r e f } } ) = - \\mathbb { E } _ { ( x , y _ { w } , y _ { l } ) \\sim \\mathcal { D } } \\left[ \\operatorname { l o g } \\sigma \\left( \\beta \\operatorname { l o g } \\frac { \\pi _ { \\theta } ( y _ { w } \\mid x ) } { \\pi _ { \\mathsf { r e f } } ( y _ { w } \\mid x ) } - \\beta \\operatorname { l o g } \\frac { \\pi _ { \\theta } ( y _ { l } \\mid x ) } { \\pi _ { \\mathsf { r e f } } ( y _ { l } \\mid x ) } \\right) \\right] .\\tag{7}\n\nThis way, we fit an implicit reward using an alternative parameterization, whose optimal policy is simply $\\pi _ { \\theta }$ . Moreover, since our procedure is equivalent to fitting a reparametrized Bradley-Terry\n\n4",
193
+ "model, it enjoys certain theoretical properties, such as consistencies under suitable assumption of the preference data distribution [4]. In Section 5, we further discuss theoretical properties of DPO in relation to other works.\n\n**What does the DPO update do?** For a mechanistic understanding of $\\mathrm { D P O } ,$ it is useful to analyze the gradient of the loss function $\\mathcal { L } _ { \\mathrm { D P O } }$ . The gradient with respect to the parameters θ can be written as:\n\n\\begin{aligned} { } & { { } \\nabla _ { \\theta } \\mathcal { L } _ { \\mathsf { D P O } } ( \\pi _ { \\theta } ; \\pi _ { \\mathsf { r e f } } ) = } \\\\ { } & { { } - \\beta \\mathbb { E } _ { ( x , y _ { w } , y _ { l } ) \\thicksim \\mathcal { D } } \\bigg [ \\underbrace { \\sigma ( \\hat { r } _ { \\theta } ( x , y _ { l } ) - \\hat { r } _ { \\theta } ( x , y _ { w } ) ) } _ { \\mathsf { h i g h e r \\; w e i g h t \\; w h e n \\; r e w a d \\; c s i m a c \\; i s \\; w r o g } } \\bigg [ \\underbrace { \\nabla _ { \\theta } \\operatorname { l o g } \\pi ( y _ { w } \\mid x ) } _ { \\mathsf { i n c r e a s e \\; l i k e l h o o d \\; o f \\; y _ { w } } } - \\underbrace { \\nabla _ { \\theta } \\operatorname { l o g } \\pi ( y _ { l } \\mid x ) } _ { \\mathsf { d e c r e a s e \\; l i k e l h o o d \\; o f \\; y _ { l } } } \\bigg ] \\bigg ] , } \\\\ \\end{aligned}\n\nwhere $\\begin{array} { r } { \\hat { r } _ { \\theta } ( x , y ) = \\beta \\log \\frac { \\pi _ { \\theta } ( y | x ) } { \\pi _ { \\mathrm { r e f } } ( y | x ) } } \\end{array}$ is the reward implicitly defined by the language model $\\pi _ { \\theta }$ and reference model $\\pi _ { \\mathrm { r e f } }$ (more in Section 5). Intuitively, the gradient of the loss function $\\mathcal { L } _ { \\mathrm { D P O } }$ increases the likelihood of the preferred completions $y _ { w }$ and decreases the likelihood of dispreferred completions $y _ { l } .$ Importantly, the examples are weighed by how much higher the implicit reward model $\\hat { r } _ { \\theta }$ rates the dispreferred completions, scaled by β, i.e, how incorrectly the implicit reward model orders the completions, accounting for the strength of the KL constraint. Our experiments suggest the importance of this weighting, as a naïve version of this method without the weighting coefficient can cause the language model to degenerate (Appendix Table 3).\n\n**DPO outline.** The general DPO pipeline is as follows: 1) Sample completions $y _ { 1 } , y _ { 2 } \\sim \\pi _ { \\mathsf { r e f } } ( \\cdot \\mid x )$ for every prompt x, label with human preferences to construct the offline dataset of preferences $\\mathcal { D } = \\{ x ^ { ( i ) } , y _ { w } ^ { ( i ) } , y _ { l } ) ^ { ( i ) } \\} _ { i = 1 } ^ { N }$ 1 and 2) optimize the language model $\\pi _ { \\theta }$ to minimize $\\mathcal { L } _ { \\mathrm { D P O } }$ for the given $\\pi _ { \\mathrm { r e f } }$ and D and desired $\\beta .$ In practice, one would like to reuse preference datasets publicly available, rather than generating samples and gathering human preferences. Since the preference datasets are sampled using $\\pi ^ { \\mathrm { \\tilde { S } F T } }$ , we initialize $\\pi _ { \\mathrm { r e f } } = \\pi ^ { \\mathrm { S F T } }$ whenever available. However, when $\\pi ^ { \\mathrm { S F T } }$ not available, we initialize $\\pi _ { \\mathrm { r e f } }$ by maximizing likelihood of preferred completions $( x , y _ { w } )$ , that $\\begin{array} { r } { \\mathbf { i s , } \\; \\pi _ { \\mathsf { r e f } } = \\arg \\operatorname* { m a x } _ { \\pi } \\mathbb { E } _ { x , y _ { w } \\sim \\mathcal { D } } \\left[ \\log \\dot { \\pi } ( y _ { w } \\mid x ) \\right] } \\end{array}$ This procedure helps mitigate the distribution shift between the true reference distribution which is unavailable, and $\\pi _ { \\mathrm { r e f } }$ used by DPO. Further details related to the implementation and hyperparameters can be found in Appendix B.\n\n### 5 Theoretical Analysis of DPO\n\nIn this section, we give further interpretation of the DPO method, provide theoretical backing, and relate advantages of DPO to issues with actor critic algorithms used for RLHF (such as PPO [39]).\n\n### 5.1 Your Language Model Is Secretly a Reward Model\n\nDPO is able to bypass both fitting an explicit reward and performing RL to learn the policy using a single maximum likelihood objective. Note the optimization objective Eq. 5 is equivalent to a Bradley-Terry model with a reward parameterization $\\begin{array} { r } { r ^ { * } ( x , y ) = \\beta \\log \\frac { \\pi _ { \\theta } ^ { * } ( y | x ) } { \\pi _ { \\mathrm { r e f } } ( y | x ) } } \\end{array}$ and we optimize our parametric model π<sub>θ</sub>, equivalently to the reward model optimization in Eq. 2 under the change of variables. In this section we will build the theory behind this reparameterization, show that it does not constrain the class of learned reward models, and allows for the exact recovery of the optimal policy. We begin with by defining an equivalence relation between reward functions.\n\n**Definition 1.** We say that two reward functions $r(x,y)$ and $r ^ { \\prime } ( x , y )$ are equivalent $衿$ $r(x,y)-r^{\\prime}(x,y)=f(x)$ for some function $\\dot { f } .$\n\nIt is easy to see that this is indeed an equivalence relation, which partitions the set of reward functions into classes. We can state the following two lemmas:\n\n**Lemma 1.** Under the Plackett-Luce, and in particular the Bradley-Terry, preference framework, two reward functions from the same class induce the same preference distribution.\n\n**Lemma 2.** Two reward functions from the same equivalence class induce the same optimal policy under the constrained RL problem.\n\nThe proofs are straightforward and we defer them to Appendix A.5. The first lemma is a well-known under-specification issue with the Plackett-Luce family of models [32]. Due to this under-specification,\n\n5",
194
+ "DPO’s reward/KL tradeoff strictly dominates PPO. Second, DPO achieves a better frontier than PPO, even when PPO can access ground truth rewards (PPO-GT).\n\n### 6.2 Can DPO scale to real preference datasets?\n\nNext, we evaluate fine-tuning performance of DPO on summarization and single-turn dialogue. For summarization, automatic evaluation metrics such as ROUGE can be poorly correlated with human preferences [40], and prior work has found that fine-tuning LMs using PPO on human preferences to provide more effective summaries. We evaluate different methods by sampling completions on the test split of TL;DR summarization dataset, and computing the average win rate against reference completions in the test set. The completions for all methods are sampled at temperatures varying from 0.0 to 1.0, and the win rates are shown in Figure 2 (right). DPO, PPO and Preferred-FT all fine-tune the same GPT-J SFT model<sup>4</sup>. We find that DPO has a win rate of approximately 61% at a temperature of 0.0, exceeding the performance of PPO at 57% at its optimal sampling temperature of 0.0. DPO also achieves a higher maximum win rate compared to the best of N baseline. We note that we did not meaningfully tune DPO’s β hyperparameter, so these results may underestimate DPO’s potential. Moreover, we find DPO to be much more robust to the sampling temperature than PPO, the performance of which can degrade to that of the base GPT-J model at high temperatures. Preferred-FT does not improve significantly over the SFT model. We also compare DPO and PPO head-to-head in human evaluations in Section 6.4, where DPO samples at temperature 0.25 were preferred 58% times over PPO samples at temperature 0.\n\nOn single-turn dialogue, we evaluate the different methods on the subset of the test split of the Anthropic HH dataset [1] with one step of human-assistant interaction. GPT-4 evaluations use the preferred completions on the test as the reference to compute the win rate for different methods. As there is no standard SFT model for this task, we start with a pre-trained Pythia-2.8B, use Preferred-FT to train a reference model on the chosen completions such that completions are within distribution of the model, and then train using DPO. We also compare against the best of 128 Preferred-FT completions (we found the Best of N baseline plateaus at 128 completions for this task; see Appendix Figure 4) and a 2-shot prompted version of the Pythia-2.8B base model, finding DPO performs as well or better for the best-performing temperatures for each method. We also evaluate an RLHF model trained with PPO on the Anthropic HH dataset <sup>5</sup>from a well-known source <sup>6</sup>, but are unable to find a prompt or sampling temperature that gives performance better than the base Pythia-2.8B model. Based on our results from TL;DR and the fact that both methods optimize the same reward function, we consider Best of 128 a rough proxy for PPO-level performance. Overall, DPO is the only computationally efficient method that improves over the preferred completions in the Anthropic HH dataset, and provides similar or better performance to the computationally demanding Best of 128 baseline. Finally, Figure 3 shows that DPO converges to its best performance relatively quickly.\n\n### 6.3 Generalization to a new input distribution\n\nTo further compare the performance of PPO and DPO under distribution shifts, we evaluate the PPO and DPO policies from our Reddit TL;DR summarization experiment on a different distribution, news articles in the test split of the CNN/DailyMail dataset [26], using the best sampling temperatures from TL;DR (0 and 0.25). The results are presented in Table 1. We computed the GPT-4 win rate against the ground-truth summaries in the datasets, using the same GPT-\n\nTable 1: GPT-4 win rates vs. ground truth summaries for out-of-distribution CNN/DailyMail input articles.\n\n<table><tbody><tr><td></td><td>Win rate</td><td>vs. ground truth</td></tr><tr><td>Alg.</td><td>Temp 0</td><td>Temp 0.25</td></tr><tr><td>DPO</td><td>0.36</td><td>0.31</td></tr><tr><td>PPO</td><td>0.26</td><td>0.23</td></tr></tbody></table>\n\n4 (C) prompt we used for Reddit TL;DR, but replacing the words “forum post” with “news article”. For this new distribution, DPO continues to outperform the PPO policy by a significant margin. This experiment provides initial evidence that DPO policies can generalize similarly well to PPO policies, even though DPO does not use the additional unlabeled Reddit TL;DR prompts that PPO uses.\n\n<sup>4</sup>[https://huggingface.co/CarperAI/openai\\_summarize\\_tldr\\_sft](https://huggingface.co/CarperAI/openai_summarize_tldr_sft)\n\n<sup>5</sup>[https://huggingface.co/reciprocate/ppo\\_hh\\_pythia-6B](https://huggingface.co/reciprocate/ppo_hh_pythia-6B)\n\n<sup>6</sup>[https://github.com/CarperAI/trlx/tree/main/examples/hh](https://github.com/CarperAI/trlx/tree/main/examples/hh)\n\n9"
195
+ ],
196
+ "seconds": 43.43942720799896,
197
+ "error": null,
198
+ "device": "mps",
199
+ "warmup_seconds": 20.455139541998506,
200
+ "extra": {
201
+ "version": "4.1.6"
202
+ }
203
+ },
204
+ "naca-report-1135-scan": {
205
+ "pages": [
206
+ "## REPORT 1135\n\n## By AMES RESEARCH STAFF\n\n### SUMMARY\n\nThis report, which is a revision and extension of NACA TN 1428, presents a compilation of equations,tables, and charts useful in the analysis of high-speed flow of a compressible fluid. The equations provide relations for continuous one-dimensional flow, normal and oblique shock waves, and Prandtl-Meyer expansions for both perfect and imperfect gases. The tables present useful dimensionless ratios for continuous one-dimensional flow and for normal shock waves as functions of Mach number for air considered as a perfect gas. One series of charts presents the characteristics of the flow of air (considered a perfect gas) for oblique shock waves and for cones in a supersonic air stream. A second series shows the effects of caloric imperfectionsoncontinuousone-dimensional flowandon theflow through normal and oblique shock waves.\n\n### INTRODUCTION\n\nThe practical analysis of compressible flow involves frequent application of a few basic results. A convenient compilation of equations, tables, and charts embodying these results is therefore ofgreat assistance in both research and design. The present report makes one of the first such compilations (ref. 1) more readily available in a revised and extendedform. Therevisionsinclude a complete rewriting of the lists of equations, as well as the correction of certain typographical errors which appeared in the earlier work. The extensions are primarily in the directions dictated by increasing flight speeds, that is, to higher Mach numbers and tohigher temperatures with the accompanying gaseous imperfections.\n\nCompilations similar to those of reference 1 have been given in other publications, as, for example, references 2 through 6. These references have been utilized in extending the tables and charts to higher values of the Mach number. The extension to imperfect gases is based on the relations presented in references 7 and 8.\n\n### SYMBOLS ANDNOTATION\n\n### PRIMARY SYMBOLS\n\na speed of sound A cross-sectionalarea of stream tube or channel\n\nnormal force $C_{N}$ normal-force coefficient for cones, $\\overline { { { { \\mathfrak { q } } } _ { \\infty } { \\mathcal { S } } _ { b } } }$ $\\mathbf{c}_{\\mathbf{p}}$ specific heat at constant pressure $\\mathbf{c}_{\\#}$ specific heat at constant volume h enthalpy per unit mass, u+pv 1 characteristic reference length M Mach number, $\\frac{V}{a}$ p pressure ? dynamic pressure, $\\pmb { \\rho } V ^ { 2 } / 2$ gR heat added per unit mass gas constant R Reynolds number, $\\frac{\\rho V l}{\\mu}$ $\\boldsymbol{S}_{\\bar{\\theta}}$ base area of cone 8 entropy per unit mass $\\pmb { r }$ absolute temperature² u internal energy per unit mass v specific volume, $\\frac{1}{e}$ $u,v$ velocity components parallel and perpendicular respectively, to free-stream flow direction $\\tilde{u},\\tilde{v}$ velocity components normal and tangential, respectively, to oblique shock wave $V$ speed of flow $V_{ 耑 }$ maximum speed obtainable by expanding to zero absolute temperature w external work performed per unit mass α angle of attack β $\\sqrt{ \\left| M^{2} - 1 \\right| }$ γ ratio of specific heats, $\\frac{c_{p}}{c_{p}}$ 8 angle of flow deflection across an oblique shock wave θ shock-wave angle measured from upstream flow direction θ molecular vibrational-energy constant μ Mach angle, $\\sin^{-1}\\frac{1}{M}$ μ absolute viscosity Prandtl-Meyer angle (angle through which a supersonic stream is turned to expand from M=1 to M>1)\n\nSupersedes NA CA TN 1428, 'Notes and Tables for Use in the Analysis of Supersonic Flow\" by the Staff of the Ames 1- by 3-foot Supersonic Wind-Tunnel Section, 1947.\n\nWhen used without subscripts, p, p, and T denote static pressure, static density, and static temperature, respectively.\n\n613"
207
+ ],
208
+ "seconds": 11.079153833001328,
209
+ "error": null,
210
+ "device": "mps",
211
+ "warmup_seconds": 20.455139541998506,
212
+ "extra": {
213
+ "version": "4.1.6"
214
+ }
215
+ }
216
+ },
217
+ "unstructured": {
218
+ "census-income-2023": {
219
+ "pages": [
220
+ "## Income in the United States: 2023\n\n## INTRODUCTION\n\nThe U.S. Census Bureau collects data and publishes estimates on income, earnings, and inequal- ity in order to evaluate national economic trends and to under- stand their effect on the well- being of households, families, and individuals.\n\nThe estimates in this report are based on data collected in the 2024 and earlier Current Population Survey Annual Social and Economic Supplements (CPS ASEC) conducted by the Census Bureau.* This report presents estimates on income in the United States for calendar year 2023. Historical estimates are expressed in real or 2023 dollars to account for the change in the cost of living over time.1 The current method for inflation-adjustment is based on the Chained Consumer Price Index for all Urban Consumers (C-CPI-U) between 2000 and 2023 and the Consumer Price Index for all Urban Consumers Retroactive Series (R-CPI-U-RS) prior to 2000.2 The C-CPI-U measured a 4.0 percent increase in consumer prices between 2022 and 2023, down from a 7.7 per- cent increase between 2021 and 2022.\n\nThis report begins with a section discussing median household income, highlighting year-to-year comparisons by characteristics such as race and Hispanic origin, nativity, region, and education. This is followed by sections on income inequality, workers, and median earnings. The income estimates in the main sections of this report are based on the concept of money income, which is pretax and does not account for the value of in-kind transfers. Estimates of post-tax income and inequality are included in Appendix B.\n\nThis report is released alongside two other reports focused on poverty estimates and health insur- ance coverage in the United States. For poverty and health insurance estimates, refer to “Poverty in the United States: 2023” and “Health Insurance Coverage in the United States: 2023.”3\n\n## Highlights\n\n- • Real median household income was $80,610 in 2023, a 4.0 percent increase from the 2022 estimate of $77,540 (Figure 1 and Table A-1). This is the first statistically significant annual increase in real median house- hold income since 2019.4\n\n* The Census Bureau reviewed this data product to ensure appropriate access, use, and disclosure avoidance protection of the confidential source data used to produce this product (Data Management System [DMS] number: P-7534374, Disclosure Review Board [DRB] approval number: CBDRB-FY24-0434). To further protect respondent privacy, all estimates in this report have undergone additional round- ing. As a result, this year’s estimates may differ from previous publications and details may not sum to totals. All compara- tive statements have undergone statistical testing and are statistically significant at the 90 percent confidence level unless otherwise noted.\n\n- • Real median household incomes increased by 5.4 per- cent for White households and by 5.7 percent for non-Hispanic White households between 2022 and 2023. There was no significant change in median incomes for Black, Asian, and Hispanic households (Figure 2 and Table A-1).\n\nU.S. Census Bureau\n\n- • Household income rose throughout the income distri- bution, increasing 6.7 percent at the 10th percentile and 4.6 percent at the 90th percentile (Table A-3).\n\n- • Income inequality as measured by the Gini index and income percentile ratios was not signifi- cantly different between 2022 and 2023 (Figure 3 and Table A-3).\n\n- • Between 2022 and 2023, the number of total workers increased by 2.2 million, or 1.3 percent. The change in the number of full-time, year-round workers was not statistically significant (Figure 5 and Table A-6).\n\n- • Real median earnings for men who worked full-time, year-round increased by 3.0 percent, and real median earn- ings increased 1.5 percent for women who worked full-time, year-round (Figure 4 and Table A-6).\n\n- • For full-time, year-round workers, the female-to-male earnings ratio in 2023 fell to 82.7 percent from 84.0 percent in 2022 (Figure 6 and Table A-6). This is the first statistically significant annual decrease in the female-to-male earnings\n\nratio since 2003.\n\n## HOUSEHOLD INCOME BY SELECTED CHARACTERISTICS\n\nThis section focuses on median household income by selected characteristics of the householder, including race and Hispanic origin, nativity, region, and education. The householder is the person (or\n\nIncome in the United States: 2023 1",
221
+ "Figure 1.\n\nMedian Household Income and Percent Change by Selected Characteristics\n\nChange: 2022 to 2023\n\n2023 Median Income\n\n<table><thead><tr><th rowspan=\"7\">ALL HOUSEHOLDS Type of Household Family households Nonfamily households Race and Hispanic Origin of Householder White</th><th colspan=\"3\">$80,610 @</th></tr><tr><th colspan=\"2\"></th><th rowspan=\"16\"></th></tr><tr><th colspan=\"2\">$102,800 ©</th></tr><tr><th colspan=\"2\">$49,600 -@</th></tr><tr><th></th><th></th></tr><tr><th></th><th></th></tr><tr><th colspan=\"2\">$84,630 @</th></tr><tr><th>White, not Hispanic</th><th></th><th>$89,050 @</th></tr></thead><tbody><tr><td>Black</td><td colspan=\"2\" rowspan=\"2\">$56,490 @</td></tr><tr><td></td></tr><tr><td>Asian</td><td></td><td></td></tr><tr><td></td><td colspan=\"2\" rowspan=\"2\">$112,800 @ $65,540 -@</td></tr><tr><td>Hispanic (any race)</td></tr><tr><td>Age of Householder</td><td colspan=\"2\" rowspan=\"3\">$92,470 @</td></tr><tr><td>Under 65 years</td></tr><tr><td></td></tr><tr><td>65 years and older</td><td colspan=\"2\">$54,710 @</td></tr><tr><td>Nativity of Householder</td><td colspan=\"2\"></td><td></td></tr><tr><td>Native-born</td><td colspan=\"2\">$81,700 -@</td><td></td></tr><tr><td>Foreign-born</td><td></td><td></td><td></td></tr><tr><td>Region</td><td colspan=\"2\">$73,360</td><td></td></tr><tr><td>Northeast</td><td colspan=\"2\">$86,250 @</td><td></td></tr><tr><td>Midwest</td><td></td><td>@</td><td></td></tr><tr><td>South</td><td colspan=\"3\" rowspan=\"2\">$81,020 $73,280 @</td></tr><tr><td>West</td></tr><tr><td></td><td colspan=\"2\" rowspan=\"2\">$88,290 @</td><td></td></tr><tr><td>Metropolitan Statistical</td><td></td></tr><tr><td>Area (MSA) Status</td><td colspan=\"2\"></td><td></td></tr><tr><td>Inside MSA</td><td colspan=\"2\">$83,590 @</td><td></td></tr><tr><td>Inside principal cities</td><td colspan=\"2\">$73,540 @</td><td></td></tr><tr><td>Outside principal cities</td><td colspan=\"2\">$90,140 @</td><td></td></tr><tr><td>Outside MSA</td><td colspan=\"2\">$62,520 @</td><td></td></tr><tr><td colspan=\"4\">Educational Attainment of Householder’</td></tr><tr><td>No high school diploma</td><td>$36,620 @</td><td></td><td></td></tr><tr><td>High school, no college</td><td colspan=\"2\">$55,810 -@</td><td></td></tr><tr><td>Some college</td><td colspan=\"2\">$73,610 @</td><td></td></tr><tr><td>Bachelor’s degree or higher</td><td></td><td>$126,800</td><td>@</td></tr></tbody></table>\n\nsignificant change\n\n1 Householders aged 25 and older. In 2023, the median household income for this group was $82,010.\n\nNote: Households as of March of the following year. Percent change estimate may be different due to rounded components. Statistically significant indicates the change is statistically different from zero at the 90 percent confidence level. Margins of error and other related estimates and notes are available in Table A-1. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at <https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf>.\n\nSource: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC).\n\n2 Income in the United States: 2023\n\nU.S. Census Bureau\n\nLorem ipsum\n\ndolor sit",
222
+ "## Table A-1.\n\n## Income Summary Measures by Selected Characteristics: 2022 and 2023\n\n(Income in 2023 dollars, adjusted using the C-CPI-U. Households as of March of the following year. Information on confidentiality protection, sampling error, nonsampling error, and definitions is available at <https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf>)\n\n<table><thead><tr><th rowspan=\"2\">Characteristic</th><th rowspan=\"2\">Number (thousands) |</th><th colspan=\"2\">Median income (dollars)</th><th rowspan=\"2\">Number (thou- sands) |</th><th colspan=\"2\">Median income (dollars)</th><th colspan=\"2\">Percent change real median income (2023 less 2022)\"?</th></tr><tr><th>Estimate |</th><th>Margin of error! (+)</th><th>Estimate}</th><th>Margin of error! (+)</th><th></th><th>Margin of error! (+)</th></tr></thead><tbody><tr><td colspan=\"9\">OUSEHOLDS</td></tr><tr><td colspan=\"9\">All households .........0.000eeeeeeeeee 131,400 77,540 1,006| 132,200 80,610 634 1.40</td></tr><tr><td colspan=\"9\">ype of Household</td></tr><tr><td>amily households ........ 0... ccc eee cece</td><td>84,330</td><td>99,250</td><td>996</td><td>84,680</td><td>102,800</td><td>1,200</td><td></td><td>1.41</td></tr><tr><td>Married-couple ... 0... cece ccc eee eee</td><td>ee 62,180</td><td>115,200</td><td>1,166</td><td>62,300</td><td>119,400</td><td>1,576</td><td></td><td>1.55</td></tr><tr><td>Female householder, no spouse present ........</td><td>15,030</td><td>58,260</td><td>1,258</td><td>15,180</td><td>59,470</td><td>1,774</td><td></td><td>3.72</td></tr><tr><td>Male householder, no spouse present...........</td><td>7,128</td><td>76,550</td><td>2,824</td><td>7,208</td><td>81,890</td><td>2,059</td><td></td><td>4.10</td></tr><tr><td>onfamily households........... 0000 eee eee</td><td>eee 47,100</td><td>47,250</td><td>861</td><td>47,530</td><td>49,600</td><td>895</td><td></td><td>2.53</td></tr><tr><td>Female householder ..............00 0c cece</td><td>eee 24,360</td><td>41,790</td><td>812</td><td>24,680</td><td>42,140</td><td>947</td><td></td><td>2.74</td></tr><tr><td>Male householder. ......... 00. cece eee eee</td><td>eee 22,740</td><td>53,990</td><td>1,029</td><td>22,850</td><td>57,200</td><td>1,190</td><td></td><td>2.81</td></tr><tr><td colspan=\"9\">ace? and Hispanic Origin of Householder</td></tr><tr><td>ite cece cece eee cece ee eeees</td><td>101,400</td><td>80,320</td><td>905</td><td>101,900</td><td>84,630</td><td>1,182</td><td></td><td>1.60</td></tr><tr><td>White, not Hispanic 0.0.0.0. e eee eee</td><td>84,490</td><td>84,280</td><td>997</td><td>84,440</td><td>89,050</td><td>1,234</td><td></td><td>1.63</td></tr><tr><td>........ ACK</td><td>18,080</td><td>54,960</td><td>1,528</td><td>18,040</td><td>56,490</td><td>1,328</td><td></td><td>3.65</td></tr><tr><td>cece cece eee eee eee e eee SIAN eee eee ee ee eeeenaes</td><td>7,609</td><td>113,100</td><td>4,040</td><td>7,655</td><td>112,800</td><td>4,187</td><td></td><td>4.69</td></tr><tr><td>cece</td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>ISPANIC (ANY FACE)... eee eee eee</td><td>19,320</td><td>65,300</td><td>1,659</td><td>19,860</td><td>65,540</td><td>1,259</td><td></td><td>2.77</td></tr><tr><td colspan=\"9\">of Householder</td></tr><tr><td>Nder 65 oo. eee eens</td><td>94,300</td><td>89,270</td><td>985</td><td>94,590</td><td>92,470</td><td>1,088</td><td></td><td>1.46</td></tr><tr><td>years</td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>15 to 24 Years... ccc cece eee eee</td><td>6,136</td><td>54,540</td><td>3,591</td><td>5,881</td><td>54,930</td><td>2,870</td><td></td><td>7.95</td></tr><tr><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>25 to 34 YOArS ook cece eee eee</td><td>ees 20,720</td><td>83,420</td><td>1,566</td><td>20,910</td><td>85,780</td><td>1,190</td><td></td><td>2.30</td></tr><tr><td>35 to 44 years 0... cece eee eee</td><td>22,530</td><td>100,500</td><td>1,870</td><td>23,060</td><td>101,300</td><td>1,234</td><td></td><td>2.00</td></tr><tr><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>45 to 54 years... cece eee eee</td><td>eee 21,500</td><td>105,600</td><td>1,440</td><td>21,660</td><td>110,700</td><td>1,887</td><td></td><td>1.95</td></tr><tr><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>55 to 64 years 0... cee eee eee</td><td>23,410</td><td>84,470</td><td>1,899</td><td>23,080</td><td>90,640</td><td>1,773</td><td></td><td>3.11</td></tr><tr><td>years and older... 2... eee eee</td><td>eee 37,130</td><td>52,290</td><td>1,014</td><td>37,630</td><td>54,710</td><td>1,018</td><td></td><td>2.39</td></tr><tr><td colspan=\"9\">ativity of Householder</td></tr><tr><td>ALIVE-DOIN occ ccc cece e eee e eee</td><td>110,300</td><td>78,200</td><td>821</td><td>110,300</td><td>81,700</td><td>683</td><td></td><td>1.22</td></tr><tr><td>MEIGN-DOMN 2... eect eens</td><td>21,140</td><td>74,180</td><td>1,384</td><td>21,920</td><td>73,360</td><td>2,546</td><td></td><td>3.32</td></tr><tr><td>Naturalized citizen...</td><td>11,770</td><td>83,970</td><td>2,237</td><td>12,220</td><td>86,060</td><td>2,773</td><td></td><td>3.75</td></tr><tr><td>0... eee</td><td>eee</td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>Not acitizen. 0...</td><td>9,375</td><td>64,490</td><td>1,826</td><td>9,700</td><td>61,440</td><td>1,109</td><td></td><td>3.01</td></tr><tr><td>ortheast ... 0. ee eee enna</td><td>22,630</td><td>83,550</td><td>2,008</td><td>22,590</td><td>86,250</td><td>1,816</td><td></td><td>2.80</td></tr><tr><td>IAWESt. eee e eee e ee</td><td>ees 28,280</td><td>75,980</td><td>2,009</td><td>28,410</td><td>81,020</td><td>1,319</td><td></td><td>3.01</td></tr><tr><td>010 9</td><td>51,080</td><td>70,940</td><td>1,533</td><td>51,550</td><td>73,280</td><td>1,750</td><td></td><td>2.84</td></tr><tr><td>[5S</td><td>29,440</td><td>86,190</td><td>2,068</td><td>29,670</td><td>88,290</td><td>2,212</td><td></td><td>3.01</td></tr><tr><td colspan=\"9\" rowspan=\"2\">esidence* side metropolitan statistical areas 113,500 80,580 1,011 114,300 83,590 1,140 1.67</td></tr><tr><td>.............. Inside principal cities....... 00... cece eee eee</td><td>eee 43,710</td><td>72,720</td><td>1,436</td><td>43,910</td><td>73,540</td><td>1,547</td><td></td><td>2.66</td></tr><tr><td>Outside principal cities....................0005</td><td>69,770</td><td>86,530</td><td>1,542</td><td>70,360</td><td>90,140</td><td>1,087</td><td></td><td>1.94</td></tr><tr><td>utside metropolitan statistical areas.............</td><td>17,950</td><td>58,180</td><td>1,393</td><td>17,950</td><td>62,520</td><td>1,723</td><td></td><td>2.82</td></tr><tr><td>ducational Attainment of Householder</td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td colspan=\"9\">Total, aged 25 and older........... 0... 125,300 79,000 708 126,300 82,010 633 1.06</td></tr><tr><td>cee ohigh school diploma 0.00 cece eee</td><td>eee 9,632</td><td>36,230</td><td>1,553</td><td>9,546</td><td>36,620</td><td>1,162</td><td></td><td>5.29</td></tr><tr><td>..........</td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr><tr><td>igh school, no college... 2... eee eee eee IME CONEGE..</td><td>eee 31,830 33,650</td><td>53,510 71,420</td><td>801 1,389</td><td>31,810 33,830</td><td>55,810</td><td>988</td><td></td><td>2.15 2.69</td></tr></tbody></table>\n\nHOUSEHOLDS\n\nAll households . . . . . . . . . . . . . . . . . . . . . . . .\n\nType of Household\n\nFamily households . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nMarried-couple . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nFemale householder, no spouse present . . . . . . . .\n\nMale householder, no spouse present . . . . . . . . . . .\n\nNonfamily households . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nFemale householder . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nMale householder . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nRace3 and Hispanic Origin of Householder\n\nWhite . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nWhite, not Hispanic . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nBlack . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nAsian . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nHispanic (any race) . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nAge of Householder\n\nUnder 65 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\n15 to 24 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\n25 to 34 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\n35 to 44 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\n45 to 54 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\n55 to 64 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\n65 years and older . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nNativity of Householder\n\nNative-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nForeign-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nNaturalized citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nNot a citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nRegion\n\nNortheast . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nMidwest . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nSouth . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nWest . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nResidence4\n\nInside metropolitan statistical areas . . . . . . . . . . . . . .\n\nInside principal cities . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nOutside principal cities . . . . . . . . . . . . . . . . . . . . . . . .\n\nOutside metropolitan statistical areas . . . . . . . . . . . . .\n\nEducational Attainment of Householder\n\nTotal, aged 25 and older . . . . . . . . . . . . . . . . . . . . . . .\n\nNo high school diploma . . . . . . . . . . . . . . . . . . . . . . . . .\n\nHigh school, no college . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nSome college . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n\nBachelor’s degree or higher . . . . . . . . . . . . . . . . . . . . . .\n\n50,180\n\n123,000\n\n1,900\n\n51,150\n\n126,800\n\n1,462\n\n*3.1\n\n1.84\n\n* An asterisk preceding an estimate indicates change is statistically different from zero at the 90 percent confidence level.\n\n1 A margin of error (MOE) is a measure of an estimate’s variability. The larger the MOE in relation to the size of the estimate, the less reliable the estimate. This number, when added to and subtracted from the estimate, forms the 90 percent confidence interval. MOEs shown in this table are based on standard errors calculated using replicate weights.\n\n2 Calculated estimate may be different due to rounded components.\n\n3 Federal surveys give respondents the option of reporting more than one race. Therefore, two basic ways of defining a race group are possible. A group, such as Asian, may be defined as those who reported Asian and no other race (the race-alone or single-race concept) or as those who reported Asian regardless of whether they also reported another race (the race-alone-or-in-combination concept). This table shows data using the first approach (race alone). The use of the single-race population does not imply that it is the preferred method of presenting or analyzing data. The Census Bureau uses a variety of approaches. Data for American Indians and Alaska Natives, Native Hawaiians and Other Pacific Islanders, and those reporting Two or More Races are not shown separately.\n\n4 Information on metropolitan statistical areas and principal cities is available at <www.census.gov/programs-surveys/metro-micro/about/glossary.html>. Source: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC).\n\nU.S. Census Bureau\n\nIncome in the United States: 2023 15"
223
+ ],
224
+ "seconds": 21.33465420900029,
225
+ "error": null,
226
+ "device": "cpu",
227
+ "warmup_seconds": 21.674663541998598,
228
+ "extra": {
229
+ "version": "0.27.36"
230
+ }
231
+ },
232
+ "dpo-arxiv-2305.18290": {
233
+ "pages": [
234
+ "4\n\n2024\n\n2\n\n0\n\n2 l u J 9 2 ] G L . s c [ 3 v 0 9 2 8 1 . 5 0 3 2\n\n:\n\nv\n\narXiv\n\ni\n\nX\n\nr\n\na\n\n## Direct Preference Optimization: Your Language Model is Secretly a Reward Model\n\nRafael Rafailov∗†\n\n## Archit Sharma∗†\n\nEric Mitchell∗†\n\nStefano Ermon†‡\n\nChristopher D. Manning†\n\nChelsea Finn†\n\n†Stanford University ‡CZ Biohub {rafailov,architsh,eric.mitchell}@cs.stanford.edu\n\n## Abstract\n\nWhile large-scale unsupervised language models (LMs) learn broad world knowl- edge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these prefer- ences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call Direct Prefer- ence Optimization (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sen- timent of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.\n\n## 1 Introduction\n\nLarge unsupervised language models (LMs) trained on very large datasets acquire surprising capabili- ties [11, 7, 42, 8]. However, these models are trained on data generated by humans with a wide variety of goals, priorities, and skillsets. Some of these goals and skillsets may not be desirable to imitate; for example, while we may want our AI coding assistant to understand common programming mistakes in order to correct them, nevertheless, when generating code, we would like to bias our model toward the (potentially rare) high-quality coding ability present in its training data. Similarly, we might want our language model to be aware of a common misconception believed by 50% of people, but we certainly do not want the model to claim this misconception to be true in 50% of queries about it! In other words, selecting the model’s desired responses and behavior from its very wide knowledge and abilities is crucial to building AI systems that are safe, performant, and controllable [28]. While existing methods typically steer LMs to match human preferences using reinforcement learning (RL),\n\n∗Equal contribution; more junior authors listed earlier.\n\n37th Conference on Neural Information Processing Systems (NeurIPS 2023).",
235
+ "where β is a parameter controlling the deviation from the base reference policy πref, namely the ini- tial SFT model πSFT. In practice, the language model policy πθ is also initialized to πSFT. The added constraint is important, as it prevents the model from deviating too far from the distri- bution on which the reward model is accurate, as well as maintaining the generation diversity and preventing mode-collapse to single high-reward answers. Due to the discrete nature of lan- guage generation, this objective is not differentiable and is typically optimized with reinforce- ment learning. The standard approach [51, 40, 1, 28] has been to construct the reward function r(x,y) = rϕ(x,y) − β(logπθ(y | x) − logπref(y | x)), and maximize using PPO [39].\n\n## 4 Direct Preference Optimization\n\nMotivated by the challenges of applying reinforcement learning algorithms on large-scale problems such as fine-tuning language models, our goal is to derive a simple approach for policy optimization using preferences directly. Unlike prior RLHF methods, which learn a reward and then optimize it via RL, our approach leverages a particular choice of reward model parameterization that enables extraction of its optimal policy in closed form, without an RL training loop. As we will describe next in detail, our key insight is to leverage an analytical mapping from reward functions to optimal policies, which enables us to transform a loss function over reward functions into a loss function over policies. This change-of-variables approach avoids fitting an explicit, standalone reward model, while still optimizing under existing models of human preferences, such as the Bradley-Terry model. In essence, the policy network represents both the language model and the (implicit) reward.\n\nDeriving the DPO objective. We start with the same RL objective as prior work, Eq. 3, under a general reward function r. Following prior work [31, 30, 19, 15], it is straightforward to show that the optimal solution to the KL-constrained reward maximization objective in Eq. 3 takes the form:\n\nπr(y | x) = 1 Z(x) πref(y | x)exp β r(x,y) , (4)\n\nwhere Z(x) = 7, Tret(y | ©) exp (4r(@, u)) is the partition function. See Appendix A.1 for a complete derivation. Even if we use the MLE estimate ry of the ground-truth reward function r™, it is still expensive to estimate the partition function 7(x) [19, 15], which makes this representation hard to utilize in practice. However, we can rearrange Eq. 4 to express the reward function in terms of its corresponding optimal policy 7, the reference policy 7yer, and the unknown partition function Z(-). Specifically, we first take the logarithm of both sides of Eq. 4 and then with some algebra we obtain:\n\nr(x,y) = β log πr(y | x) πref(y | x) + β logZ(x). (5)\n\nWe can apply this reparameterization to the ground-truth reward r∗ and corresponding optimal model π∗. Fortunately, the Bradley-Terry model depends only on the difference of rewards between two completions, i.e., p∗(y1 ≻ y2 | x) = σ(r∗(x,y1) − r∗(x,y2)). Substituting the reparameterization in Eq. 5 for r∗(x,y) into the preference model Eq. 1, the partition function cancels, and we can express the human preference probability in terms of only the optimal policy π∗ and reference policy πref. Thus, the optimal RLHF policy π∗ under the Bradley-Terry model satisfies the preference model:\n\n1 1 +exp (Blog T*(yolx) Blog w{usl Tret (Y2|x) Tret (Ys |@) (yi > yo | x)=\n\nThe derivation is in Appendix A.2. While Eq. 6 uses the Bradley-Terry model, we can similarly derive expressions under the more general Plackett-Luce models [32, 23], shown in Appendix A.3.\n\nNow that we have the probability of human preference data in terms of the optimal policy rather than the reward model, we can formulate a maximum likelihood objective for a parametrized policy πθ. Analogous to the reward modeling approach (i.e. Eq. 2), our policy objective becomes:\n\nLDPO(πθ;πref) = −E(x,yw,yl)∼D logσ β log πθ(yw | x) πref(yw | x) − β log πθ(yl | x) πref(yl | x) . (7)\n\nThis way, we fit an implicit reward using an alternative parameterization, whose optimal policy is simply πθ. Moreover, since our procedure is equivalent to fitting a reparametrized Bradley-Terry\n\n4\n\n(6)",
236
+ "model, it enjoys certain theoretical properties, such as consistencies under suitable assumption of the preference data distribution [4]. In Section 5, we further discuss theoretical properties of DPO in relation to other works.\n\nWhat does the DPO update do? For a mechanistic understanding of DPO, it is useful to analyze the gradient of the loss function LDPO. The gradient with respect to the parameters θ can be written as:\n\n∇θLDPO(πθ;πref) =\n\nVoLppo(T93 Met) = - FBeyem~P | olFolewe) Folin) | Yologn(yw |2) ~ Voloemw le) |], _j—————~_ ———_“_— e+ F< VeeS__\n\nhigher weight when reward estimate is wrong increase likelihood of yw decrease likelihood of yl\n\nwhere ˆrθ(x,y) = β log πθ(y|x)\n\nπref(y|x) is the reward implicitly defined by the language model πθ and refer- ence model πref (more in Section 5). Intuitively, the gradient of the loss function LDPO increases the likelihood of the preferred completions yw and decreases the likelihood of dispreferred completions yl. Importantly, the examples are weighed by how much higher the implicit reward model ˆrθ rates the dispreferred completions, scaled by β, i.e, how incorrectly the implicit reward model orders the completions, accounting for the strength of the KL constraint. Our experiments suggest the importance of this weighting, as a naïve version of this method without the weighting coefficient can cause the language model to degenerate (Appendix Table 3).\n\nDPO outline. The general DPO pipeline is as follows: 1) Sample completions y1,y2 ∼ πref(· | x) for every prompt x, label with human preferences to construct the offline dataset of preferences\n\nD = {x(i),y(i)\n\nw ,yl)(i)}N\n\ni=1 and 2) optimize the language model πθ to minimize LDPO for the given πref and D and desired β. In practice, one would like to reuse preference datasets publicly available, rather than generating samples and gathering human preferences. Since the preference datasets are sampled using πSFT, we initialize πref = πSFT whenever available. However, when πSFT is not available, we initialize πref by maximizing likelihood of preferred completions (x,yw), that Ex,yw∼D [logπ(yw | x)]. This procedure helps mitigate the distribution shift is, πref = argmaxπ between the true reference distribution which is unavailable, and πref used by DPO. Further details related to the implementation and hyperparameters can be found in Appendix B.\n\n## 5 Theoretical Analysis of DPO\n\nIn this section, we give further interpretation of the DPO method, provide theoretical backing, and relate advantages of DPO to issues with actor critic algorithms used for RLHF (such as PPO [39]).\n\n## 5.1 Your Language Model Is Secretly a Reward Model\n\nDPO is able to bypass both fitting an explicit reward and performing RL to learn the policy using a single maximum likelihood objective. Note the optimization objective Eq. 5 is equivalent to a Bradley-Terry model with a reward parameterization r∗(x,y) = β log π∗ θ(y|x) πref(y|x) and we optimize our parametric model πθ, equivalently to the reward model optimization in Eq. 2 under the change of variables. In this section we will build the theory behind this reparameterization, show that it does not constrain the class of learned reward models, and allows for the exact recovery of the optimal policy. We begin with by defining an equivalence relation between reward functions.\n\ntwo reward functions r(x,y) and r′(x,y) are equivalent\n\nDefinition 1. We say that r(x,y) − r′(x,y) = f(x) for some function f. iff\n\nIt is easy to see that this is indeed an equivalence relation, which partitions the set of reward functions into classes. We can state the following two lemmas:\n\nLemma 1. Under the Plackett-Luce, and in particular the Bradley-Terry, preference framework, two reward functions from the same class induce the same preference distribution.\n\nLemma 2. Two reward functions from the same equivalence class induce the same optimal policy under the constrained RL problem.\n\nThe proofs are straightforward and we defer them to Appendix A.5. The first lemma is a well-known under-specification issue with the Plackett-Luce family of models [32]. Due to this under-specification,\n\n5",
237
+ "DPO’s reward/KL tradeoff strictly dominates PPO. Second, DPO achieves a better frontier than PPO, even when PPO can access ground truth rewards (PPO-GT).\n\n## 6.2 Can DPO scale to real preference datasets?\n\nNext, we evaluate fine-tuning performance of DPO on summarization and single-turn dialogue. For summarization, automatic evaluation metrics such as ROUGE can be poorly correlated with human preferences [40], and prior work has found that fine-tuning LMs using PPO on human preferences to provide more effective summaries. We evaluate different methods by sampling completions on the test split of TL;DR summarization dataset, and computing the average win rate against reference completions in the test set. The completions for all methods are sampled at temperatures varying from 0.0 to 1.0, and the win rates are shown in Figure 2 (right). DPO, PPO and Preferred-FT all fine-tune the same GPT-J SFT model4. We find that DPO has a win rate of approximately 61% at a temperature of 0.0, exceeding the performance of PPO at 57% at its optimal sampling temperature of 0.0. DPO also achieves a higher maximum win rate compared to the best of N baseline. We note that we did not meaningfully tune DPO’s β hyperparameter, so these results may underestimate DPO’s potential. Moreover, we find DPO to be much more robust to the sampling temperature than PPO, the performance of which can degrade to that of the base GPT-J model at high temperatures. Preferred-FT does not improve significantly over the SFT model. We also compare DPO and PPO head-to-head in human evaluations in Section 6.4, where DPO samples at temperature 0.25 were preferred 58% times over PPO samples at temperature 0.\n\nOn single-turn dialogue, we evaluate the different methods on the subset of the test split of the Anthropic HH dataset [1] with one step of human-assistant interaction. GPT-4 evaluations use the preferred completions on the test as the reference to compute the win rate for different methods. As there is no standard SFT model for this task, we start with a pre-trained Pythia-2.8B, use Preferred-FT to train a reference model on the chosen completions such that completions are within distribution of the model, and then train using DPO. We also compare against the best of 128 Preferred-FT completions (we found the Best of N baseline plateaus at 128 completions for this task; see Appendix Figure 4) and a 2-shot prompted version of the Pythia-2.8B base model, finding DPO performs as well or better for the best-performing temperatures for each method. We also evaluate an RLHF model trained with PPO on the Anthropic HH dataset 5 from a well-known source 6, but are unable to find a prompt or sampling temperature that gives performance better than the base Pythia-2.8B model. Based on our results from TL;DR and the fact that both methods optimize the same reward function, we consider Best of 128 a rough proxy for PPO-level performance. Overall, DPO is the only computationally efficient method that improves over the preferred completions in the Anthropic HH dataset, and provides similar or better performance to the computationally demanding Best of 128 baseline. Finally, Figure 3 shows that DPO converges to its best performance relatively quickly.\n\n## 6.3 Generalization to a new input distribution\n\nWin rate vs. ground truth\n\n<table><tbody><tr><td>lo further compare the performance of PPO and DPO un- Jer distribution shifts, we evaluate the PPO and DPO poli-</td><td>Alg.</td><td>Temp 0</td><td>Temp 0.25</td></tr><tr><td>split CNN/DailyMail dataset [26], using the best sampling tempet- itures from TL;DR (0 and 0.25). The results are presented n Table 1. We computed the GPT-4 win rate against the</td><td>ph) 1; tyth CNN/DailyMail</td><td>GPT-4 win summaries for input</td><td>rates vs. ground out-of-distributior articles.</td></tr></tbody></table>\n\nground-truth summaries in the datasets, using the same GPT-\n\n4 (C) prompt we used for Reddit TL;DR, but replacing the words “forum post” with “news article”. For this new distribution, DPO continues to outperform the PPO policy by a significant margin. This experiment provides initial evidence that DPO policies can generalize similarly well to PPO policies, even though DPO does not use the additional unlabeled Reddit TL;DR prompts that PPO uses.\n\n4https://huggingface.co/CarperAI/openai_summarize_tldr_sft 5https://huggingface.co/reciprocate/ppo_hh_pythia-6B 6https://github.com/CarperAI/trlx/tree/main/examples/hh\n\n9"
238
+ ],
239
+ "seconds": 20.124369707998994,
240
+ "error": null,
241
+ "device": "cpu",
242
+ "warmup_seconds": 21.674663541998598,
243
+ "extra": {
244
+ "version": "0.27.36"
245
+ }
246
+ },
247
+ "naca-report-1135-scan": {
248
+ "pages": [
249
+ "## REPORT 1135\n\n## EQUATIONS, TABLES, AND CHARTS FOR COMPRESSIBLE FLOW '\n\n## By Ames Researce STAFF\n\n## SUMMARY\n\nThis report, which is a revision and extension of NACA TN 1428, presents a compilation of equations, tables, and charts useful in the analysis of high-speed flow of a compressible fluid. The equations provide relations for continuous one-dimensional flow, normal and oblique shock waves, and Prandtl-Meyer expansions for both perfect and imperfect gases. The tables present useful dimensionless ratios for continuous one-dimen- sional flow and for normal shock waves as functions of Mach number for air considered as a perfect gas. One series of charts presents the characteristics of the flow of air (considered a perfect gas) for oblique shock waves and for cones in a supersonic air stream. A second series shows the effects of caloric imperfec- tions on continuous one-dimensional flow and on the flow through normal and oblique shock waves.\n\n## INTRODUCTION\n\nThe practical analysis of compressible flow involves fre- quent application of a few basic results. A convenient compilation of equations, tables, and charts embodying these results is therefore of great assistance in both research and design. The present report makes one of the first such compilations (ref. 1) more readily available in a revised and extended form. ‘The revisions include a complete rewriting of the lists of equations, as well as the correction of certain typographical errors which appeared in the earlier work. The extensions are primarily in the directions dictated by increasing flight speeds, that is, to higher Mach numbers and to higher temperatures with the accompanying gaseous imperfections.\n\nCompilations similar to those of reference 1 have been given in other publications, as, for example, references 2 through 6. These references have been utilized in extending the tables and charts to higher values of the Mach number. The extension to imperfect gases is based on the relations presented in references 7 and 8.\n\n## SYMBOLS AND NOTATION\n\n<table><tbody><tr><td>. normal-force coefficient for cones,</td><td>normal force ————g—— aod</td></tr><tr><td>specific heat at constant pressure specific heat at constant volume enthalpy per unit mass, u-+ pv characteristic reference length</td><td></td></tr><tr><td>Mach number, r</td><td></td></tr><tr><td>velocity components respectively, to free-stream flow velocity components normal and respectively, to oblique shock wave speed of flow maximum speed obtainable by zero absolute temperature external work performed per unit mass angle of attack</td><td>direction tangential, expanding to</td></tr><tr><td>Reynolds number, oe base area of cone entropy per unit mass ~ absolute temperature ” internal energy per unit mass ‘i 1 specific volume, &gt; parallel and vi ratio of specific heats, a e angle of flow deflection across an wave shock-wave angle measured from direction molecular vibrational-energy constant Mach angle, sin~! iv</td><td>perpendicular oblique shock upstream flow</td></tr><tr><td></td><td></td></tr><tr><td></td><td></td></tr><tr><td></td><td></td></tr><tr><td>absolute viscosity Prandtl-Meyer angle (angle through supersonic stream is turned to M=1 to M&gt;1) -</td><td>which a expand from</td></tr></tbody></table>\n\n## PRIMARY SYMBOLS\n\n- speed of sound 8\n\n- cross-sectional area of stream tube or channel s\n\n! Supersedes NACA TN 1428, ‘‘Notes and Tables for Use in the Analysis of Supersonic Flow” by the Staff of the Ames 1- by 3-foot Supersonic Wind-Tunnel Section, 1947, 2 When used without subscripts, p, p, and 7 denote static pressure, static density, and static temperature, respectively.\n\n613"
250
+ ],
251
+ "seconds": 8.442342958001973,
252
+ "error": null,
253
+ "device": "cpu",
254
+ "warmup_seconds": 21.674663541998598,
255
+ "extra": {
256
+ "version": "0.27.36"
257
+ }
258
+ }
259
+ },
260
+ "pypdf": {
261
+ "census-income-2023": {
262
+ "pages": [
263
+ "U.S. Census Bureau Income in the United States: 2023 1\nIncome in the United States: 2023\nINTRODUCTION\nThe U.S. Census Bureau collects \ndata and publishes estimates on \nincome, earnings, and inequal -\nity in order to evaluate national \neconomic trends and to under-\nstand their effect on the well-\nbeing of households, families, and \nindividuals. \nThe estimates in this report \nare based on data collected in \nthe 2024 and earlier Current \nPopulation Survey Annual Social \nand Economic Supplements (CPS \nASEC) conducted by the Census \nBureau.* This report presents \nestimates on income in the United \nStates for calendar year 2023. \nHistorical estimates are expressed \nin real or 2023 dollars to account \nfor the change in the cost of living \nover time.1 The current method \nfor inflation-adjustment is based \non the Chained Consumer Price \nIndex for all Urban Consumers \n(C-CPI-U) between 2000 and \n2023 and the Consumer Price \nIndex for all Urban Consumers \nRetroactive Series (R-CPI-U-RS) \nprior to 2000. 2 The C-CPI-U \nmeasured a 4.0 percent increase \nin consumer prices between 2022 \nand 2023, down from a 7.7 per-\ncent increase between 2021 and \n2022. \n* The Census Bureau reviewed this data \nproduct to ensure appropriate access, use, \nand disclosure avoidance protection of the \nconfidential source data used to produce \nthis product (Data Management System \n[DMS] number: P-7534374, Disclosure \nReview Board [DRB] approval number: \nCBDRB-FY24-0434). To further protect \nrespondent privacy, all estimates in this \nreport have undergone additional round -\ning. As a result, this year’s estimates may \ndiffer from previous publications and \ndetails may not sum to totals. All compara -\ntive statements have undergone statistical \ntesting and are statistically significant at \nthe 90 percent confidence level unless \notherwise noted.\nThis report begins with a section \ndiscussing median household \nincome, highlighting year-to-year \ncomparisons by characteristics \nsuch as race and Hispanic origin, \nnativity, region, and education. \nThis is followed by sections on \nincome inequality, workers, and \nmedian earnings. The income \nestimates in the main sections \nof this report are based on the \nconcept of money income, which \nis pretax and does not account \nfor the value of in-kind transfers. \nEstimates of post-tax income \nand inequality are included in \nAppendix B.\nThis report is released alongside \ntwo other reports focused on \npoverty estimates and health insur -\nance coverage in the United States. \nFor poverty and health insurance \nestimates, refer to “Poverty in the \nUnited States: 2023” and “Health \nInsurance Coverage in the United \nStates: 2023.”3 \nHighlights\n • Real median household income \nwas $80,610 in 2023, a 4.0 \npercent increase from the 2022 \nestimate of $77,540 (Figure 1 \nand Table A-1). This is the first \nstatistically significant annual \nincrease in real median house -\nhold income since 2019. 4 \n • Real median household \nincomes increased by 5.4 per -\ncent for White households and \nby 5.7 percent for non-Hispanic \nWhite households between \n2022 and 2023. There was no \nsignificant change in median \nincomes for Black, Asian, and \nHispanic households (Figure 2 \nand Table A-1).\n • Household income rose \nthroughout the income distri -\nbution, increasing 6.7 percent \nat the 10th percentile and 4.6 \npercent at the 90th percentile \n(Table A-3).\n • Income inequality as measured \nby the Gini index and income \npercentile ratios was not signifi -\ncantly different between 2022 \nand 2023 (Figure 3 and Table \nA-3). \n • Between 2022 and 2023, \nthe number of total workers \nincreased by 2.2 million, or \n1.3 percent. The change in the \nnumber of full-time, year-round \nworkers was not statistically \nsignificant (Figure 5 and Table \nA-6).\n • Real median earnings for \nmen who worked full-time, \nyear-round increased by 3.0 \npercent, and real median earn -\nings increased 1.5 percent for \nwomen who worked full-time, \nyear-round (Figure 4 and Table \nA-6). \n • For full-time, year-round \nworkers, the female-to-male \nearnings ratio in 2023 fell to \n82.7 percent from 84.0 percent \nin 2022 (Figure 6 and Table \nA-6). This is the first statistically \nsignificant annual decrease in \nthe female-to-male earnings \nratio since 2003.\nHOUSEHOLD INCOME BY \nSELECTED CHARACTERISTICS \nThis section focuses on median \nhousehold income by selected \ncharacteristics of the householder, \nincluding race and Hispanic origin, \nnativity, region, and education. \nThe householder is the person (or ",
264
+ "2 Income in the United States: 2023 U.S. Census Bureau\nALL HOUSEHOLDS\nType of Household\nFamily households\nNonfamily households\nRace and Hispanic \n Origin of Householder\nWhite\n White/comma.tab not Hispanic\nBlack\nAsian\nHispanic (any race)\nAge of Householder\nUnder /six.tab/five.tab years\n/six.tab/five.tab years and older\nNativity of Householder\nNative-born\nForeign-born\nRegion\nNortheast\nMidwest\nSouth\nWest\nMetropolitan Statistical \n Area (MSA) Status\nInside MSA\n Inside principal cities\n Outside principal cities\nOutside MSA \nEducational Attainment \n of Householder/one.num \nNo high school diploma\nHigh school/comma.tab no college\nSome college\nBachelor’s degree or higher\nChange/colon.tab /two.tab/zero.tab/two.tab/two.tab to /two.tab/zero.tab/two.tab/three.tab/two.tab/zero.tab/two.tab/three.tab Median Income\nFigure /one.tab/period.tab\nMedian Household Income and Percent Change by Selected Characteristics\n1 Householders aged 25 and older. In 2023, the median household income for this group was $82,010. \nNote: Households as of March of the following year. Percent change estimate may be different due to rounded components/period.tab \nStatistically significant indicates the change is statistically different from zero at the 90 percent confidence level. Margins of error \nand other related estimates and notes are available in Table A-1. Information on confidentiality protection, sampling error, \nnonsampling error, and definitions is available at <https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf>.\nSource/colon.tab U/period.tabS/period.tab Census Bureau/comma.tab Current Population Survey/comma.tab /two.tab/zero.tab/two.tab/three.tab and /two.tab/zero.tab/two.tab/four.tab Annual Social and Economic Supplements (CPS ASEC)/period.tab\nDenotes a statistically \nsignificant change\n/four.tab/period.tab/zero.tab\n/three.tab/period.tab/six.tab\n/five.tab/period.tab/zero.tab\n/five.tab/period.tab/four.tab\n/five.tab/period.tab/seven.tab\n/two.tab/period.tab/eight.tab\n/three.tab/period.tab/six.tab\n/four.tab/period.tab/six.tab\n-/one.tab/period.tab/one.tab\n/three.tab/period.tab/two.tab\n/three.tab/period.tab/three.tab\n/two.tab/period.tab/four.tab\n/three.tab/period.tab/seven.tab\n/one.tab/period.tab/one.tab\n-/zero.tab/period.tab/two.tab\n/four.tab/period.tab/two.tab\n/one.tab/period.tab/one.tab\n/four.tab/period.tab/three.tab\n/three.tab/period.tab/one.tab\n/three.tab/period.tab/one.tab\n/four.tab/period.tab/five.tab\n/zero.tab/period.tab/four.tab\n/seven.tab/period.tab/five.tab\n/six.tab/period.tab/six.tab\nLorem ipsum \ndolor sit \n/dollar.tab/eight.tab/zero.tab/comma.tab/six.tab/one.tab/zero.tab\n/dollar.tab/one.tab/zero.tab/two.tab/comma.tab/eight.tab/zero.tab/zero.tab\n/dollar.tab/four.tab/nine.tab/comma.tab/six.tab/zero.tab/zero.tab\n/dollar.tab/eight.tab/four.tab/comma.tab/six.tab/three.tab/zero.tab\n/dollar.tab/eight.tab/nine.tab/comma.tab/zero.tab/five.tab/zero.tab\n/dollar.tab/five.tab/six.tab/comma.tab/four.tab/nine.tab/zero.tab\n/dollar.tab/one.tab/one.tab/two.tab/comma.tab/eight.tab/zero.tab/zero.tab\n/dollar.tab/six.tab/five.tab/comma.tab/five.tab/four.tab/zero.tab\n/dollar.tab/nine.tab/two.tab/comma.tab/four.tab/seven.tab/zero.tab\n/dollar.tab/five.tab/four.tab/comma.tab/seven.tab/one.tab/zero.tab\n/dollar.tab/eight.tab/one.tab/comma.tab/seven.tab/zero.tab/zero.tab\n/dollar.tab/seven.tab/three.tab/comma.tab/three.tab/six.tab/zero.tab\n/dollar.tab/eight.tab/six.tab/comma.tab/two.tab/five.tab/zero.tab\n/dollar.tab/eight.tab/one.tab/comma.tab/zero.tab/two.tab/zero.tab\n/dollar.tab/seven.tab/three.tab/comma.tab/two.tab/eight.tab/zero.tab\n/dollar.tab/eight.tab/eight.tab/comma.tab/two.tab/nine.tab/zero.tab\n/dollar.tab/eight.tab/three.tab/comma.tab/five.tab/nine.tab/zero.tab\n/dollar.tab/seven.tab/three.tab/comma.tab/five.tab/four.tab/zero.tab\n/dollar.tab/nine.tab/zero.tab/comma.tab/one.tab/four.tab/zero.tab\n/dollar.tab/six.tab/two.tab/comma.tab/five.tab/two.tab/zero.tab\n/dollar.tab/three.tab/six.tab/comma.tab/six.tab/two.tab/zero.tab\n/dollar.tab/five.tab/five.tab/comma.tab/eight.tab/one.tab/zero.tab\n/dollar.tab/seven.tab/three.tab/comma.tab/six.tab/one.tab/zero.tab\n/dollar.tab/one.tab/two.tab/six.tab/comma.tab/eight.tab/zero.tab/zero.tab",
265
+ "U.S. Census Bureau Income in the United States: 2023 15\nTable A-1.\nIncome Summary Measures by Selected Characteristics: 2022 and 2023\n(Income in 2023 dollars, adjusted using the C-CPI-U. Households as of March of the following year. \nInformation on confidentiality protection, sampling error, nonsampling error, and definitions is available at \n<https:/ /www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf>)\nCharacteristic\n2022 2023 Percent change in \nreal median income \n(2023 less 2022)*, 2\nNumber \n(thousands)\nMedian income \n(dollars) Number \n(thou-\nsands)\nMedian income \n(dollars)\nEstimate\nMargin of \nerror1 (±) Estimate\nMargin of \nerror1 (±) Estimate\nMargin of \nerror1 (±)\nHOUSEHOLDS\n   All households . . . . . . . . . . . . . . . . . . . . . . . . 131,400 77,540 1,006 132,200 80,610 634 *4.0 1.40\nType of Household\nFamily households . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 84,330 99,250 996 84,680 102,800 1,200 *3.6 1.41\n Married-couple . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 62,180 115,200 1,166 62,300 119,400 1,576 *3.6 1.55\n Female householder, no spouse present . . . . . . . . 15,030 58,260 1,258 15,180 59,470 1,774 2.1 3.72\n Male householder, no spouse present . . . . . . . . . . . 7,128 76,550 2,824 7,208 81,890 2,059 *7.0 4.10\nNonfamily households . . . . . . . . . . . . . . . . . . . . . . . . . . . 47,100 47,250 861 47,530 49,600 895 *5.0 2.53\n Female householder . . . . . . . . . . . . . . . . . . . . . . . . . . 24,360 41,790 812 24,680 42,140 947 0.8 2.74\n Male householder . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 22,740 53,990 1,029 22,850 57,200 1,190 *5.9 2.81\nRace3 and Hispanic Origin of Householder\nWhite . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 101,400 80,320 905 101,900 84,630 1,182 *5.4 1.60\n White, not Hispanic . . . . . . . . . . . . . . . . . . . . . . . . . . . 84,490 84,280 997 84,440 89,050 1,234 *5.7 1.63\nBlack . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 18,080 54,960 1,528 18,040 56,490 1,328 2.8 3.65\nAsian . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 7,609 113,100 4,040 7,655 112,800 4,187 –0.2 4.69\nHispanic (any race) . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 19,320 65,300 1,659 19,860 65,540 1,259 0.4 2.77\nAge of Householder\nUnder 65 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 94,300 89,270 985 94,590 92,470 1,088 *3.6 1.46\n 15 to 24 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 6,136 54,540 3,591 5,881 54,930 2,870 0.7 7.95\n 25 to 34 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 20,720 83,420 1,566 20,910 85,780 1,190 *2.8 2.30\n 35 to 44 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 22,530 100,500 1,870 23,060 101,300 1,234 0.8 2.00\n 45 to 54 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 21,500 105,600 1,440 21,660 110,700 1,887 *4.9 1.95\n 55 to 64 years . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 23,410 84,470 1,899 23,080 90,640 1,773 *7.3 3.11\n65 years and older . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 37,130 52,290 1,014 37,630 54,710 1,018 *4.6 2.39\nNativity of Householder\nNative-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 110,300 78,200 821 110,300 81,700 683 *4.5 1.22\nForeign-born . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 21,140 74,180 1,384 21,920 73,360 2,546 –1.1 3.32\n Naturalized citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . . 11,770 83,970 2,237 12,220 86,060 2,773 2.5 3.75\n Not a citizen . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 9,375 64,490 1,826 9,700 61,440 1,109 *–4.7 3.01\nRegion\nNortheast . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 22,630 83,550 2,008 22,590 86,250 1,816 *3.2 2.80\nMidwest . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 28,280 75,980 2,009 28,410 81,020 1,319 *6.6 3.01\nSouth . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 51,080 70,940 1,533 51,550 73,280 1,750 *3.3 2.84\nWest . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 29,440 86,190 2,068 29,670 88,290 2,212 2.4 3.01\nResidence4\nInside metropolitan statistical areas . . . . . . . . . . . . . . 113,500 80,580 1,011 114,300 83,590 1,140 *3.7 1.67\n Inside principal cities . . . . . . . . . . . . . . . . . . . . . . . . . . 43,710 72,720 1,436 43,910 73,540 1,547 1.1 2.66\n Outside principal cities . . . . . . . . . . . . . . . . . . . . . . . . 69,770 86,530 1,542 70,360 90,140 1,087 *4.2 1.94\nOutside metropolitan statistical areas . . . . . . . . . . . . . 17,950 58,180 1,393 17,950 62,520 1,723 *7.5 2.82\nEducational Attainment of Householder\n Total, aged 25 and older . . . . . . . . . . . . . . . . . . . . . . . 125,300 79,000 708 126,300 82,010 633 *3.8 1.06\nNo high school diploma . . . . . . . . . . . . . . . . . . . . . . . . . 9,632 36,230 1,553 9,546 36,620 1,162 1.1 5.29\nHigh school, no college . . . . . . . . . . . . . . . . . . . . . . . . . . 31,830 53,510 801 31,810 55,810 988 *4.3 2.15\nSome college . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 33,650 71,420 1,389 33,830 73,610 1,540 *3.1 2.69\nBachelor’s degree or higher . . . . . . . . . . . . . . . . . . . . . . 50,180 123,000 1,900 51,150 126,800 1,462 *3.1 1.84\n* An asterisk preceding an estimate indicates change is statistically different from zero at the 90 percent confidence level.\n1 A margin of error (MOE) is a measure of an estimate’s variability. The larger the MOE in relation to the size of the estimate, the less reliable the estimate. This number, \nwhen added to and subtracted from the estimate, forms the 90 percent confidence interval. MOEs shown in this table are based on standard errors calculated using \nreplicate weights.\n2 Calculated estimate may be different due to rounded components.\n3 Federal surveys give respondents the option of reporting more than one race. Therefore, two basic ways of defining a race group are possible. A group, such as \nAsian, may be defined as those who reported Asian and no other race (the race-alone or single-race concept) or as those who reported Asian regardless of whether they \nalso reported another race (the race-alone-or-in-combination concept). This table shows data using the first approach (race alone). The use of the single-race population \ndoes not imply that it is the preferred method of presenting or analyzing data. The Census Bureau uses a variety of approaches. Data for American Indians and Alaska \nNatives, Native Hawaiians and Other Pacific Islanders, and those reporting Two or More Races are not shown separately.\n4 Information on metropolitan statistical areas and principal cities is available at <www.census.gov/programs-surveys/metro-micro/about/glossary.html>.\nSource: U.S. Census Bureau, Current Population Survey, 2023 and 2024 Annual Social and Economic Supplements (CPS ASEC)."
266
+ ],
267
+ "seconds": 0.0914645830016525,
268
+ "error": null,
269
+ "device": "cpu",
270
+ "warmup_seconds": 0.08742583300045226,
271
+ "extra": {
272
+ "version": "6.21.4"
273
+ }
274
+ },
275
+ "dpo-arxiv-2305.18290": {
276
+ "pages": [
277
+ "Direct Preference Optimization:\nYour Language Model is Secretly a Reward Model\nRafael Rafailov∗ † Archit Sharma∗ † Eric Mitchell∗ †\nStefano Ermon†‡ Christopher D. Manning† Chelsea Finn†\n†Stanford University ‡CZ Biohub\n{rafailov,architsh,eric.mitchell}@cs.stanford.edu\nAbstract\nWhile large-scale unsupervised language models (LMs) learn broad world knowl-\nedge and some reasoning skills, achieving precise control of their behavior is\ndifficult due to the completely unsupervised nature of their training. Existing\nmethods for gaining such steerability collect human labels of the relative quality of\nmodel generations and fine-tune the unsupervised LM to align with these prefer-\nences, often with reinforcement learning from human feedback (RLHF). However,\nRLHF is a complex and often unstable procedure, first fitting a reward model that\nreflects the human preferences, and then fine-tuning the large unsupervised LM\nusing reinforcement learning to maximize this estimated reward without drifting\ntoo far from the original model. In this paper we introduce a new parameterization\nof the reward model in RLHF that enables extraction of the corresponding optimal\npolicy in closed form, allowing us to solve the standard RLHF problem with only a\nsimple classification loss. The resulting algorithm, which we call Direct Prefer-\nence Optimization (DPO), is stable, performant, and computationally lightweight,\neliminating the need for sampling from the LM during fine-tuning or performing\nsignificant hyperparameter tuning. Our experiments show that DPO can fine-tune\nLMs to align with human preferences as well as or better than existing methods.\nNotably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sen-\ntiment of generations, and matches or improves response quality in summarization\nand single-turn dialogue while being substantially simpler to implement and train.\n1 Introduction\nLarge unsupervised language models (LMs) trained on very large datasets acquire surprising capabili-\nties [11, 7, 42, 8]. However, these models are trained on data generated by humans with a wide variety\nof goals, priorities, and skillsets. Some of these goals and skillsets may not be desirable to imitate; for\nexample, while we may want our AI coding assistant to understand common programming mistakes\nin order to correct them, nevertheless, when generating code, we would like to bias our model toward\nthe (potentially rare) high-quality coding ability present in its training data. Similarly, we might want\nour language model to be aware of a common misconception believed by 50% of people, but we\ncertainly do not want the model to claim this misconception to be true in 50% of queries about it!\nIn other words, selecting the model’s desired responses and behavior from its very wide knowledge\nand abilities is crucial to building AI systems that are safe, performant, and controllable [28]. While\nexisting methods typically steer LMs to match human preferences using reinforcement learning (RL),\n∗Equal contribution; more junior authors listed earlier.\n37th Conference on Neural Information Processing Systems (NeurIPS 2023).\narXiv:2305.18290v3 [cs.LG] 29 Jul 2024",
278
+ "where β is a parameter controlling the deviation from the base reference policy πref, namely the ini-\ntial SFT model πSFT. In practice, the language model policy πθ is also initialized to πSFT. The\nadded constraint is important, as it prevents the model from deviating too far from the distri-\nbution on which the reward model is accurate, as well as maintaining the generation diversity\nand preventing mode-collapse to single high-reward answers. Due to the discrete nature of lan-\nguage generation, this objective is not differentiable and is typically optimized with reinforce-\nment learning. The standard approach [ 51, 40, 1, 28] has been to construct the reward function\nr(x, y) =rϕ(x, y) − β(log πθ(y | x) − log πref(y | x)), and maximize using PPO [39].\n4 Direct Preference Optimization\nMotivated by the challenges of applying reinforcement learning algorithms on large-scale problems\nsuch as fine-tuning language models, our goal is to derive a simple approach for policy optimization\nusing preferences directly. Unlike prior RLHF methods, which learn a reward and then optimize it\nvia RL, our approach leverages a particular choice of reward model parameterization that enables\nextraction of its optimal policy in closed form, without an RL training loop. As we will describe\nnext in detail, our key insight is to leverage an analytical mapping from reward functions to optimal\npolicies, which enables us to transform a loss function over reward functions into a loss function\nover policies. This change-of-variables approach avoids fitting an explicit, standalone reward model,\nwhile still optimizing under existing models of human preferences, such as the Bradley-Terry model.\nIn essence, the policy network represents both the language model and the (implicit) reward.\nDeriving the DPO objective. We start with the same RL objective as prior work, Eq. 3, under a\ngeneral reward function r. Following prior work [31, 30, 19, 15], it is straightforward to show that\nthe optimal solution to the KL-constrained reward maximization objective in Eq. 3 takes the form:\nπr(y | x) = 1Z(x) πref(y | x) exp\u0012 1β r(x, y)\u0013 , (4)\nwhere Z(x) = Py πref(y | x) exp\u0010 1β r(x, y)\u0011 is the partition function. See Appendix A.1 for a\ncomplete derivation. Even if we use the MLE estimate rϕ of the ground-truth reward function r∗, it is\nstill expensive to estimate the partition function Z(x) [19, 15], which makes this representation hard\nto utilize in practice. However, we can rearrange Eq. 4 to express the reward function in terms of its\ncorresponding optimal policy πr, the reference policy πref, and the unknown partition function Z(·).\nSpecifically, we first take the logarithm of both sides of Eq. 4 and then with some algebra we obtain:\nr(x, y) =β log πr(y | x)πref(y | x) + β log Z(x). (5)\nWe can apply this reparameterization to the ground-truth reward r∗ and corresponding optimal model\nπ∗. Fortunately, the Bradley-Terry model depends only on the difference of rewards between two\ncompletions, i.e., p∗(y1 ≻ y2 | x) =σ(r∗(x, y1) − r∗(x, y2)). Substituting the reparameterization\nin Eq. 5 for r∗(x, y) into the preference model Eq. 1, the partition function cancels, and we can\nexpress the human preference probability in terms of only the optimal policy π∗ and reference policy\nπref. Thus, the optimal RLHF policy π∗ under the Bradley-Terry model satisfies the preference model:\np∗(y1 ≻ y2 | x) = 1\n1 + exp\n\u0010\nβ log π∗(y2|x)πref(y2|x) − β log π∗(y1|x)πref(y1|x) \u0011 (6)\nThe derivation is in Appendix A.2. While Eq. 6 uses the Bradley-Terry model, we can similarly\nderive expressions under the more general Plackett-Luce models [32, 23], shown in Appendix A.3.\nNow that we have the probability of human preference data in terms of the optimal policy rather than\nthe reward model, we can formulate a maximum likelihood objective for a parametrized policyπθ.\nAnalogous to the reward modeling approach (i.e. Eq. 2), our policy objective becomes:\nLDPO(πθ; πref) =−E(x,yw,yl)∼D\n\u0014\nlog σ\n\u0012\nβ log πθ(yw | x)πref(yw | x) − β log πθ(yl | x)πref(yl | x) \u0013\u0015 . (7)\nThis way, we fit an implicit reward using an alternative parameterization, whose optimal policy is\nsimply πθ. Moreover, since our procedure is equivalent to fitting a reparametrized Bradley-Terry\n4",
279
+ "model, it enjoys certain theoretical properties, such as consistencies under suitable assumption of the\npreference data distribution [4]. In Section 5, we further discuss theoretical properties of DPO in\nrelation to other works.\nWhat does the DPO update do? For a mechanistic understanding of DPO, it is useful to analyze the\ngradient of the loss function LDPO. The gradient with respect to the parameters θ can be written as:\n∇θLDPO(πθ; πref) =\n− βE(x,yw,yl)∼D\n\u0014\nσ(ˆrθ(x, yl) − ˆrθ(x, yw))| {z }\nhigher weight when reward estimate is wrong\n\u0014\n∇θ log π(yw | x)| {z }\nincrease likelihood of yw\n− ∇θ log π(yl | x)| {z }\ndecrease likelihood of yl\n\u0015\u0015\n,\nwhere ˆrθ(x, y) =β log πθ(y|x)πref(y|x) is the reward implicitly defined by the language model πθ and refer-\nence model πref (more in Section 5). Intuitively, the gradient of the loss function LDPO increases the\nlikelihood of the preferred completions yw and decreases the likelihood of dispreferred completions\nyl. Importantly, the examples are weighed by how much higher the implicit reward model ˆrθ rates\nthe dispreferred completions, scaled by β, i.e, how incorrectly the implicit reward model orders\nthe completions, accounting for the strength of the KL constraint. Our experiments suggest the\nimportance of this weighting, as a naïve version of this method without the weighting coefficient can\ncause the language model to degenerate (Appendix Table 3).\nDPO outline. The general DPO pipeline is as follows: 1) Sample completions y1, y2 ∼ πref(· |x)\nfor every prompt x, label with human preferences to construct the offline dataset of preferences\nD = {x(i), y(i)w , yl)(i)}Ni=1 and 2) optimize the language model πθ to minimize LDPO for the given\nπref and D and desired β. In practice, one would like to reuse preference datasets publicly available,\nrather than generating samples and gathering human preferences. Since the preference datasets\nare sampled using πSFT, we initialize πref = πSFT whenever available. However, when πSFT is\nnot available, we initialize πref by maximizing likelihood of preferred completions (x, yw), that\nis, πref = arg maxπ Ex,yw∼D [log π(yw | x)]. This procedure helps mitigate the distribution shift\nbetween the true reference distribution which is unavailable, and πref used by DPO. Further details\nrelated to the implementation and hyperparameters can be found in Appendix B.\n5 Theoretical Analysis of DPO\nIn this section, we give further interpretation of the DPO method, provide theoretical backing, and\nrelate advantages of DPO to issues with actor critic algorithms used for RLHF (such as PPO [39]).\n5.1 Your Language Model Is Secretly a Reward Model\nDPO is able to bypass both fitting an explicit reward and performing RL to learn the policy using\na single maximum likelihood objective. Note the optimization objective Eq. 5 is equivalent to a\nBradley-Terry model with a reward parameterization r∗(x, y) =β log π∗θ (y|x)πref(y|x) and we optimize our\nparametric model πθ, equivalently to the reward model optimization in Eq. 2 under the change of\nvariables. In this section we will build the theory behind this reparameterization, show that it does\nnot constrain the class of learned reward models, and allows for the exact recovery of the optimal\npolicy. We begin with by defining an equivalence relation between reward functions.\nDefinition 1. We say that two reward functions r(x, y) and r′(x, y) are equivalent iff\nr(x, y) − r′(x, y) =f (x) for some function f.\nIt is easy to see that this is indeed an equivalence relation, which partitions the set of reward functions\ninto classes. We can state the following two lemmas:\nLemma 1. Under the Plackett-Luce, and in particular the Bradley-Terry, preference framework, two\nreward functions from the same class induce the same preference distribution.\nLemma 2. Two reward functions from the same equivalence class induce the same optimal policy\nunder the constrained RL problem.\nThe proofs are straightforward and we defer them to Appendix A.5. The first lemma is a well-known\nunder-specification issue with the Plackett-Luce family of models [32]. Due to this under-specification,\n5",
280
+ "DPO’s reward/KL tradeoff strictly dominates PPO. Second, DPO achieves a better frontier than PPO,\neven when PPO can access ground truth rewards(PPO-GT).\n6.2 Can DPO scale to real preference datasets?\nNext, we evaluate fine-tuning performance of DPO on summarization and single-turn dialogue. For\nsummarization, automatic evaluation metrics such as ROUGE can be poorly correlated with human\npreferences [40], and prior work has found that fine-tuning LMs using PPO on human preferences\nto provide more effective summaries. We evaluate different methods by sampling completions on\nthe test split of TL;DR summarization dataset, and computing the average win rate against reference\ncompletions in the test set. The completions for all methods are sampled at temperatures varying\nfrom 0.0 to 1.0, and the win rates are shown in Figure 2 (right). DPO, PPO and Preferred-FT all\nfine-tune the same GPT-J SFT model4. We find that DPO has a win rate of approximately 61% at a\ntemperature of 0.0, exceeding the performance of PPO at 57% at its optimal sampling temperature\nof 0.0. DPO also achieves a higher maximum win rate compared to the best of N baseline. We\nnote that we did not meaningfully tune DPO’s β hyperparameter, so these results may underestimate\nDPO’s potential. Moreover, we find DPO to be much more robust to the sampling temperature than\nPPO, the performance of which can degrade to that of the base GPT-J model at high temperatures.\nPreferred-FT does not improve significantly over the SFT model. We also compare DPO and PPO\nhead-to-head in human evaluations in Section 6.4, where DPO samples at temperature 0.25 were\npreferred 58% times over PPO samples at temperature 0.\nOn single-turn dialogue, we evaluate the different methods on the subset of the test split of the\nAnthropic HH dataset [1] with one step of human-assistant interaction. GPT-4 evaluations use the\npreferred completions on the test as the reference to compute the win rate for different methods. As\nthere is no standard SFT model for this task, we start with a pre-trained Pythia-2.8B, use Preferred-FT\nto train a reference model on the chosen completions such that completions are within distribution\nof the model, and then train using DPO. We also compare against the best of 128 Preferred-FT\ncompletions (we found the Best of N baseline plateaus at 128 completions for this task; see Appendix\nFigure 4) and a 2-shot prompted version of the Pythia-2.8B base model, finding DPO performs as\nwell or better for the best-performing temperatures for each method. We also evaluate an RLHF\nmodel trained with PPO on the Anthropic HH dataset 5 from a well-known source 6, but are unable\nto find a prompt or sampling temperature that gives performance better than the base Pythia-2.8B\nmodel. Based on our results from TL;DR and the fact that both methods optimize the same reward\nfunction, we consider Best of 128 a rough proxy for PPO-level performance. Overall, DPO is the\nonly computationally efficient method that improves over the preferred completions in the Anthropic\nHH dataset, and provides similar or better performance to the computationally demanding Best of\n128 baseline. Finally, Figure 3 shows that DPO converges to its best performance relatively quickly.\n6.3 Generalization to a new input distribution\nWin rate vs. ground truth\nAlg. Temp 0 Temp 0.25\nDPO 0.36 0.31\nPPO 0.26 0.23\nTable 1: GPT-4 win rates vs. ground\ntruth summaries for out-of-distribution\nCNN/DailyMail input articles.\nTo further compare the performance of PPO and DPO un-\nder distribution shifts, we evaluate the PPO and DPO poli-\ncies from our Reddit TL;DR summarization experiment on\na different distribution, news articles in the test split of the\nCNN/DailyMail dataset [26], using the best sampling temper-\natures from TL;DR (0 and 0.25). The results are presented\nin Table 1. We computed the GPT-4 win rate against the\nground-truth summaries in the datasets, using the same GPT-\n4 (C) prompt we used for Reddit TL;DR, but replacing the words “forum post” with “news article”.\nFor this new distribution, DPO continues to outperform the PPO policy by a significant margin. This\nexperiment provides initial evidence that DPO policies can generalize similarly well to PPO policies,\neven though DPO does not use the additional unlabeled Reddit TL;DR prompts that PPO uses.\n4https://huggingface.co/CarperAI/openai_summarize_tldr_sft\n5https://huggingface.co/reciprocate/ppo_hh_pythia-6B\n6https://github.com/CarperAI/trlx/tree/main/examples/hh\n9"
281
+ ],
282
+ "seconds": 0.0961459999998624,
283
+ "error": null,
284
+ "device": "cpu",
285
+ "warmup_seconds": 0.08742583300045226,
286
+ "extra": {
287
+ "version": "6.21.4"
288
+ }
289
+ },
290
+ "naca-report-1135-scan": {
291
+ "pages": [
292
+ ""
293
+ ],
294
+ "seconds": 0.0006254580002860166,
295
+ "error": null,
296
+ "device": "cpu",
297
+ "warmup_seconds": 0.08742583300045226,
298
+ "extra": {
299
+ "version": "6.21.4"
300
+ }
301
+ }
302
+ }
303
+ }
304
+ }