%PDF-1.7
%
1 0 obj
<< /Metadata 3 0 R /Names 4 0 R /OpenAction 5 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
endobj
2 0 obj
<< /Author (Guilin Zhang; Chuanyi Sun; Kai Zhao; Xu Chu; Shahryar Sarkani; John M. Fossaceca) /Creator (arXiv GenPDF \(tex2pdf:a6404ea\)) /DOI (https://doi.org/10.48550/arXiv.2606.04145) /License (http://creativecommons.org/licenses/by/4.0/) /PTEX.Fullbanner (This is pdfTeX, Version 3.141592653-2.6-1.40.28 \(TeX Live 2025\) kpathsea version 6.4.1) /Producer (pikepdf 8.15.1) /Title (EvalStop: Using World Feedback to Detect and Correct Reward Overoptimization in Multi-Tenant RLHF Platforms) /Trapped /False /arXivID (https://arxiv.org/abs/2606.04145v3) >>
endobj
3 0 obj
<< /Subtype /XML /Type /Metadata /Length 1793 >>
stream
EvalStop: Using World Feedback to Detect and Correct Reward Overoptimization in Multi-Tenant RLHF PlatformsGuilin ZhangChuanyi SunKai ZhaoXu ChuShahryar SarkaniJohn M. Fossacecahttp://creativecommons.org/licenses/by/4.0/cs.LGcs.AIcs.DC
endstream
endobj
4 0 obj
<< /Dests 7 0 R >>
endobj
5 0 obj
<< /D [ 8 0 R /Fit ] /S /GoTo >>
endobj
6 0 obj
<< /Count 10 /Kids [ 9 0 R 10 0 R ] /Type /Pages >>
endobj
7 0 obj
<< /Kids [ 11 0 R 12 0 R ] /Limits [ (Doc-Start) (table.caption.9) ] >>
endobj
8 0 obj
<< /Annots [ 13 0 R 14 0 R 15 0 R 16 0 R 17 0 R 18 0 R 19 0 R 20 0 R 21 0 R 22 0 R 23 0 R 24 0 R 25 0 R 26 0 R 27 0 R 28 0 R 29 0 R 30 0 R 31 0 R 32 0 R 33 0 R 34 0 R 35 0 R 36 0 R 37 0 R 38 0 R 39 0 R 40 0 R 41 0 R 42 0 R 43 0 R 44 0 R 45 0 R ] /Contents [ 46 0 R 47 0 R 48 0 R 49 0 R ] /MediaBox [ 0 0 612 792 ] /Parent 9 0 R /Resources 50 0 R /Type /Page >>
endobj
9 0 obj
<< /Count 6 /Kids [ 8 0 R 51 0 R 52 0 R 53 0 R 54 0 R 55 0 R ] /Parent 6 0 R /Type /Pages >>
endobj
10 0 obj
<< /Count 4 /Kids [ 56 0 R 57 0 R 58 0 R 59 0 R ] /Parent 6 0 R /Type /Pages >>
endobj
11 0 obj
<< /Kids [ 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R 65 0 R ] /Limits [ (Doc-Start) (figure.caption.8) ] >>
endobj
12 0 obj
<< /Kids [ 66 0 R 67 0 R 68 0 R 69 0 R 70 0 R 71 0 R ] /Limits [ (page.1) (table.caption.9) ] >>
endobj
13 0 obj
<< /A << /D (cite.gao2023scaling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 149.687 530.192 189.909 541.136 ] /Subtype /Link /Type /Annot >>
endobj
14 0 obj
<< /A << /D (cite.gao2023scaling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 193.706 530.192 215.484 541.136 ] /Subtype /Link /Type /Annot >>
endobj
15 0 obj
<< /A << /D (cite.rafailov2023direct) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 333.984 542.394 395.604 553.338 ] /Subtype /Link /Type /Annot >>
endobj
16 0 obj
<< /A << /D (cite.rafailov2023direct) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 399.985 542.394 422.301 553.338 ] /Subtype /Link /Type /Annot >>
endobj
17 0 obj
<< /A << /D (cite.ouyang2022training) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 482.097 542.394 541.141 553.338 ] /Subtype /Link /Type /Annot >>
endobj
18 0 obj
<< /A << /D (cite.ouyang2022training) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 306.444 530.439 328.701 541.383 ] /Subtype /Link /Type /Annot >>
endobj
19 0 obj
<< /A << /D (cite.stiennon2020learning) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 331.254 506.529 388.568 517.473 ] /Subtype /Link /Type /Annot >>
endobj
20 0 obj
<< /A << /D (cite.stiennon2020learning) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 391.048 506.529 412.567 517.473 ] /Subtype /Link /Type /Annot >>
endobj
21 0 obj
<< /A << /D (cite.ouyang2022training) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 415.321 506.529 468.827 517.473 ] /Subtype /Link /Type /Annot >>
endobj
22 0 obj
<< /A << /D (cite.ouyang2022training) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 471.308 506.529 492.827 517.473 ] /Subtype /Link /Type /Annot >>
endobj
23 0 obj
<< /A << /D (cite.gao2023scaling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 355.097 363.067 396.442 374.011 ] /Subtype /Link /Type /Annot >>
endobj
24 0 obj
<< /A << /D (cite.gao2023scaling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 400.438 363.067 422.754 374.011 ] /Subtype /Link /Type /Annot >>
endobj
25 0 obj
<< /A << /D (cite.skalse2022defining) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 398.617 327.201 447.49 338.145 ] /Subtype /Link /Type /Annot >>
endobj
26 0 obj
<< /A << /D (cite.skalse2022defining) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 450.363 327.201 471.882 338.145 ] /Subtype /Link /Type /Annot >>
endobj
27 0 obj
<< /A << /D (cite.pan2022effects) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 475.029 327.201 512.906 338.145 ] /Subtype /Link /Type /Annot >>
endobj
28 0 obj
<< /A << /D (cite.pan2022effects) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 515.779 327.201 537.298 338.145 ] /Subtype /Link /Type /Annot >>
endobj
29 0 obj
<< /A << /D (figure.caption.2) /S /GoTo >> /Border [ 0 0 0 ] /C [ 1 0 0 ] /H /I /Rect [ 334.947 315.246 341.921 326.19 ] /Subtype /Link /Type /Annot >>
endobj
30 0 obj
<< /A << /D (subsection.2.3) /S /GoTo >> /Border [ 0 0 0 ] /C [ 1 0 0 ] /H /I /Rect [ 350.718 315.246 365.164 326.19 ] /Subtype /Link /Type /Annot >>
endobj
31 0 obj
<< /A << /D (cite.gu2019tiresias) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 476.536 273.503 512.768 284.372 ] /Subtype /Link /Type /Annot >>
endobj
32 0 obj
<< /A << /D (cite.gu2019tiresias) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 515.776 273.503 537.833 284.372 ] /Subtype /Link /Type /Annot >>
endobj
33 0 obj
<< /A << /D (cite.qiao2021pollux) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 338.836 261.448 382.751 272.392 ] /Subtype /Link /Type /Annot >>
endobj
34 0 obj
<< /A << /D (cite.qiao2021pollux) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 385.791 261.448 408.088 272.392 ] /Subtype /Link /Type /Annot >>
endobj
35 0 obj
<< /A << /D (cite.narayanan2020gavel) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 444.279 261.448 512.427 272.392 ] /Subtype /Link /Type /Annot >>
endobj
36 0 obj
<< /A << /D (cite.narayanan2020gavel) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 515.467 261.448 537.763 272.392 ] /Subtype /Link /Type /Annot >>
endobj
37 0 obj
<< /A << /D (cite.jayaram2023sia) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 324.913 249.493 381.725 260.436 ] /Subtype /Link /Type /Annot >>
endobj
38 0 obj
<< /A << /D (cite.jayaram2023sia) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 384.71 249.493 406.529 260.436 ] /Subtype /Link /Type /Annot >>
endobj
39 0 obj
<< /A << /D (cite.zhang2017slaq) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 406.745 201.672 456.874 212.616 ] /Subtype /Link /Type /Annot >>
endobj
40 0 obj
<< /A << /D (cite.zhang2017slaq) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 459.92 201.672 482.217 212.616 ] /Subtype /Link /Type /Annot >>
endobj
41 0 obj
<< /A << /D (cite.moskovitz2024confronting) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 327.051 117.986 391.43 128.955 ] /Subtype /Link /Type /Annot >>
endobj
42 0 obj
<< /A << /D (cite.moskovitz2024confronting) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 394.042 117.986 415.562 128.955 ] /Subtype /Link /Type /Annot >>
endobj
43 0 obj
<< /A << /D (cite.gao2023scaling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 418.448 117.986 457.573 128.955 ] /Subtype /Link /Type /Annot >>
endobj
44 0 obj
<< /A << /D (cite.gao2023scaling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 460.186 117.986 481.705 128.955 ] /Subtype /Link /Type /Annot >>
endobj
45 0 obj
<< /A << /S /URI /URI (https://arxiv.org/abs/2606.04145v3) >> /BS << /W 0 >> /NM (fitz-L0) /Rect [ 12 223.23999 32 568.76 ] /Subtype /Link >>
endobj
46 0 obj
<< /Length 10 /Filter /FlateDecode >>
stream
x+ |
endstream
endobj
47 0 obj
<< /Filter /FlateDecode /Length 5189 >>
stream
xڵ;]ܶ:NL H:SdK#Ųv]IEC_7!gUD FG$+[nۗO*\-?}"*֫DfBU?j#?BefuۯtjY.`\=慌ҕ"FnVRBdRIڮX0ȫ˾9|DsWַ덊$답kyz=?ZFF"Em-ź)͡/oy_65|=T}Zg*u^o_Typ6m| &H䫱(&Y -/`{R ,01N"aifDi62
n3n곛$"jCD2[y99p3fk`"
疍b!,|w}2B}^HR>
)-~XKH1Ꭶ
S)TsY;DI
M~vmN"S$1dBIb%8\&
yȰ^F~Irг N6T˺hmb_o4"+:CѴ&vЗζûd?
ϐ[ټ-
EѾ
eG],ES8th׀yd'(a;jSC^ā@y:P"_A
U2w;
lO啘KZ5 rNVْvî^PG+;SzTRD"`[ ڎFnf(Xoᇤ*/]Fμ%pk,e_"sķZi&lht}[:lKn$M=v}uҶZTٱD$2hבk;C$'P8l_-IJ*i,bxz[nnI ,-=ڮPٶnZ$#;_#'-GrģT 'U^Zqw6Μnayk'6`J?<]N`˹9Wp6AI+KH7#s