DOCSLIB.ORG

The Replication Crisis in Empirical Science: Implications for Human Subject Research in Mixed Reality

Total Page：16

File Type：pdf, Size：1020Kb

Download full-text PDF Read full-text
The Replication Crisis in Empirical Science: Implications for Human Subject Research in Mixed Reality !"#$%&'( !) !"#$%#&'()*+(,-$./(0(0$(-$12&(/()*'$3)(#-)#4$ 52&'()*+(,-0$6,/$782*-$389:#)+$%#0#*/)"$ (-$;(<#=$%#*'(+> !"#$%&'(%#)&'*#++,#-./'001%#)'2'314 5(126* !"##"##"$$"%&'(')%*+",)-#"'. !/+0(.1%2%3/,)45)-%6767 !"#$%&' ! !"#$%#&'()*+(,-$./(0(0 ! %#&/,12)(3('(+4$*-1$5-6#/#-+(*'$7+*+(0+()0 ! !"#$%&'()(*+'(%),- ! .$/'01*233'4%*5)6'1*!789:;' ! %#&/,12)(3('(+4$8/,9#)+:$804)",',;4 ! <"*+$=,#0$(+$>#*-? ! <"*+$7",2'1$<#$=,? ! !"#$%#&'()*+(,-$./(0(0$(-$@+"#/$A(#'10 ()'*+',$%-.#%/&*01%2%2 ! !"#$%#&'()*+(,-$./(0(0 ! %#&/,12)(3('(+4$*-1$5-6#/#-+(*'$7+*+(0+()0 ! !"#$%&'()(*+'(%),- ! .$/'01*233'4%*5)6'1*!789:;' ! %#&/,12)(3('(+4$8/,9#)+:$804)",',;4 ! <"*+$=,#0$(+$>#*-? ! <"*+$7",2'1$<#$=,? ! !"#$%#&'()*+(,-$./(0(0$(-$@+"#/$A(#'10 ()'*+',$%-.#%/&*01%2%2*3+',1/4"-%5%$%#6*01%2%27 !"#$%&'()%*+,-. ()'*81/5$'9 ! A*('2/#$+,$/#&'()*+#$B*-4$&23'(0"#1$6(-1(-;0C$#D#-$+#E+3,,F$6(-1(-;0 ! %#0#*/)"$3(*0#0 ! .;<:)49%)$,*<)9(=*$,:"*()-,)3)49,%*>! ?*@A@BC*0'(;:%(*#;<:)(&'D ! 5':'4%)$,*<)9(=*$,:"*()-,)3)49,%*0'(;:%(*(':'4%'D*3$0*9,9:"()( ! E'#$0%),-*<)9(=*$,:"*()-,)3)49,%*0'(;:%(*0'#$0%'D*),*#9#'0 ! %#&'()*+(,-$0+21(#0$/*/#'4$62-1#1C$/*/#'4$&23'(0"#1 ! F)%%:'*),4',%)8'*%$*D$*%&'G ! +&'0'3$0'1*G$(%*4$,D;4%'D*(%;D)'(*90'*'H#:$09%$0"*),*,9%;0' :;%4'&-' ! .*-)#/$G(,',;4 ! I@JJ*K,9:"()(=*LBM*$3*49,4'0*D0;-(*39):*),*4:),)49:*%0)9:( ! F'D*%$*0'#:)49%)$,*(%;D)'(*$,*D0;-*'33'4%)8','((*>I@JJNI@JIC ! 5-$,+"#/$6(#'10C$*11(+(,-*'$/#&'()*+(,-$0+21(#0$6,'',H#1 !"#$%#& '()"*+,-.)/ 0123)&4()"*+,-.)/ 5-6)& 78' 89:;< =2>)$ 88' ;:?@ 0-.+#$-*4A$%.+.1.)4B#&40)1&#*#>+,-*4C+%#&/)&%4-$/4!.&#D) E' 8:87 =F!4GH)&-"64C)I)*#"2)$.4A$%.+.1.) J' J:9< ()"&#/1,+3+*+.64K&#L),.M4K%6,H#*#>6 @;' @?:N< !"#$%&'()%*+,-. :;%4'&-' ! %#&'()*+(,-$0+21(#0$),-12)+#1$(-$3(,B#1()(-#C$&04)",',;4 ! 72/D#4$1*+*C$3*0#1$,-$I2#0+(,-: ! O!98'*"$;*39):'D*%$*0'#0$D;4'*($G'<$D"*':('P(*'H#'0)G',%QR O+)*/ '4P)% QH)2+%.&6 E<' 5+#*#>6 <<' KH6%+,%4:4R$>+$))&+$> ;N' S)/+,+$) ;<' R-&.H4:4R$I+&#$2)$. ;9' T.H)& ;7' !"#$%&'()%*+,-. ()'*<9,/1#.&-'*/=*+',$%-.#%/& !"#$%&'()%*+,-. +',1/4"-%5%$%#6*.&4*<&='1'&#%.$*>#.#%2#%-2 ! !"#$%#&'()*+(,-$./(0(0 ! %#&/,12)(3('(+4$*-1$5-6#/#-+(*'$7+*+(0+()0 ! !"#$%&'()(*+'(%),- ! .$/'01*233'4%*5)6'1*!789:;' ! %#&/,12)(3('(+4$8/,9#)+:$804)",',;4 ! <"*+$=,#0$(+$>#*-? ! <"*+$7",2'1$<#$=,? ! !"#$%#&'()*+(,-$./(0(0$(-$@+"#/$A(#'10 !"#$%&'()(*+'(%),- ! !"#$%&'%("%&)*+,%-"-.$#(&")%/0#,#/(+,&'(&/'% *,"1%'#1-$+%/0#,#/(+,&'(&/' 405+&%4 +#+/&012#3 !"#$%&&'())(*'+',-. !"#$%&'(%$"(%)*++,-.(%&.$('/#$,0(+1% ! !"#$#%&"$#'$()*%+)#"$,"$!!-$.#"/"'$#%&"0$!"-$&'('$#%&" ! !"#$%&'()$"#"(%#"()*+(&+&,-%).+/'0(!! 1 !" 2(" !! !" 617%8'9#/&:';%' !! !" 617%8'9#/&:' 9&#4%'1#<%17%=> ;%'?0='0+0=1> ! !"#$%&'()$"#"(.'(+/"(&+&,-%).+/0(!! 1 !" 2(3 !!4!" !"#$%&'()(*+'(%),-*./$0'12/' 56 7"8"-+&()"')%9-"($:&+)$"'.'(##0(!! 1 !" 2("$ ! 6@A<A*'4/;B%914'?041%='/3:%='41%=%#'C2%$23<> ;6 7"8"-+&(/,--($:&+)$"'.'(#$0(!! 1 !" 2(3 ! D#<290&'#++#421%'#?'1%410;&%'78+#17%424 <6 =+/')#,>)('%?&-./@(A.')#.9,).+/(%'',?./@(#$%.'()#,"6 B6 C,/(%/("D&"#.?"/)(%/A(>+--">)('%?&-"'E(:."-A./@('%?&-./@( ')%).').>(%6 ! 6@A<A*'5%04/=%'4/;B%914'/3:%='41%=%#'03:'5#3#'9#3:212#34> F6 C"G"##./@()+('%?&-./@(A.')#.9,).+/4(>%->,-%)"(>+/A.).+/%-( &#+9%9.-.):(+G('""./@(% @.8"/(#$0(&H(% I(#$%J6 ! E?'+=#;0;2&218'24'&#$'6! F')A)G>*'$%'0=%'/3&2H%&8'1#'4%%'" $7%3' #! 24'1=/%A''I%'=%B%91'#!*'03:'%5;=09%'#"A ! E?'+=#;0;2&218'24'3#1'&#$'6! J')A)G>*'$%'0=%'&2H%&8'1#'4%%'" $7%3' #! 24'1=/%A''I%':#'3#1'=%B%91'#!A 23#1-$+%45%62%7#8&9#(&")%:&(0%;(+,+"%6&+:&)9 56 K:&+)$"'.'(##0(!! 1 !" 2(" ! K/;B%914'?041%='/3:%='41%=%#'C2%$23<A ;6 L,--($:&+)$"'.'(#$0(!! 1 !" 2(3 ! K/;B%914'405%'4+%%:'$7%17%='41%=%#'#='5#3#'C2%$23<A <6 =+/')#,>)"A('%?&-./@(A.')#.9,).+/(%'',?./@(#$ .'()#,"6 B6 C%/(%/("D&"#.?"/)(%/A(>+--">)"A('%?&-"'0 ! L('+0=1292+0314*'9#&&%91%:'M(-'405+&%4 ! "! N'LOAPLM'4%9Q'"" N'LPAPPR'4%9Q'"! S "" N'MAR-L'4%9 F6 =%->,-%)"A(>+/A.).+/%-(&#+9%9.-.):(+G('""./@(56MN<('">(@.8"/( #$0(&H(56MN<('">(I(#$%J(2(36BBF6 ! ! N')APPG'3#1'&#$*'$%'0=%'&2H%&8'1#'4%%'MAR-L'4%9'$7%3'#! 24' 1=/%A''I%':#'3#1'=%B%91'#!A'' ! T724'%U+%=25%31':2:'3#1'1%&&'/4'1701'4/;B%914'$%=%'?041%='/3:%=' 41%=%#'C2%$23<A !K$03'%1'0&A'())L. 23#1-$+%<5%2**+/(%"*%=)(+)'&(>%")% ?@%A//$.B+B%C#>+,%D+,/+-(&") 56 K:&+)$"'.'(##0(!& 1 !' 2(" ! T%41%:'9#341031'03:':%9=%0423<'231%34218A''K/;B%914'?041%=' /3:%=':%9=%0423<'231%34218A ;6 L,--($:&+)$"'.'(#$0(!& 1 !' 2(3 ! K/;B%914'405%'4+%%:'$7%17%='9#341031'#=':%9=%0423<'231%34218A <6 =+/')#,>)"A('%?&-./@(A.')#.9,).+/(%'',?./@(#$ .'()#,"6 B6 C%/(%/("D&"#.?"/)(%/A(>+--">)"A('%?&-"'0 ! -'+0=1292+0314*'9#&&%91%:'M,(-'405+&%4 ! "# N'(GR(AP'54%9Q'"$ N'(LLRAR'54%9Q'"# S "$ N''(G(AG'54%9 F6 =%->,-%)"A(>+/A.).+/%-(&#+9%9.-.):(+G('""./@(;F;6F(?'"> @.8"/(#$0(&H(;F;6F(?'"> I(#$%J(2(3633N6 ! ! N')A))-'24'&#$'6! F')A)M>Q'$%'0=%'/3&2H%&8'1#'4%%'(G(AG'54%9 $7%3'#! 24'1=/%A''I%'=%B%91'#!*'03:'%5;=09%'#"A ! T724'%U+%=25%31'4/<<%414'1701'4/;B%914'0=%'?041%='/3:%=' :%9=%0423<'231%34218A !D2C23<'K$03'%1'0&A'())L. 3$4'*5$,()1'/6%)$,(7 ! 12"$3'(4%#%'()5$6/',),%5%#7$"8$# 9$$#$: ! O,>$(+G(')%).').>'(./8+-8"'($+*()+(>%->,-%)"()$.'( &#+9%9.-.):E('+,#>"(+G(?+')(+G(')%).').>P'(>+?&-"D.): ! Q+@.>(+G($:&+)$"'.'()"')./@()$"('%?"(#"@%#A-"''(+G($+*( &H(% I(#$%J(.'(>%->,-%)"A ! RG(:+,(>%/(>%->,-%)"(&H(% I(#$%J4(:+,(>%/()"')(%($:&+)$"'.' ! 12"$(;55$276'#2".%.$$# ! #$ ,',%--:(./(G+#?('H!#4(!(4(SJ(2(3 ! T.8"'($:&+)$"'.'()"')./@(%(A+,9-"U/"@%).8"(-+@.>0 %'',?"(#$ %'()$"(+&&+'.)"(+G(##4()$"/(#"V">)(#$ ! !$.-+'+&$:(.'()$%)(>%/(/"8"#(&#+8"(' 2(34(9">%,'"(3(.'( &+./)(8%-,"(./(A+?%./(+G(#"%-(/,?9"#' ! ## ,',%--:(./(G+#?('H!#4(!(4(SJ(W(3E(*"(A+/P)(X/+*(*$%)( 8%-,"(.)(*.--()%X"4(9,)(?%./(./)"#"')(.'()$%)(.)(.'(/+)(3 8&',*8'*9':'0%*!! ! <)53;5)#"$%&'&"8$# 9$$#$:=$>2"($4'$>"$/"?"3#$$#@ ! R/('>."/>"(@"/"#%--:4(($2 363F ! Y,)4(V,')(%('+>.%-(>+/8"/).+/ ! A2)#$3)($>"$.)7$>2"($>"$/"?"3#$$# )#$%&B CDCCE@ ! ZRG(#$ .'()#,"4()$"#"(.'(+/-:(%/(3633N(&#+9%9.-.):(+G(@"))./@( +,#(#"',-)'4(%/A()$.'(.'(,/-.X"-:6[ " V#==%91W ! Z\$"#"(.'(+/-:(%(3633N(&#+9%9.-.):()$%)(+,#(#"',-)(.'(./( "##+#6[ " I=#3<*'1724'4101%5%31'=%?%=4'1#'!6'#!#>*';/1'1701X4'3#1'$701'$%' 90&9/&01%:A ! Z\$"#"(.'(+/-:(%(3633N(&#+9%9.-.):()$%)(#$ >+,-A($%8"(9""/( )#,"(./()$.'("D&"#.?"/)6[ " I=#3<*'1724'4101%5%31'=%?%=4'1#'!6'#!#Y'" >*';/1'1701X4'3#1'$701' $%'90&9/&01%:A !V#7%3'MRRP. 8&',*8'*;$,<%*9':'0%*!! ! A2)#$3)($>"$.)7$>2"($>"$4'(F#$/"?"3#$$# )#$ %&B CDGGH@ ! Z]"($%8"(&#+8"A()$%)(#$ .'()#,"6[ ! Z^,#("D&"#.?"/)(./A.>%)"'()$%)(#$ .'()#,"6[ " I=#3<*'78+#17%424'1%4123<'9033#1'+=#C%'#!Z'$6%"*'%$*'[>'N')A ! I#)#%.#%3%)(.$4'$('#$)+/""$'($>2)#$J)%5%(+$#'$/"?"3#$ $# &")(.D ! =+/'"#8%).8"(8."*&+./)(H_.'$"#J0( " I%'5/41'4/4+%3:'B/:<5%31*'03:'9033#1'408'0381723<'0;#/1' 17%'1=/17'#?'#!A ! `-)"#/%).8"(8."*&+./)(HL":?%/ a(!"%#'+/J0( " I%'903'099%+1'#! 2?'$%'70C%'4/??292%31'%U+%=25%310&'+#$%=*' 03:'17%=%?#=%'0'&#$'+=#;0;2&218'#?'18+%'EE'%==#=A !"#$%&&'())(*'+'RR. )'*-#-,.,+$,2%3(#+*/,/4 ! KJ$276'#2".%.$#".#%(+$>).$),.'5;#"- ! RG(#$ .'()#,"4()$"/(% >%//+)(+>>,#S$+*"8"#4(% $%'( +>>,##"AS)$"#"G+#"(#$ .'(G%-'"6 ! "6@60(RG(%(&"#'+/(.'(%(O%#).%/4()$"/()$":(%#"(/+)(%(?"?9"#(+G( =+/@#"''(H)#,"JS)$.'(&"#'+/(.'(%(?"?9"#(+G( =+/@#"''S)$"#"G+#"()$":(%#"(/+)(%(O%#).%/6(H>+##">)(#"',-)J ! "6@60(RG(%(&"#'+/(.'(%/(`?"#.>%/4()$"/()$":(%#"(/+)(%( ?"?9"#(+G(=+/@#"''(HG%-'"JS)$.'(&"#'+/(.'(%(?"?9"#(+G( =+/@#"''S)$"#"G+#"()$":(%#"(/+)(%/(`?"#.>%/6( H./>+##">)(#"',-)4(9,)(>+##">)(-+@.>%-(#"%'+/./@J " ( "&! ( )(&! )" 1 1 1 1 "&! ( 1 L L L )(& 5#:/4 1#&&%34 L 1 1 1 ! )" L L 1 1 !V#7%3'MRRP. )'*-#-,.,+$,2%3(#+*/,/4 ! M'>"*"/=$276'#2".%.$#".#%(+$%.$6/',),%5%.#%3- ! RG(#$ .'()#,"4()$"/(% .'($.@$-:(,/-.X"-:S$+*"8"#4(% $%'( +>>,##"AS)$"#"G+#"(#$ .'($.@$-:(,/-.X"-:6 ! "6@60(RG(%(&"#'+/(.'(%/(`?"#.>%/4()$"/()$":(%#"(&#+9%9-:(/+)( %(?"?9"#(+G(=+/@#"''(H)#,"4(#.@$)bJS)$.'(&"#'+/(.'(%( ?"?9"#(+G(=+/@#"''S)$"#"G+#"()$":(%#"(&#+9%9-:(/+)(%/( `?"#.>%/6( H./>+##">)(#"',-)4(9,)(>+##">)($:&+)$"'.'()"')./@(#"%'+/./@J " ( "&! ( )(&! )" 1 1 1 1 "&! ( 1 L L L )(& 5#:/4 1#&&%34 L 1 1 1 ! )" L L 1 1 !V#7%3'MRRP. +',1/4"-%5%$%#6*.&4*<&='1'&#%.$*>#.#%2#%-2 ! !"#$%#&'()*+(,-$./(0(0 ! %#&/,12)(3('(+4$*-1$5-6#/#-+(*'$7+*+(0+()0 ! !"#$%&'()(*+'(%),- ! .$/'01*233'4%*5)6'1*!789:;' ! %#&/,12)(3('(+4$8/,9#)+:$804)",',;4 ! <"*+$=,#0$(+$>#*-? ! <"*+$7",2'1$<#$=,? ! !"#$%#&'()*+(,-$./(0(0$(-$@+"#/$A(#'10 =,%'/#/'%),- ">*#>*6,1*.$?'/ 7">.'.+/ C"V">)(#$ 7+/P)(#"V">)(#$ %(#"',-)c ):&"(RR("##+# \#,"( # G%-'" ')%)"( $ & 2(5(1 ) 2(&+*"# & 2() +G()$"( ):&"(R("##+# %#@,"(# b # )#," $ *+#-A $ & 2(( & 2(5(1 ( " RG(#$ .'()#,"0 #! #" ! & 24'+=#;0;2&218'$%'50H%'0' 18+%'E'%==#=Z'$%'1723H'$%'70C%'0' +#$%='N =%4/&1*';/1'$%'0=%'$=#3< M'S ' " RG(## .'()#,"0 ! ' 24'+=#;0;2&218'$%'50H%'0' 18+%'EE'%==#=Z'0'=%4/&1'$04'17%=%*' ;/1'$%'5244%:'21 ' & ! \#$%= 24'0'5#=%'9#55#3'1%=5' %! %" 1703'' =,0/'6(),-*.$?'/*@"*=,0/'6(),-*" #! #" ! K55;.#/)#".$% N$6'>"/ +#$%= #/)4"'JJ ! K(3/").%(+$%- ! R/>#"%'"'(&+*"# ' & %! %" ! 7">#"%'"'():&"(RR("##+# ! R/>#"%'"'():&"(R("##+# #! #" ! O"3/").%(+$%- +#$%= ! 7">#"%'"'(&+*"# ! R/>#"%'"'():&"(RR("##+# ! 7">#"%'"'():&"(R("##+# ' & %! %" !"#$%&'(")*+,-%$*./*0%&'1$(")* &*2())%$*344%#5 ! KJ$#2"$"JJ"3#$.%P"$%.$ #! #" 5)/+"- ! !+*"#(./>#"%'"' +#$%= ! \:&"(RR("##+# A">#"%'"' ! ( %/A():&"(R("##+# ')%:( )$"('%?" ' & %! %" ! Q(.;/6/%.%(+57=$5)/+"$ #! #" "JJ"3#.$)/"$").%"/$#'$ 4"#"3#$#2)($.&)55$ +#$%= "JJ"3#. ' & %! %" 5/2'(#+,/4%)*6('%-7% 8*..(2$,/4%9*'(%:#$# #! #" #! #" +#$%= ' & +#$%= %! %" " R/>#"%'./@('%?&-"('.d"(H*J0 ! ]%9=%04%4'C0=2039% ! E39=%04%4'+#$%= ! ]%9=%04%4'18+%'EE'%==#= ! & 03:'18+%'E'%==#= 4108'17%' 405% ' & " \$"#"(%#"()">$/.e,"'()$%)( %! %" @.8"()$"(8%-,"(+G(* #"e,.#"A( " "%=%*'%??%91'42^%'=%50234'17%'405%*' G+#(%(>"#)%./(&+*"#(-"8"-6
Load more

Recommended publications
  • Arxiv:2102.03380V2 [Cs.AI] 29 Jun 2021
    Reproducibility in Evolutionary Computation MANUEL LÓPEZ-IBÁÑEZ, University of Málaga, Spain JUERGEN BRANKE, University of Warwick, UK LUÍS PAQUETE, University of Coimbra, CISUC, Department of Informatics Engineering, Portugal Experimental studies are prevalent in Evolutionary Computation (EC), and concerns about the reproducibility and replicability of such studies have increased in recent times, reflecting similar concerns in other scientific fields. In this article, we discuss, within the context of EC, the different types of reproducibility and suggest a classification that refines the badge system of the Association of Computing Machinery (ACM) adopted by ACM Transactions on Evolutionary Learning and Optimization (https://dlnext.acm.org/journal/telo). We identify cultural and technical obstacles to reproducibility in the EC field. Finally, we provide guidelines and suggest tools that may help to overcome some of these reproducibility obstacles. Additional Key Words and Phrases: Evolutionary Computation, Reproducibility, Empirical study, Benchmarking 1 INTRODUCTION As in many other fields of Computer Science, most of the published research in Evolutionary Computation (EC) relies on experiments to justify their conclusions. The ability of reaching similar conclusions by repeating an experiment performed by other researchers is the only way a research community can reach a consensus on an empirical claim until a mathematical proof is discovered. From an engineering perspective, the assumption that experimental findings hold under similar conditions is essential for making sound decisions and predicting their outcomes when tackling a real-world problem. The “reproducibility crisis” refers to the realisation that many experimental findings described in peer-reviewed scientific publications cannot be reproduced, because e.g. they lack the necessary data, they lack enough details to repeat the experiment or repeating the experiment leads to different conclusions.
    [Show full text]
  • 1 Psychology As a Robust Science Dr Amy Orben Lent Term 2020 When
    Psychology as a Robust Science Dr Amy Orben Lent Term 2020 When: Lent Term 2020; Wednesdays 2-4pm (Weeks 0-6; 15.1.2020 – 26.2.2020), Mondays 2-4pm (Week 7; 2.3.2020) Where: Lecture Theatre, Experimental Psychology Department, Downing Site Summary: Is psychology a robust science? To answer such a question, this course will encourage you to think critically about how psychological research is conducted and how conclusions are drawn. To enable you to truly understand how psychology functions as a science, however, this course will also need to discuss how psychologists are incentivised, how they publish and how their beliefs influence the inferences they make. By engaging with such issues, this course will probe and challenge the basic features and functions of our discipline. We will uncover multiple methodological, statistical and systematic issues that could impair the robustness of scientific claims we encounter every day. We will discuss the controversy around psychology and the replicability of its results, while learning about new initiatives that are currently reinventing the basic foundations of our field. The course will equip you with some of the basic tools necessary to conduct robust psychological research fit for the 21st century. The course will be based on a mix of set readings, class discussions and lectures. Readings will include a diverse range of journal articles, reviews, editorials, blog posts, newspaper articles, commentaries, podcasts, videos, and tweets. No exams or papers will be set; but come along with a critical eye and a willingness to discuss some difficult and controversial issues. Core readings • Chris Chambers (2017).
    [Show full text]
  • Promoting an Open Research Culture
    Promoting an open research culture Brian Nosek University of Virginia -- Center for Open Science http://briannosek.com/ -- http://cos.io/ The McGurk Effect Ba Ba? Da Da? Ga Ga? McGurk & MacDonald, 1976, Nature Adelson, 1995 Adelson, 1995 Norms Counternorms Communality Secrecy Open sharing Closed Norms Counternorms Communality Secrecy Open sharing Closed Universalism Particularlism Evaluate research on own merit Evaluate research by reputation Norms Counternorms Communality Secrecy Open sharing Closed Universalism Particularlism Evaluate research on own merit Evaluate research by reputation Disinterestedness Self-interestedness Motivated by knowledge and discovery Treat science as a competition Norms Counternorms Communality Secrecy Open sharing Closed Universalism Particularlism Evaluate research on own merit Evaluate research by reputation Disinterestedness Self-interestedness Motivated by knowledge and discovery Treat science as a competition Organized skepticism Organized dogmatism Consider all new evidence, even Invest career promoting one’s own against one’s prior work theories, findings Norms Counternorms Communality Secrecy Open sharing Closed Universalism Particularlism Evaluate research on own merit Evaluate research by reputation Disinterestedness Self-interestedness Motivated by knowledge and discovery Treat science as a competition Organized skepticism Organized dogmatism Consider all new evidence, even Invest career promoting one’s own against one’s prior work theories, findings Quality Quantity Anderson, Martinson, & DeVries,
    [Show full text]
  • Hop://Cos.Io/ Brian Nosek University of Virginia Hop://Briannosek.Com
    hp://cos.io/ Brian Nosek University of Virginia h"p://briannosek.com/ General Article Psychological Science XX(X) 1 –8 False-Positive Psychology: Undisclosed © The Author(s) 2011 Reprints and permission: sagepub.com/journalsPermissions.nav Flexibility in Data Collection and Analysis DOI: 10.1177/0956797611417632 Allows Presenting Anything as Significant http://pss.sagepub.com Joseph P. Simmons1, Leif D. Nelson2, and Uri Simonsohn1 1The Wharton School, University of Pennsylvania, and 2Haas School of Business, University of California, Berkeley Abstract In this article, we accomplish two things. First, we show that despite empirical psychologists’ nominal endorsement of a low rate of false-positive findings (≤ .05), flexibility in data collection, analysis, and reporting dramatically increases actual false-positive rates. In many cases, a researcher is more likely to falsely find evidence that an effect exists than to correctly find evidence that it does not. We present computer simulations and a pair of actual experiments that demonstrate how unacceptably easy it is to accumulate (and report) statistically significant evidence for a false hypothesis. Second, we suggest a simple, low-cost, and straightforwardly effective disclosure-based solution to this problem. The solution involves six concrete requirements for authors and four guidelines for reviewers, all of which impose a minimal burden on the publication process. Keywords methodology, motivated reasoning, publication, disclosure Received 3/17/11; Revision accepted 5/23/11 Our job as scientists is to discover truths about the world. We Which control variables should be considered? Should spe- generate hypotheses, collect data, and examine whether or not cific measures be combined or transformed or both? the data are consistent with those hypotheses.
    [Show full text]
  • The Reproducibility Crisis in Research and Open Science Solutions Andrée Rathemacher University of Rhode Island, [email protected] Creative Commons License
    University of Rhode Island DigitalCommons@URI Technical Services Faculty Presentations Technical Services 2017 "Fake Results": The Reproducibility Crisis in Research and Open Science Solutions Andrée Rathemacher University of Rhode Island, [email protected] Creative Commons License This work is licensed under a Creative Commons Attribution 4.0 License. Follow this and additional works at: http://digitalcommons.uri.edu/lib_ts_presentations Part of the Scholarly Communication Commons, and the Scholarly Publishing Commons Recommended Citation Rathemacher, Andrée, ""Fake Results": The Reproducibility Crisis in Research and Open Science Solutions" (2017). Technical Services Faculty Presentations. Paper 48. http://digitalcommons.uri.edu/lib_ts_presentations/48http://digitalcommons.uri.edu/lib_ts_presentations/48 This Speech is brought to you for free and open access by the Technical Services at DigitalCommons@URI. It has been accepted for inclusion in Technical Services Faculty Presentations by an authorized administrator of DigitalCommons@URI. For more information, please contact [email protected]. “Fake Results” The Reproducibility Crisis in Research and Open Science Solutions “It can be proven that most claimed research findings are false.” — John P. A. Ioannidis, 2005 Those are the words of John Ioannidis (yo-NEE-dees) in a highly-cited article from 2005. Now based at Stanford University, Ioannidis is a meta-scientist who conducts “research on research” with the goal of making improvements. Sources: Ionnidis, John P. A. “Why Most
    [Show full text]
  • Why Replications Do Not Fix the Reproducibility Crisis: a Model And
    1 63 2 Why Replications Do Not Fix the Reproducibility 64 3 65 4 Crisis: A Model and Evidence from a Large-Scale 66 5 67 6 Vignette Experiment 68 7 69 8 Adam J. Berinskya,1, James N. Druckmanb,1, and Teppei Yamamotoa,1,2 70 9 71 a b 10 Department of Political Science, Massachusetts Institute of Technology, 77 Massachusetts Avenue, Cambridge, MA 02139; Department of Political Science, Northwestern 72 University, 601 University Place, Evanston, IL 60208 11 73 12 This manuscript was compiled on January 9, 2018 74 13 75 14 There has recently been a dramatic increase in concern about collective “file drawer” (4). When publication decisions depend 76 15 whether “most published research findings are false” (Ioannidis on factors beyond research quality, the emergent scientific 77 16 2005). While the extent to which this is true in different disciplines re- consensus may be skewed. Encouraging replication seems to 78 17 mains debated, less contested is the presence of “publication bias,” be one way to correct a biased record of published research 79 18 which occurs when publication decisions depend on factors beyond resulting from this file drawer problem (5–7). Yet, in the 80 19 research quality, most notably the statistical significance of an ef- current landscape, one must also consider potential publication 81 20 fect. Evidence of this “file drawer problem” and related phenom- biases at the replication stage. While this was not an issue 82 21 ena across fields abounds, suggesting that an emergent scientific for OSC since they relied on over 250 scholars to replicate the 83 22 consensus may represent false positives.
    [Show full text]
  • 2020 Impact Report
    Center for Open Science IMPACT REPORT 2020 Maximizing the impact of science together. COS Mission Our mission is to increase the openness, integrity, and reproducibility of research. But we don’t do this alone. COS partners with stakeholders across the research community to advance the infrastructure, methods, norms, incentives, and policies shaping the future of research to achieve the greatest impact on improving credibility and accelerating discovery. Letter from the Executive Director “Show me” not “trust me”: Science doesn’t ask for Science is trustworthy because it does not trust itself. Transparency is a replacement for trust. Transparency fosters self-correction when there are errors trust, it earns trust with transparency. and increases confidence when there are not. The credibility of science has center stage in 2020. A raging pandemic. Partisan Transparency is critical for maintaining science’s credibility and earning public interests. Economic and health consequences. Misinformation everywhere. An trust. The events of 2020 make clear the urgency and potential consequences of amplified desire for certainty on what will happen and how to address it. losing that credibility and trust. In this climate, all public health and economic research will be politicized. All The Center for Open Science is profoundly grateful for all of the collaborators, findings are understood through a political lens. When the findings are against partners, and supporters who have helped advance its mission to increase partisan interests, the scientists are accused of reporting the outcomes they want openness, integrity, and reproducibility of research. Despite the practical, and avoiding the ones they don’t. When the findings are aligned with partisan economic, and health challenges, 2020 was a remarkable year for open science.
    [Show full text]
  • 1 Maximizing the Reproducibility of Your Research Open
    1 Maximizing the Reproducibility of Your Research Open Science Collaboration1 Open Science Collaboration (in press). Maximizing the reproducibility of your research. In S. O. Lilienfeld & I. D. Waldman (Eds.), Psychological Science Under Scrutiny: Recent Challenges and Proposed Solutions. New York, NY: Wiley. Authors’ Note: Preparation of this chapter was supported by the Center for Open Science and by a Veni Grant (016.145.049) awarded to Hans IJzerman. Correspondence can be addressed to Brian Nosek, [email protected]. 1 Alexander A. Aarts, Nuenen, The Netherlands; Frank A. Bosco, Virginia Commonwealth University; Katherine S. Button, University of Bristol; Joshua Carp, Center for Open Science; Susann Fiedler, Max Planck Institut for Research on Collective Goods; James G. Field, Virginia Commonwealth University; Roger Giner-Sorolla, University of Kent; Hans IJzerman, Tilburg University; Melissa Lewis, Center for Open Science; Marcus Munafò, University of Bristol; Brian A. Nosek, University of Virginia; Jason M. Prenoveau, Loyola University Maryland; Jeffrey R. Spies, Center for Open Science 2 Commentators in this book and elsewhere describe evidence that modal scientific practices in design, analysis, and reporting are interfering with the credibility and veracity of the published literature (Begley & Ellis, 2012; Ioannidis, 2005; Miguel et al., 2014; Simmons, Nelson, & Simonsohn, 2011). The reproducibility of published findings is unknown (Open Science Collaboration, 2012a), but concern that is lower than desirable is widespread -
    [Show full text]
  • The Reproducibility Crisis in Scientific Research
    James Madison University JMU Scholarly Commons Senior Honors Projects, 2010-current Honors College Spring 2019 The reproducibility crisis in scientific esearr ch Sarah Eline Follow this and additional works at: https://commons.lib.jmu.edu/honors201019 Part of the Medicine and Health Sciences Commons, and the Statistics and Probability Commons Recommended Citation Eline, Sarah, "The reproducibility crisis in scientific esearr ch" (2019). Senior Honors Projects, 2010-current. 667. https://commons.lib.jmu.edu/honors201019/667 This Thesis is brought to you for free and open access by the Honors College at JMU Scholarly Commons. It has been accepted for inclusion in Senior Honors Projects, 2010-current by an authorized administrator of JMU Scholarly Commons. For more information, please contact [email protected]. Running Head: REPRODUCIBILITY CRISIS 1 The Reproducibility Crisis in Scientific Research Sarah Eline Senior Honors Thesis Running Head: REPRODUCIBILITY CRISIS 2 While evidence-based medicine has its origins well before the 19th century, it was not until the beginning of the 1990s that it began dominating the field of science. Evidence-based practice is defined as “the conscientious and judicious use of current best evidence in conjunction with clinical expertise and patient values to guide health care decisions” (Titler, 2008, para. 3). In 1992, only two journal articles mentioned the phrase evidence-based medicine; however just five years later, that number rose to over 1000. In a very short period of time, evidence-based medicine had evolved to become synonymous with the practices that encompassed the medical field (Sackett, 1996). With evidence-based medicine came a decline in qualitative research and a shift towards quantitative research.
    [Show full text]
  • Intro to Science Studies I
    PHIL 209A / SOCG 255A / HIGR 238 / COGR 225A Intro to Science Studies I Fall 2017 Instructor: Kerry McKenzie [email protected] Seminars: Tuesday 9.30-12.20pm, HSS 3027. Office Hours: Wednesday 2-4pm, HSS 8088. 1 Overview. This course is a philosophically slanted introduction to Science Studies. Our central question is a conceptual one, whose relevance to Science Studies should speak for itself – namely, what is science, and what distinguishes science from other fields? In grappling with this question we’ll familiarize ourselves with central works in the philosophy of science canon, and make glancing acquaintance with some more contemporary issues in scientific epistemology and metaphysics. But our guiding motif is a normative one: what, if anything, makes science entitled to the privileged status that it enjoys within culture? In more detail. The ‘question of demarcation’ – that of what, if anything, makes what we call ‘science’ science – was a central preoccupation of many of the major 20th century philosophers of science. While interest in this topic waned after the appearance of Larry Laudan’s ‘Demise of the Demarcation Problem’ in 1983, the question of what separates science from ‘pseudoscience’ is now making something of a comeback. In this course, we will review the approaches to demarcation offered by the philosophers Popper, Kuhn, and Lakatos – all of which serve as concise introductions to the dominant themes of their work as a whole – and then examine some more contemporary approaches more centred on pragmatics and the philosophy of language. We will then consider how homeopathy – for most a pseudoscience par excellence – fares with regard to the criteria we’ll have studied.
    [Show full text]
  • Psychology's Replication Crisis and the Grant Culture: Righting the Ship
    PPSXXX10.1177/1745691616687745LilienfeldRighting the Ship 687745research-article2017 Perspectives on Psychological Science 2017, Vol. 12(4) 660 –664 Psychology’s Replication Crisis and the © The Author(s) 2017 Reprints and permissions: Grant Culture: Righting the Ship sagepub.com/journalsPermissions.nav DOI:https://doi.org/10.1177/1745691616687745 10.1177/1745691616687745 www.psychologicalscience.org/PPS Scott O. Lilienfeld Department of Psychology, Emory University Abstract The past several years have been a time for soul searching in psychology, as we have gradually come to grips with the reality that some of our cherished findings are less robust than we had assumed. Nevertheless, the replication crisis highlights the operation of psychological science at its best, as it reflects our growing humility. At the same time, institutional variables, especially the growing emphasis on external funding as an expectation or de facto requirement for faculty tenure and promotion, pose largely unappreciated hazards for psychological science, including (a) incentives for engaging in questionable research practices, (b) a single­minded focus on programmatic research, (c) intellectual hyperspecialization, (d) disincentives for conducting direct replications, (e) stifling of creativity and intellectual risk taking, (f) researchers promising more than they can deliver, and (g) diminished time for thinking deeply. Preregistration should assist with (a), but will do little about (b) through (g). Psychology is beginning to right the ship, but it will need to confront the increasingly deleterious impact of the grant culture on scientific inquiry. Keywords replication, grants, preregistration, confirmation bias “Science is a struggle for truth against method­ psychological phenomena, such as behavioral priming, ological, psychological, and sociological obstacles” the effects of facial expressions on emotions, and the (Fanelli & Ioannidis, 2013, p.
    [Show full text]
  • Using the OSF (Open Science Framework)
    Using the OSF Center for Open Science at BITSS 2014 Johanna Cohoon & Caner Uguz Pull out your laptop and visit www.osf.io Anderson, Martinson, & DeVries, 2007 NORMS COUNTERNORMS Communality Secrecy Open sharing Closed Universalism Particularism Evaluate research on own merit Evaluate research by reputation Disinterestedness Self interestedness Motivated by knowledge and discovery Treat science as a competition Organized skepticism Organized dogmatism Consider all new evidence, even Invest career promoting one’s own against one’s prior work theories, findings Quality Quantity COUNTERNORMS NORMS Incentives Incentives for individual success are focused on getting it published, not getting it right. Change the incentives. Center for Open Science COMMUNITY INFRASTRUCTURE METASCIENCE Community ■ Foster discussion of transparency and reproducibility issues ■ Support community efforts to increase openness in the sciences ■ Promote the establishment of reporting standards Center for Open Science COMMUNITY INFRASTRUCTURE METASCIENCE Infrastructure ■ Tools for scientists that increase transparency and access to resources ■ Connect services and allow users to easily organize and share their work Center for Open Science COMMUNITY INFRASTRUCTURE METASCIENCE Metascience ■ How are scientists currently behaving? ■ Do transparency and replication have a positive impact on research? ■ Reproducibility Project: Psychology and Cancer Biology; Many Labs 1, 2, and 3; Archival Project; CREP Open your laptop! www.osf.io OSF Nudge the incentive system ■ Support pre-registration ■ Create alternative means of measuring impact ■ Enable sharing of materials Facilitate good research ■ Organize files ■ Allow collaboration meanwhile Logs Commenting Tags Statistics Download counts Addons OSF Nudge the incentive system ■ Support pre-registration ■ Create alternative means of measuring impact ■ Enable sharing of materials Facilitate good research ■ Organize files ■ Allow collaboration In the future ■ Check lists ■ Notifications ■ Messaging ■ Dataverse, Evernote, Trello, Plot.ly.
    [Show full text]