@article {229,
	title = {Supporting Experimental Computer Science},
	number = {ANL MCS Technical Memo 326},
	year = {2012},
	abstract = {The ability to conduct consistent, controlled,  and  repeatable  large-scale experiments in all areas of computer science related to parallel, large-scale,  or distributed computing and networking is critical to the future and development of computer science. Yet conducting such experiments is still too often a challenge for researchers, students, and practitioners because of the unavailability of dedicated resources, inability to create controlled experimental conditions, and variability in software. Availability, repeatability, and open sharing of electronic products are all still difficult to achieve.

To discuss those challenges and share experiences in their solution, the Workshop on Experimental Support for Computer Science [1] brought together scientists involved in building and operating infrastructures dedicated to supporting  computer science experiments to discuss challenges and solutions in this space.  The workshop was held in November 2011 and was collocated  with  the  SC11  conference  in  Seattle,  Wash. Our objectives were to share experiences and knowledge related to supporting large-scale experiments conducted on experimental infrastructures, understand user requirements, and discuss methodologies and opportunities created by emerging technologies.

This report ties together the workshop presentations and discussion  and  the  consensus  that emerged on the state of the field and directions for moving forward. In Section 2 we set the stage by describing the experimental culture and existing methodology in computer science. In Section 3, we describe the properties of  the  experimental testbeds whose representatives were participating in the workshop{\textemdash}Grid{\textquoteright}5000 in France and Future Grid and Open Cirrus in the United States{\textemdash}as well as the projects that these testbeds support. The layers of experimental infrastructure are described in Section 4, followed in Sections 5 and 6 by profiles of tools and approaches taken by the respective testbeds to provide  basic  experiment  management  services and experiment orchestration. In Section 7 we summarize the workshop findings.},
	url = {http://www.nimbusproject.org/downloads/Supporting_Experimental_Computer_Science_final_draft.pdf},
	author = {Kate Keahey and Fr{\'e}d{\'e}ric Desprez}
}
