SPEC CPU2006 benchmarks

This is a work in-progress. Everyone should feel free to extend this page with their experiences to help new users get started.

Input sets and Binaries

We can't provide the binaries or input files because of licensing restrictions, but It's not hard to build the binaries by yourself. In this short article, we will share our experiences about what we have done so far.

Build the cross-compiler for alpha machine

Download the crosstool-0.43.tar.gz from http://kegel.com/crosstool and modify these three lines in the demo-alpha.sh :

RESULT_TOP=where_you_want_to_put_the_compiler
GCC_LANGUAGES="c,c++,fortran"
eval `cat alpha.dat gcc-4.1.0-glibc-2.3.6.dat` sh all.sh --notest

Then follow the steps in the crosstool-howto page to build the cross compiler.

Build the SPEC2006 alpha binaries

Install the SPEC2006 from DVD and modify the CC, CXX, and FC in config/alpha.cfg.

For example:
    CC =  /home/mjwu/crosstool/gcc-4.1.0-glibc-2.3.6/alpha/bin/alpha-gcc
    CXX = /home/mjwu/crosstool/gcc-4.1.0-glibc-2.3.6/alpha/bin/alpha-g++
    FC =  /home/mjwu/crosstool/gcc-4.1.0-glibc-2.3.6/alpha/bin/alpha-gfortran

Then follow the instructions in the ./Docs/install-guide-unix.html to build the binaries

For example:
    runspec --config=alpha.cfg --action=buld --tune=base bzip2

Expand M5 system call functions

Four extra system call functions are needed by SPEC2006. The functions can be modified in:

In m5-stable-mt/src/arch/alpha/linux/process.cc:

 /* 130 */ SyscallDesc("ftruncate", ftruncateFunc),
 /* 144 */ SyscallDesc("getrlimit",  ignoreFunc),
 /* 341 */ SyscallDesc("mremap", mremapFunc<AlphaLinux>),
 /* 367 */ SyscallDesc("getcwd", getcwdFunc),

In m5-stable-mt/src/sim/syscall_emul.hh :

///Target getcwd() handler.
SyscallReturn getcwdFunc(SyscallDesc *desc, int num,
                               LiveProcess *p, ThreadContext *tc);

//A simple implementation
template <class OS>
SyscallReturn
mremapFunc(SyscallDesc *desc, int num, LiveProcess *p, ThreadContext *tc)
{
    Addr start = tc->getSyscallArg(0);
    uint64_t length_old = tc->getSyscallArg(1);
    uint64_t length_new = tc->getSyscallArg(2);

    if ((start  % TheISA::VMPageSize) != 0 ||
        (length_new % TheISA::VMPageSize) != 0) {
        warn("mremap failing: arguments not page-aligned: "
             "start 0x%x length 0x%x",
             start, length_new);
        return -EINVAL;
    }

    if (start != 0) {
        warn("mremap: ignoring suggested map address 0x%x, using 0x%x",
             start, p->mmap_end);
    }

    // pick next address from our "mmap region"
    if(length_old < length_new){
       warn("mremap size  0x%x  %d -> %d",start,length_old,length_new);
       start = p->mmap_end;
       p->pTable->allocate(start, length_new-length_old);
       p->mmap_end += (length_new-length_old);
       start = tc->getSyscallArg(0);

    }else{
       warn("mremap size  0x%x  %d -> %d",start,length_old,length_new);
    }

    return start;
}

In m5-stable-mt/src/sim/syscall_emul.cc:

SyscallReturn
getcwdFunc(SyscallDesc *desc, int num, LiveProcess *p, ThreadContext *tc)
{
    char pathname[256];
    int path_len = tc->getSyscallArg(1);
    getcwd(pathname,path_len);
    BufferArg path(tc->getSyscallArg(0), path_len);
    strncpy((char *)path.bufferPtr(), pathname, path_len);

    path.copyOut(tc->getMemPort());

    return 0;
}

Create the SPEC2006 processes for M5 SE mode

A good reference for the correct command line options can be found here: SPEC_CPU2006_Commands.

For your convenience, here is our benchmark python file for the M5 SE mode.

#Mybench.py

#400.perlbench
perlbench = LiveProcess()
perlbench.executable =  binary_dir+'400.perlbench_base.alpha-gcc'
perlbench.cmd = [perlbench.executable] + ['-I./lib', 'attrs.pl']
perlbench.output = 'attrs.out'

#401.bzip2
bzip2 = LiveProcess()
bzip2.executable =  binary_dir+'401.bzip2_base.alpha-gcc'
data=data_dir+'401.bzip2/data/all/input/input.program'
bzip2.cmd = [bzip2.executable] + [data, '1']
bzip2.output = 'input.program.out'

#403.gcc
gcc = LiveProcess()
gcc.executable =  binary_dir+'403.gcc_base.alpha-gcc'
data=data_dir+'403.gcc/data/test/input/cccp.i'
output='/import/home1/mjwu/work_spec2006/403.gcc/m5/cccp.s'
gcc.cmd = [gcc.executable] + [data]+['-o',output]
gcc.output = 'ccc.out'

#410.bwaves
bwaves = LiveProcess()
bwaves.executable =  binary_dir+'410.bwaves_base.alpha-gcc'
bwaves.cmd = [bwaves.executable]

#416.gamess
gamess=LiveProcess()
gamess.executable =  binary_dir+'416.gamess_base.alpha-gcc'
gamess.cmd = [gamess.executable]
gamess.input='exam29.config'
gamess.output='exam29.output'

#429.mcf
mcf = LiveProcess()
mcf.executable =  binary_dir+'429.mcf_base.alpha-gcc'
data=data_dir+'429.mcf/data/test/input/inp.in'
mcf.cmd = [mcf.executable] + [data]
mcf.output = 'inp.out'

#433.milc
milc=LiveProcess()
milc.executable = binary_dir+'433.milc_base.alpha-gcc'
stdin=data_dir+'433.milc/data/test/input/su3imp.in'
milc.cmd = [milc.executable]
milc.input=stdin
milc.output='su3imp.out'

#434.zeusmp
zeusmp=LiveProcess()
zeusmp.executable =  binary_dir+'434.zeusmp_base.alpha-gcc'
zeusmp.cmd = [zeusmp.executable]
zeusmp.output = 'zeusmp.stdout'

#435.gromacs
gromacs = LiveProcess()
gromacs.executable =  binary_dir+'435.gromacs_base.alpha-gcc'
data=data_dir+'435.gromacs/data/test/input/gromacs.tpr'
gromacs.cmd = [gromacs.executable] + ['-silent','-deffnm',data,'-nice','0']

#436.cactusADM
cactusADM = LiveProcess()
cactusADM.executable =  binary_dir+'436.cactusADM_base.alpha-gcc'
data=data_dir+'436.cactusADM/data/test/input/benchADM.par'
cactusADM.cmd = [cactusADM.executable] + [data]
cactusADM.cmd = 'benchADM.out'

#437.leslie3d
leslie3d=LiveProcess()
leslie3d.executable =  binary_dir+'437.leslie3d_base.alpha-gcc'
stdin=data_dir+'437.leslie3d/data/test/input/leslie3d.in'
leslie3d.cmd = [leslie3d.executable]
leslie3d.input=stdin
leslie3d.output='leslie3d.stdout'

#444.namd
namd = LiveProcess()
namd.executable =  binary_dir+'444.namd_base.alpha-gcc'
input=data_dir+'444.namd/data/all/input/namd.input'
namd.cmd = [namd.executable] + ['--input',input,'--iterations','1','--output','namd.out']
namd.output='namd.stdout'

#445.gobmk
gobmk=LiveProcess()
gobmk.executable =  binary_dir+'445.gobmk_base.alpha-gcc'
stdin=data_dir+'445.gobmk/data/test/input/capture.tst'
gobmk.cmd = [gobmk.executable]+['--quiet','--mode','gtp']
gobmk.input=stdin
gobmk.output='capture.out'

#447.dealII
dealII=LiveProcess()
dealII.executable =  binary_dir+'447.dealII_base.alpha-gcc'
dealII.cmd = [gobmk.executable]+['8']
dealII.output='log'


#450.soplex
soplex=LiveProcess()
soplex.executable =  binary_dir+'450.soplex_base.alpha-gcc'
data=data_dir+'450.soplex/data/test/input/test.mps'
soplex.cmd = [soplex.executable]+['-m10000',data]
soplex.output = 'test.out'

#453.povray
povray=LiveProcess()
povray.executable =  binary_dir+'453.povray_base.alpha-gcc'
data=data_dir+'453.povray/data/test/input/SPEC-benchmark-test.ini'
#povray.cmd = [povray.executable]+['SPEC-benchmark-test.ini']
povray.cmd = [povray.executable]+[data]
povray.output = 'SPEC-benchmark-test.stdout'

#454.calculix
calculix=LiveProcess()
calculix.executable =  binary_dir+'454.calculix_base.alpha-gcc'
data='/import/RaidHome/mjwu/work_spec2006/454.calculix/m5/beampic'
calculix.cmd = [calculix.executable]+['-i',data]
calculix.output = 'beampic.log'

#456.hmmer
hmmer=LiveProcess()
hmmer.executable =  binary_dir+'456.hmmer_base.alpha-gcc'
data=data_dir+'456.hmmer/data/test/input/bombesin.hmm'
#data='/import/RaidHome/mjwu/work_spec2006/456.hmmer/m5/bombesin.hmm'
hmmer.cmd = [hmmer.executable]+['--fixed', '0', '--mean', '325', '--num', '5000', '--sd', '200', '--seed', '0', data]
hmmer.output = 'bombesin.out'

#458.sjeng
sjeng=LiveProcess()
sjeng.executable =  binary_dir+'458.sjeng_base.alpha-gcc'
data=data_dir+'458.sjeng/data/test/input/test.txt'
sjeng.cmd = [sjeng.executable]+[data]
sjeng.output = 'test.out'

#459.GemsFDTD
GemsFDTD=LiveProcess()
GemsFDTD.executable =  binary_dir+'459.GemsFDTD_base.alpha-gcc'
GemsFDTD.cmd = [GemsFDTD.executable]
GemsFDTD.output = 'test.log'

#462.libquantum
libquantum=LiveProcess()
libquantum.executable =  binary_dir+'462.libquantum_base.alpha-gcc'
libquantum.cmd = [libquantum.executable],'33','5'
libquantum.output = 'test.out'

#464.h264ref
h264ref=LiveProcess()
h264ref.executable =  binary_dir+'464.h264ref_base.alpha-gcc'
data=data_dir+'464.h264ref/data/test/input/foreman_test_encoder_baseline.cfg'
h264ref.cmd = [h264ref.executable]+['-d',data]
h264ref.output = 'foreman_test_encoder_baseline.out'

#470.lbm
lbm=LiveProcess()
lbm.executable =  binary_dir+'470.lbm_base.alpha-gcc'
data=data_dir+'470.lbm/data/test/input/100_100_130_cf_a.of'
lbm.cmd = [lbm.executable]+['20', 'reference.dat', '0', '1' ,data]
lbm.output = 'lbm.out'

#471.omnetpp
omnetpp=LiveProcess()
omnetpp.executable =  binary_dir+'471.omnetpp_base.alpha-gcc'
data=data_dir+'471.omnetpp/data/test/input/omnetpp.ini'
omnetpp.cmd = [omnetpp.executable]+[data]
omnetpp.output = 'omnetpp.log'

#473.astar
astar=LiveProcess()
astar.executable =  binary_dir+'473.astar_base.alpha-gcc'
astar.cmd = [astar.executable]+['lake.cfg']
astar.output = 'lake.out'

#481.wrf
wrf=LiveProcess()
wrf.executable =  binary_dir+'481.wrf_base.alpha-gcc'
wrf.cmd = [wrf.executable]+['namelist.input']
wrf.output = 'rsl.out.0000'

#482.sphinx
sphinx3=LiveProcess()
sphinx3.executable =  binary_dir+'482.sphinx_livepretend_base.alpha-gcc'
sphinx3.cmd = [sphinx3.executable]+['ctlfile', '.', 'args.an4']
sphinx3.output = 'an4.out'

#483.xalancbmk
xalancbmk=LiveProcess()
xalancbmk.executable =  binary_dir+'483.Xalan_base.alpha-gcc'
xalancbmk.cmd = [xalancbmk.executable]+['-v','test.xml','xalanc.xsl']
xalancbmk.output = 'test.out'

#998.specrand
specrand_i=LiveProcess()
specrand_i.executable = binary_dir+'998.specrand_base.alpha-gcc'
specrand_i.cmd = [specrand_i.executable] + ['324342','24239']
specrand_i.output = 'rand.24239.out'

#999.specrand
specrand_f=LiveProcess()
specrand_f.executable = binary_dir+'999.specrand_base.alpha-gcc'
specrand_f.cmd = [specrand_i.executable] + ['324342','24239']
specrand_f.output = 'rand.24239.out'

M5 python configure file

Here is our system configuration python file for the M5 SE mode.

#cmp.py
# Simple configuration script

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('./configs')
import Simulation
from Caches import *
import Mybench

# Get paths we might need.  It's expected this file is in m5/configs/example.
config_path = os.path.dirname(os.path.abspath(__file__))
print config_path
config_root = os.path.dirname(config_path)+"/configs"
print config_root
m5_root = os.path.dirname(config_root)
print m5_root

parser = optparse.OptionParser()

# Benchmark options

parser.add_option("-b", "--benchmark", default="",
                 help="The benchmark to be loaded.")

parser.add_option("-c", "--chkpt", default="",
                 help="The checkpoint to load.")

execfile(os.path.join(config_root, "configs", "Options.py"))

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if options.benchmark == 'perlbench':
   process = Mybench.perlbench
elif options.benchmark == 'bzip2':
   process = Mybench.bzip2
elif options.benchmark == 'gcc':
   process = Mybench.gcc
elif options.benchmark == 'bwaves':
   process = Mybench.bwaves
elif options.benchmark == 'gamess':
   process = Mybench.gamess
elif options.benchmark == 'mcf':
   process = Mybench.mcf
elif options.benchmark == 'milc':
   process = Mybench.milc
elif options.benchmark == 'zeusmp':
   process = Mybench.zeusmp
elif options.benchmark == 'gromacs':
   process = Mybench.gromacs
elif options.benchmark == 'cactusADM':
   process = Mybench.cactusADM
elif options.benchmark == 'leslie3d':
   process = Mybench.leslie3d
elif options.benchmark == 'namd':
   process = Mybench.namd
elif options.benchmark == 'gobmk':
   process = Mybench.gobmk;
elif options.benchmark == 'dealII':
   process = Mybench.dealII
elif options.benchmark == 'soplex':
   process = Mybench.soplex
elif options.benchmark == 'povray':
   process = Mybench.povray
elif options.benchmark == 'calculix':
   process = Mybench.calculix
elif options.benchmark == 'hmmer':
   process = Mybench.hmmer
elif options.benchmark == 'sjeng':
   process = Mybench.sjeng
elif options.benchmark == 'GemsFDTD':
   process = Mybench.GemsFDTD
elif options.benchmark == 'libquantum':
   process = Mybench.libquantum
elif options.benchmark == 'h264ref':
   process = Mybench.h264ref
elif options.benchmark == 'tonto':
   process = Mybench.tonto
elif options.benchmark == 'lbm':
   process = Mybench.lbm
elif options.benchmark == 'omnetpp':
   process = Mybench.omnetpp
elif options.benchmark == 'astar':
   process = Mybench.astar
elif options.benchmark == 'wrf':
   process = Mybench.wrf
elif options.benchmark == 'sphinx3':
   process = Mybench.sphinx3
elif options.benchmark == 'xalancbmk':
   process = Mybench.xalancbmk
elif options.benchmark == 'specrand_i':
   process = Mybench.specrand_i
elif options.benchmark == 'specrand_f':
   process = Mybench.specrand_f

if options.chkpt != "":
   process.chkpt = options.chkpt

(CPUClass, test_mem_mode, FutureClass) = Simulation.setCPUClass(options)

CPUClass.clock = '1.0GHz'

#np = options.num_cpus 
np = 1

system = System(cpu = [CPUClass(cpu_id=i) for i in xrange(np)],
                physmem = PhysicalMemory(range=AddrRange("4096MB")),
                membus = Bus(), mem_mode = 'timing')

system.physmem.port = system.membus.port

for i in xrange(np):   
    if options.caches:
        system.cpu[i].addPrivateSplitL1Caches(L1Cache(size = '64kB'),
                                              L1Cache(size = '64kB'))
    if options.l2cache:
        system.l2 = L2Cache(size='2MB')
        system.tol2bus = Bus()
        system.l2.cpu_side = system.tol2bus.port
        system.l2.mem_side = system.membus.port
        system.cpu[i].connectMemPorts(system.tol2bus)
    else:
        system.cpu[i].connectMemPorts(system.membus)
    system.cpu[i].workload = process[i]


root = Root(system = system)

Simulation.run(options, root, system, FutureClass)

The SPEC2006 testing dataset results

We use the quard-core Xeon 2.5GHz with 16G memory machine. The operation system is 64bits CentOS 5.2.

benchmark	datatype	programming language	input data	number of instructions	host seconds	comment
400.perlbench	integer	C	attrs.out	-	-	fatal: fault (unalign) detected @ PC 0x12009cedc
401.bzip2	integer	C	input.program	3171671617	1353.56	o.k.
403.gcc	integer	C	cccp.i	-	-	never end, but o.k. for smaller input
410.bwaves	floating	Fortran	test	119365801487	51703.94	o.k.
416.gamess	floating	Fortran	exam29	-	-	abormal exit
429.mcf	integer	C	inp.in	5112705810	3386.09	o.k.

        benchmark      datatype  programming language   input     number of instructions         host seconds     
    
    
    
    
    433.milc              floating point C su3imp.in  38027871822  18402.06  output is a little different with the reference
    434.zeusmp          floating point   Fortran   zmp_inp  62107158516 27746.77 output is a little different with the reference
    435.gromacs        floating point C/Fortran  gromacs.tpr    10861507208    4457.33  output error, maybe caused by mremap
   436.cactusADM    floating point C/Fortran  benchADM.par   fatal: fault (unalign) detected @ PC 0x120026614
   437.leslie3d          floating point   Fortran   leslie3d.in  87402135744  41635.96  o.k.
    444.namd             floating point   C++    namd.input      64449976020  26798.88  o.k.
   445.gobmk           integer               C         capture.tst     494502991   260.29  o.k.
  447.dealII              floating point   C++      8                    wrong output
  450.soplex             floating point   C++    test.mps        72422927  31.95   o.k.
  453.povray            floating point   C++    test.ini       3597778011      1737.24   o.k.
  454.calculix            floating point   C        beampic.inp    251699786     101.04    o.k.
   456.hmmer            integer               C     bombesin.hmm    2386768547  997.97  o.k.
  458.sjeng               integer               C      test.txt    21682684235   9406.80  o.k.
   459.GemsFDTD    floating point   Fortran   test.in  11046857318  5289.88  o.k.
   462.libquantum     integer               C      33 5     292639209   111.64   o.k.
   464.h264ref           integer               C      foreman_test_encoder_baseline.cfg    154340641371   67426.00   o.k.
   465.tonto              floating point   Fortran     compile error
   470.lbm                floating point   C  100_100_130_cf_a.of    7058506019    4599.69   o.k.
   471.omnetpp       integer               C++    omnetpp.ini 2450821721  1153.36   o.k.
   473.astar               integer             C++   lake.cfg            35796103621    16433.83  output is a little different with the reference
   481.wrf                  floating point C/Fortran           STOP wrf_abort. Need library
   482.sphinx3        floating point   C   args.an4    9352006427    4011.67   o.k.
   483.xalancbmk     integer             C++    test.xml       501493417    276.77   o.k.
   998.specrand         integer     C     324342 24239   71348559   32.93   o.k.
  999.specrand         floating point     C     324342 24239  71348559   32.31.58   o.k.