szaydel
9/17/2013 - 11:54 PM

fio test with iscsi+Multipath ZFS pool 8 vdevs, 16 disks - LUNs are 512b bs

fio test with iscsi+Multipath ZFS pool 8 vdevs, 16 disks - LUNs are 512b bs

root@cl01:/volumes/p0/test/d0# fio ~/rand.fio
random-writers: (g=0): rw=randwrite, bs=32K-32K/32K-32K/32K-32K, ioengine=solarisaio, iodepth=4
...
random-writers: (g=0): rw=randwrite, bs=32K-32K/32K-32K/32K-32K, ioengine=solarisaio, iodepth=4
random-readers: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=solarisaio, iodepth=4
...
random-readers: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=solarisaio, iodepth=4
fio-2.1.2
Starting 8 processes
Jobs: 1 (f=1): [___w____] [99.2% done] [0KB/830.3MB/0KB /s] [0/26.6K/0 iops] [eta 00m:02s]
random-writers: (groupid=0, jobs=1): err= 0: pid=24588: Tue Sep 17 16:52:06 2013
  write: io=0B, bw=53940KB/s, iops=1685, runt=233276msec
    slat (usec): min=2, max=170142, avg=54.71, stdev=1377.98
    clat (usec): min=1, max=542017, avg=2310.99, stdev=9923.96
     lat (usec): min=26, max=542028, avg=2365.70, stdev=10027.42
    clat percentiles (usec):
     |  1.00th=[    1],  5.00th=[    1], 10.00th=[    2], 20.00th=[   23],
     | 30.00th=[   39], 40.00th=[   51], 50.00th=[   63], 60.00th=[   77],
     | 70.00th=[  100], 80.00th=[  157], 90.00th=[ 9920], 95.00th=[10048],
     | 99.00th=[40192], 99.50th=[60160], 99.90th=[120320], 99.95th=[160768],
     | 99.99th=[264192]
    bw (KB  /s): min= 3801, max=896512, per=26.26%, avg=52952.25, stdev=101339.11
    lat (usec) : 2=5.83%, 4=9.42%, 10=0.17%, 20=4.14%, 50=19.16%
    lat (usec) : 100=30.98%, 250=15.34%, 500=3.15%, 750=0.62%, 1000=0.22%
    lat (msec) : 2=0.24%, 4=0.10%, 10=4.07%, 20=3.41%, 50=2.54%
    lat (msec) : 100=0.45%, 250=0.15%, 500=0.01%, 750=0.01%
  cpu          : usr=85.95%, sys=8.53%, ctx=878275, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=393216/d=0, short=r=0/w=0/d=0
random-writers: (groupid=0, jobs=1): err= 0: pid=24589: Tue Sep 17 16:52:06 2013
  write: io=0B, bw=57944KB/s, iops=1810, runt=217156msec
    slat (usec): min=2, max=247978, avg=59.87, stdev=1530.53
    clat (usec): min=1, max=679996, avg=2142.44, stdev=9551.51
     lat (usec): min=24, max=680047, avg=2202.31, stdev=9679.68
    clat percentiles (usec):
     |  1.00th=[    1],  5.00th=[    1], 10.00th=[    2], 20.00th=[    2],
     | 30.00th=[   22], 40.00th=[   44], 50.00th=[   55], 60.00th=[   70],
     | 70.00th=[   91], 80.00th=[  129], 90.00th=[  988], 95.00th=[10048],
     | 99.00th=[40192], 99.50th=[50432], 99.90th=[120320], 99.95th=[144384],
     | 99.99th=[246784]
    bw (KB  /s): min= 2514, max=707264, per=28.31%, avg=57097.53, stdev=112571.20
    lat (usec) : 2=9.12%, 4=13.93%, 10=0.11%, 20=6.51%, 50=15.72%
    lat (usec) : 100=27.77%, 250=14.35%, 500=1.99%, 750=0.35%, 1000=0.14%
    lat (msec) : 2=0.16%, 4=0.06%, 10=3.57%, 20=3.22%, 50=2.44%
    lat (msec) : 100=0.41%, 250=0.14%, 500=0.01%, 750=0.01%
  cpu          : usr=85.39%, sys=9.07%, ctx=925780, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=393216/d=0, short=r=0/w=0/d=0
random-writers: (groupid=0, jobs=1): err= 0: pid=24590: Tue Sep 17 16:52:06 2013
  write: io=0B, bw=54140KB/s, iops=1691, runt=232416msec
    slat (usec): min=2, max=259880, avg=55.26, stdev=1248.88
    clat (usec): min=1, max=460397, avg=2302.31, stdev=10286.08
     lat (usec): min=24, max=460408, avg=2357.57, stdev=10360.84
    clat percentiles (usec):
     |  1.00th=[    1],  5.00th=[    1], 10.00th=[    2], 20.00th=[   17],
     | 30.00th=[   42], 40.00th=[   53], 50.00th=[   65], 60.00th=[   86],
     | 70.00th=[  107], 80.00th=[  155], 90.00th=[  556], 95.00th=[19840],
     | 99.00th=[40704], 99.50th=[60160], 99.90th=[120320], 99.95th=[158720],
     | 99.99th=[242688]
    bw (KB  /s): min= 3080, max=513664, per=26.55%, avg=53531.45, stdev=100533.03
    lat (usec) : 2=5.27%, 4=9.73%, 10=0.22%, 20=5.73%, 50=15.63%
    lat (usec) : 100=30.72%, 250=18.91%, 500=3.50%, 750=0.82%, 1000=0.23%
    lat (msec) : 2=0.27%, 4=0.10%, 10=2.57%, 20=2.76%, 50=2.80%
    lat (msec) : 100=0.57%, 250=0.17%, 500=0.01%
  cpu          : usr=86.07%, sys=8.48%, ctx=997346, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=393216/d=0, short=r=0/w=0/d=0
random-writers: (groupid=0, jobs=1): err= 0: pid=24591: Tue Sep 17 16:52:06 2013
  write: io=0B, bw=50415KB/s, iops=1575, runt=249585msec
    slat (usec): min=3, max=286718, avg=37.28, stdev=1298.58
    clat (usec): min=1, max=498722, avg=2494.69, stdev=10826.29
     lat (usec): min=27, max=498734, avg=2531.97, stdev=10913.09
    clat percentiles (usec):
     |  1.00th=[    2],  5.00th=[   19], 10.00th=[   35], 20.00th=[   41],
     | 30.00th=[   46], 40.00th=[   54], 50.00th=[   64], 60.00th=[   76],
     | 70.00th=[   96], 80.00th=[  143], 90.00th=[ 2768], 95.00th=[19840],
     | 99.00th=[49920], 99.50th=[61696], 99.90th=[129536], 99.95th=[168960],
     | 99.99th=[259072]
    bw (KB  /s): min= 3136, max=1793024, per=23.38%, avg=47154.40, stdev=145470.43
    lat (usec) : 2=0.91%, 4=2.02%, 10=0.03%, 20=2.17%, 50=29.02%
    lat (usec) : 100=37.00%, 250=14.83%, 500=2.73%, 750=0.66%, 1000=0.26%
    lat (msec) : 2=0.30%, 4=0.11%, 10=3.09%, 20=3.13%, 50=2.95%
    lat (msec) : 100=0.58%, 250=0.18%, 500=0.01%
  cpu          : usr=87.69%, sys=8.10%, ctx=733597, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=393216/d=0, short=r=0/w=0/d=0
random-readers: (groupid=0, jobs=1): err= 0: pid=24592: Tue Sep 17 16:52:06 2013
  read : io=0B, bw=1191.5MB/s, iops=38113, runt= 10317msec
    slat (usec): min=1, max=122457, avg= 9.26, stdev=325.54
    clat (usec): min=1, max=410116, avg=92.05, stdev=1706.94
     lat (usec): min=20, max=410125, avg=101.31, stdev=1741.06
    clat percentiles (usec):
     |  1.00th=[    2],  5.00th=[   24], 10.00th=[   28], 20.00th=[   33],
     | 30.00th=[   37], 40.00th=[   43], 50.00th=[   49], 60.00th=[   57],
     | 70.00th=[   68], 80.00th=[   90], 90.00th=[  135], 95.00th=[  193],
     | 99.00th=[  382], 99.50th=[  510], 99.90th=[ 1144], 99.95th=[ 1912],
     | 99.99th=[70144]
    bw (MB  /s): min=  107, max= 1956, per=27.34%, avg=1206.58, stdev=606.46
    lat (usec) : 2=0.99%, 4=1.02%, 10=0.39%, 20=1.11%, 50=46.97%
    lat (usec) : 100=32.54%, 250=14.18%, 500=2.28%, 750=0.30%, 1000=0.09%
    lat (msec) : 2=0.08%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%
  cpu          : usr=68.67%, sys=122.23%, ctx=701003, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=393216/w=0/d=0, short=r=0/w=0/d=0
random-readers: (groupid=0, jobs=1): err= 0: pid=24593: Tue Sep 17 16:52:06 2013
  read : io=0B, bw=1103.5MB/s, iops=35310, runt= 11136msec
    slat (usec): min=1, max=240152, avg=10.33, stdev=471.21
    clat (usec): min=1, max=349134, avg=99.11, stdev=1691.99
     lat (usec): min=18, max=349140, avg=109.44, stdev=1756.55
    clat percentiles (usec):
     |  1.00th=[    2],  5.00th=[   24], 10.00th=[   28], 20.00th=[   33],
     | 30.00th=[   38], 40.00th=[   44], 50.00th=[   51], 60.00th=[   60],
     | 70.00th=[   73], 80.00th=[   98], 90.00th=[  153], 95.00th=[  221],
     | 99.00th=[  418], 99.50th=[  532], 99.90th=[ 1096], 99.95th=[ 2024],
     | 99.99th=[74240]
    bw (MB  /s): min=   93, max= 3271, per=25.24%, avg=1114.02, stdev=682.90
    lat (usec) : 2=0.46%, 4=0.68%, 10=0.13%, 20=1.14%, 50=45.89%
    lat (usec) : 100=32.05%, 250=15.86%, 500=3.19%, 750=0.38%, 1000=0.09%
    lat (msec) : 2=0.07%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%
  cpu          : usr=65.05%, sys=113.64%, ctx=691354, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=393216/w=0/d=0, short=r=0/w=0/d=0
random-readers: (groupid=0, jobs=1): err= 0: pid=24594: Tue Sep 17 16:52:06 2013
  read : io=0B, bw=1124.4MB/s, iops=35979, runt= 10929msec
    slat (usec): min=1, max=110075, avg= 8.68, stdev=218.58
    clat (usec): min=1, max=592834, avg=98.63, stdev=2195.71
     lat (usec): min=19, max=592840, avg=107.31, stdev=2212.60
    clat percentiles (usec):
     |  1.00th=[   25],  5.00th=[   28], 10.00th=[   31], 20.00th=[   34],
     | 30.00th=[   39], 40.00th=[   44], 50.00th=[   49], 60.00th=[   56],
     | 70.00th=[   67], 80.00th=[   89], 90.00th=[  141], 95.00th=[  209],
     | 99.00th=[  418], 99.50th=[  532], 99.90th=[ 1032], 99.95th=[ 1688],
     | 99.99th=[91648]
    bw (MB  /s): min=   24, max= 2132, per=25.10%, avg=1107.74, stdev=589.77
    lat (usec) : 2=0.01%, 4=0.03%, 10=0.01%, 20=0.15%, 50=50.17%
    lat (usec) : 100=32.76%, 250=13.41%, 500=2.87%, 750=0.39%, 1000=0.10%
    lat (msec) : 2=0.07%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
  cpu          : usr=70.81%, sys=110.37%, ctx=633236, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=393216/w=0/d=0, short=r=0/w=0/d=0
random-readers: (groupid=0, jobs=1): err= 0: pid=24595: Tue Sep 17 16:52:06 2013
  read : io=0B, bw=1157.6MB/s, iops=37039, runt= 10616msec
    slat (usec): min=1, max=130172, avg=10.22, stdev=368.52
    clat (usec): min=1, max=496695, avg=94.04, stdev=1743.42
     lat (usec): min=21, max=496716, avg=104.26, stdev=1792.32
    clat percentiles (usec):
     |  1.00th=[    2],  5.00th=[   23], 10.00th=[   28], 20.00th=[   34],
     | 30.00th=[   39], 40.00th=[   44], 50.00th=[   51], 60.00th=[   59],
     | 70.00th=[   72], 80.00th=[   93], 90.00th=[  143], 95.00th=[  201],
     | 99.00th=[  378], 99.50th=[  490], 99.90th=[ 1080], 99.95th=[ 2800],
     | 99.99th=[78336]
    bw (MB  /s): min=  112, max= 1648, per=26.34%, avg=1162.71, stdev=507.99
    lat (usec) : 2=0.85%, 4=0.99%, 10=0.58%, 20=1.53%, 50=44.49%
    lat (usec) : 100=33.41%, 250=15.13%, 500=2.54%, 750=0.29%, 1000=0.07%
    lat (msec) : 2=0.06%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%
  cpu          : usr=65.77%, sys=120.97%, ctx=695382, majf=0, minf=0
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=393216/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=49152MB, aggrb=4413.9MB/s, minb=1103.5MB/s, maxb=1191.5MB/s, mint=10317msec, maxt=11136msec
  WRITE: io=49152MB, aggrb=201661KB/s, minb=50415KB/s, maxb=57944KB/s, mint=217156msec, maxt=249585msec