read_4k: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 read_16k: (g=0): rw=read, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 read_128k: (g=0): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 read_256k: (g=0): rw=read, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 read_1m: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 read_4m: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 write_4k: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 write_16k: (g=0): rw=write, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 write_128k: (g=0): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 write_256k: (g=0): rw=write, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 write_1m: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 write_4m: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randread_4k: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randread_16k: (g=0): rw=randread, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randread_128k: (g=0): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randread_256k: (g=0): rw=randread, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randread_1m: (g=0): rw=randread, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randread_4m: (g=0): rw=randread, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randwrite_4k: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randwrite_16k: (g=0): rw=randwrite, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randwrite_128k: (g=0): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randwrite_256k: (g=0): rw=randwrite, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randwrite_1m: (g=0): rw=randwrite, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randwrite_4m: (g=0): rw=randwrite, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randrw_4k: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randrw_16k: (g=0): rw=randrw, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randrw_128k: (g=0): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randrw_256k: (g=0): rw=randrw, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randrw_1m: (g=0): rw=randrw, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randrw_4m: (g=0): rw=randrw, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 fio-3.28 Starting 30 processes read_4k: (groupid=0, jobs=1): err= 0: pid=2091: Thu Jun 1 19:15:16 2023 read: IOPS=234, BW=937KiB/s (960kB/s)(1648MiB/1800761msec) bw ( KiB/s): min= 7, max= 5138, per=1.00%, avg=1129.78, stdev=758.37, samples=2977 iops : min= 1, max= 1284, avg=282.23, stdev=189.57, samples=2977 cpu : usr=0.04%, sys=0.10%, ctx=10111, majf=7, minf=127 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=421950,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_16k: (groupid=0, jobs=1): err= 0: pid=2092: Thu Jun 1 19:15:16 2023 read: IOPS=185, BW=2970KiB/s (3041kB/s)(5221MiB/1800115msec) bw ( KiB/s): min= 31, max=20480, per=3.28%, avg=3710.95, stdev=2395.54, samples=2866 iops : min= 1, max= 1280, avg=231.72, stdev=149.68, samples=2866 cpu : usr=0.03%, sys=0.09%, ctx=8181, majf=4, minf=309 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=334142,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_128k: (groupid=0, jobs=1): err= 0: pid=2093: Thu Jun 1 19:15:16 2023 read: IOPS=90, BW=11.3MiB/s (11.9MB/s)(19.9GiB/1800136msec) bw ( KiB/s): min= 171, max=46592, per=11.65%, avg=13182.93, stdev=7695.14, samples=3159 iops : min= 1, max= 364, avg=102.85, stdev=60.07, samples=3159 cpu : usr=0.03%, sys=0.12%, ctx=21675, majf=12, minf=2112 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=163326,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_256k: (groupid=0, jobs=1): err= 0: pid=2094: Thu Jun 1 19:15:16 2023 read: IOPS=60, BW=15.1MiB/s (15.9MB/s)(26.6GiB/1801884msec) bw ( KiB/s): min= 510, max=78492, per=15.72%, avg=17790.61, stdev=11264.65, samples=3125 iops : min= 1, max= 306, avg=69.38, stdev=43.96, samples=3125 cpu : usr=0.02%, sys=0.11%, ctx=23304, majf=20, minf=4156 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=109118,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_1m: (groupid=0, jobs=1): err= 0: pid=2095: Thu Jun 1 19:15:16 2023 read: IOPS=13, BW=13.5MiB/s (14.1MB/s)(23.7GiB/1802313msec) bw ( KiB/s): min= 2031, max=170324, per=14.09%, avg=15938.11, stdev=12020.65, samples=3090 iops : min= 1, max= 166, avg=15.50, stdev=11.73, samples=3090 cpu : usr=0.01%, sys=0.10%, ctx=21881, majf=28, minf=16452 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=24254,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_4m: (groupid=0, jobs=1): err= 0: pid=2096: Thu Jun 1 19:15:16 2023 read: IOPS=4, BW=16.0MiB/s (16.8MB/s)(28.2GiB/1803246msec) bw ( KiB/s): min= 5468, max=196608, per=17.31%, avg=19588.95, stdev=13166.28, samples=2985 iops : min= 1, max= 48, avg= 4.72, stdev= 3.22, samples=2985 cpu : usr=0.01%, sys=0.10%, ctx=21907, majf=36, minf=65608 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.1% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=7230,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_4k: (groupid=0, jobs=1): err= 0: pid=2097: Thu Jun 1 19:15:16 2023 write: IOPS=39, BW=157KiB/s (160kB/s)(276MiB/1802061msec); 0 zone resets bw ( KiB/s): min= 5, max= 1624, per=0.51%, avg=472.14, stdev=170.38, samples=1187 iops : min= 1, max= 406, avg=117.90, stdev=42.56, samples=1187 cpu : usr=0.01%, sys=0.02%, ctx=1918, majf=3, minf=44 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,70590,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_16k: (groupid=0, jobs=1): err= 0: pid=2098: Thu Jun 1 19:15:16 2023 write: IOPS=47, BW=761KiB/s (780kB/s)(1340MiB/1801966msec); 0 zone resets bw ( KiB/s): min= 21, max= 6144, per=2.03%, avg=1873.97, stdev=659.50, samples=1456 iops : min= 1, max= 384, avg=116.97, stdev=41.19, samples=1456 cpu : usr=0.01%, sys=0.02%, ctx=2338, majf=4, minf=47 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,85758,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_128k: (groupid=0, jobs=1): err= 0: pid=2099: Thu Jun 1 19:15:16 2023 write: IOPS=44, BW=5639KiB/s (5775kB/s)(9920MiB/1801244msec); 0 zone resets bw ( KiB/s): min= 171, max=22016, per=7.63%, avg=7041.79, stdev=4555.79, samples=2869 iops : min= 1, max= 172, avg=54.91, stdev=35.57, samples=2869 cpu : usr=0.03%, sys=0.05%, ctx=12626, majf=5, minf=52 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,79358,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_256k: (groupid=0, jobs=1): err= 0: pid=2100: Thu Jun 1 19:15:16 2023 write: IOPS=37, BW=9694KiB/s (9926kB/s)(16.7GiB/1801684msec); 0 zone resets bw ( KiB/s): min= 475, max=41984, per=12.28%, avg=11335.54, stdev=7232.19, samples=3066 iops : min= 1, max= 164, avg=44.19, stdev=28.22, samples=3066 cpu : usr=0.04%, sys=0.07%, ctx=17707, majf=12, minf=57 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,68222,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_1m: (groupid=0, jobs=1): err= 0: pid=2101: Thu Jun 1 19:15:16 2023 write: IOPS=13, BW=13.4MiB/s (14.1MB/s)(23.6GiB/1802564msec); 0 zone resets bw ( KiB/s): min= 1521, max=127230, per=17.51%, avg=16161.51, stdev=10943.48, samples=3042 iops : min= 1, max= 124, avg=15.71, stdev=10.68, samples=3042 cpu : usr=0.05%, sys=0.06%, ctx=21594, majf=38, minf=64 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,24190,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_4m: (groupid=0, jobs=1): err= 0: pid=2102: Thu Jun 1 19:15:16 2023 write: IOPS=4, BW=17.3MiB/s (18.1MB/s)(30.5GiB/1805540msec); 0 zone resets bw ( KiB/s): min= 4348, max=180224, per=22.57%, avg=20832.22, stdev=12452.97, samples=3033 iops : min= 1, max= 44, avg= 5.03, stdev= 3.04, samples=3033 cpu : usr=0.06%, sys=0.06%, ctx=21974, majf=15, minf=59 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,7806,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_4k: (groupid=0, jobs=1): err= 0: pid=2103: Thu Jun 1 19:15:16 2023 read: IOPS=23, BW=94.2KiB/s (96.5kB/s)(166MiB/1800809msec) bw ( KiB/s): min= 7, max= 1544, per=0.09%, avg=107.34, stdev=73.43, samples=3142 iops : min= 1, max= 386, avg=26.79, stdev=18.36, samples=3142 cpu : usr=0.01%, sys=0.05%, ctx=21674, majf=12, minf=123 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=42430,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_16k: (groupid=0, jobs=1): err= 0: pid=2104: Thu Jun 1 19:15:16 2023 read: IOPS=22, BW=367KiB/s (376kB/s)(646MiB/1800801msec) bw ( KiB/s): min= 21, max= 4072, per=0.37%, avg=420.32, stdev=292.64, samples=3124 iops : min= 1, max= 254, avg=26.21, stdev=18.28, samples=3124 cpu : usr=0.02%, sys=0.05%, ctx=21682, majf=9, minf=310 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=41342,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_128k: (groupid=0, jobs=1): err= 0: pid=2105: Thu Jun 1 19:15:16 2023 read: IOPS=21, BW=2794KiB/s (2861kB/s)(4920MiB/1802996msec) bw ( KiB/s): min= 171, max=38989, per=2.82%, avg=3193.58, stdev=2468.62, samples=3137 iops : min= 1, max= 304, avg=24.87, stdev=19.28, samples=3137 cpu : usr=0.01%, sys=0.06%, ctx=21904, majf=26, minf=2114 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=39358,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_256k: (groupid=0, jobs=1): err= 0: pid=2106: Thu Jun 1 19:15:16 2023 read: IOPS=18, BW=4776KiB/s (4891kB/s)(8400MiB/1800921msec) bw ( KiB/s): min= 342, max=61050, per=4.82%, avg=5454.32, stdev=3861.99, samples=3133 iops : min= 1, max= 238, avg=21.23, stdev=15.08, samples=3133 cpu : usr=0.01%, sys=0.07%, ctx=21650, majf=26, minf=4160 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=33598,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_1m: (groupid=0, jobs=1): err= 0: pid=2107: Thu Jun 1 19:15:16 2023 read: IOPS=11, BW=11.1MiB/s (11.6MB/s)(19.5GiB/1801033msec) bw ( KiB/s): min= 1391, max=190845, per=11.59%, avg=13112.09, stdev=10964.31, samples=3094 iops : min= 1, max= 186, avg=12.74, stdev=10.70, samples=3094 cpu : usr=0.01%, sys=0.09%, ctx=21942, majf=19, minf=16436 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=19966,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_4m: (groupid=0, jobs=1): err= 0: pid=2108: Thu Jun 1 19:15:16 2023 read: IOPS=3, BW=15.2MiB/s (15.9MB/s)(26.7GiB/1805384msec) bw ( KiB/s): min= 5468, max=295503, per=17.30%, avg=19571.81, stdev=17000.85, samples=2830 iops : min= 1, max= 72, avg= 4.72, stdev= 4.16, samples=2830 cpu : usr=0.01%, sys=0.10%, ctx=22001, majf=0, minf=65581 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.5%, >=64=99.1% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=6846,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_4k: (groupid=0, jobs=1): err= 0: pid=2109: Thu Jun 1 19:15:16 2023 write: IOPS=21, BW=87.5KiB/s (89.6kB/s)(154MiB/1802949msec); 0 zone resets bw ( KiB/s): min= 5, max= 673, per=0.11%, avg=99.17, stdev=60.40, samples=3157 iops : min= 1, max= 168, avg=24.75, stdev=15.10, samples=3157 cpu : usr=0.01%, sys=0.05%, ctx=21346, majf=30, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,39422,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_16k: (groupid=0, jobs=1): err= 0: pid=2110: Thu Jun 1 19:15:16 2023 write: IOPS=21, BW=350KiB/s (359kB/s)(617MiB/1803736msec); 0 zone resets bw ( KiB/s): min= 21, max= 2432, per=0.43%, avg=395.87, stdev=236.95, samples=3169 iops : min= 1, max= 152, avg=24.68, stdev=14.80, samples=3169 cpu : usr=0.02%, sys=0.05%, ctx=21475, majf=41, minf=76 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,39486,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_128k: (groupid=0, jobs=1): err= 0: pid=2111: Thu Jun 1 19:15:16 2023 write: IOPS=19, BW=2459KiB/s (2518kB/s)(4328MiB/1802349msec); 0 zone resets bw ( KiB/s): min= 171, max=22272, per=3.07%, avg=2832.45, stdev=1856.11, samples=3109 iops : min= 1, max= 174, avg=22.05, stdev=14.49, samples=3109 cpu : usr=0.02%, sys=0.05%, ctx=21318, majf=6, minf=51 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,34622,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_256k: (groupid=0, jobs=1): err= 0: pid=2112: Thu Jun 1 19:15:16 2023 write: IOPS=17, BW=4555KiB/s (4665kB/s)(8016MiB/1801857msec); 0 zone resets bw ( KiB/s): min= 509, max=34372, per=5.63%, avg=5200.58, stdev=3202.78, samples=3137 iops : min= 1, max= 134, avg=20.24, stdev=12.50, samples=3137 cpu : usr=0.02%, sys=0.06%, ctx=21421, majf=16, minf=63 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,32062,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_1m: (groupid=0, jobs=1): err= 0: pid=2113: Thu Jun 1 19:15:16 2023 write: IOPS=13, BW=13.5MiB/s (14.2MB/s)(23.8GiB/1801673msec); 0 zone resets bw ( KiB/s): min= 1368, max=96448, per=17.35%, avg=16013.99, stdev=10435.46, samples=3095 iops : min= 1, max= 94, avg=15.57, stdev=10.18, samples=3095 cpu : usr=0.05%, sys=0.07%, ctx=21607, majf=15, minf=73 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,24382,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_4m: (groupid=0, jobs=1): err= 0: pid=2114: Thu Jun 1 19:15:16 2023 write: IOPS=4, BW=17.3MiB/s (18.1MB/s)(30.5GiB/1805684msec); 0 zone resets bw ( KiB/s): min= 5475, max=147751, per=22.54%, avg=20809.71, stdev=12084.38, samples=3035 iops : min= 1, max= 36, avg= 5.02, stdev= 2.96, samples=3035 cpu : usr=0.06%, sys=0.07%, ctx=21927, majf=25, minf=75 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,7806,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_4k: (groupid=0, jobs=1): err= 0: pid=2115: Thu Jun 1 19:15:16 2023 read: IOPS=16, BW=66.3KiB/s (67.9kB/s)(117MiB/1801055msec) bw ( KiB/s): min= 7, max= 448, per=0.07%, avg=76.17, stdev=46.80, samples=3111 iops : min= 1, max= 112, avg=19.00, stdev=11.70, samples=3111 write: IOPS=5, BW=21.8KiB/s (22.3kB/s)(38.3MiB/1801055msec); 0 zone resets bw ( KiB/s): min= 5, max= 200, per=0.03%, avg=28.42, stdev=18.81, samples=2737 iops : min= 1, max= 50, avg= 7.06, stdev= 4.71, samples=2737 cpu : usr=0.02%, sys=0.05%, ctx=21430, majf=19, minf=71 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=29861,9817,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_16k: (groupid=0, jobs=1): err= 0: pid=2116: Thu Jun 1 19:15:16 2023 read: IOPS=16, BW=264KiB/s (270kB/s)(464MiB/1801823msec) bw ( KiB/s): min= 31, max= 2084, per=0.27%, avg=303.68, stdev=194.63, samples=3110 iops : min= 1, max= 130, avg=18.92, stdev=12.16, samples=3110 write: IOPS=5, BW=86.5KiB/s (88.5kB/s)(152MiB/1801823msec); 0 zone resets bw ( KiB/s): min= 21, max= 705, per=0.12%, avg=112.01, stdev=76.91, samples=2760 iops : min= 1, max= 44, avg= 6.94, stdev= 4.81, samples=2760 cpu : usr=0.02%, sys=0.05%, ctx=21577, majf=7, minf=60 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=29685,9737,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_128k: (groupid=0, jobs=1): err= 0: pid=2117: Thu Jun 1 19:15:16 2023 read: IOPS=14, BW=1917KiB/s (1963kB/s)(3374MiB/1802363msec) bw ( KiB/s): min= 171, max=18725, per=1.98%, avg=2236.18, stdev=1463.15, samples=3074 iops : min= 1, max= 146, avg=17.40, stdev=11.43, samples=3074 write: IOPS=4, BW=628KiB/s (643kB/s)(1106MiB/1802363msec); 0 zone resets bw ( KiB/s): min= 170, max= 6156, per=0.92%, avg=852.12, stdev=595.79, samples=2642 iops : min= 1, max= 48, avg= 6.59, stdev= 4.66, samples=2642 cpu : usr=0.01%, sys=0.06%, ctx=21378, majf=24, minf=83 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=26990,8848,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_256k: (groupid=0, jobs=1): err= 0: pid=2118: Thu Jun 1 19:15:16 2023 read: IOPS=14, BW=3661KiB/s (3749kB/s)(6447MiB/1803108msec) bw ( KiB/s): min= 503, max=33792, per=3.75%, avg=4239.47, stdev=2769.90, samples=3095 iops : min= 1, max= 132, avg=16.49, stdev=10.81, samples=3095 write: IOPS=4, BW=1245KiB/s (1275kB/s)(2193MiB/1803108msec); 0 zone resets bw ( KiB/s): min= 296, max=12288, per=1.80%, avg=1666.76, stdev=1182.64, samples=2676 iops : min= 1, max= 48, avg= 6.45, stdev= 4.63, samples=2676 cpu : usr=0.02%, sys=0.06%, ctx=21440, majf=30, minf=87 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=25787,8771,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_1m: (groupid=0, jobs=1): err= 0: pid=2119: Thu Jun 1 19:15:16 2023 read: IOPS=11, BW=11.0MiB/s (11.6MB/s)(19.4GiB/1802921msec) bw ( KiB/s): min= 2027, max=114917, per=11.72%, avg=13261.55, stdev=8792.50, samples=3045 iops : min= 1, max= 112, avg=12.88, stdev= 8.58, samples=3045 write: IOPS=3, BW=3729KiB/s (3818kB/s)(6565MiB/1802921msec); 0 zone resets bw ( KiB/s): min= 1324, max=32833, per=5.85%, avg=5401.12, stdev=3570.58, samples=2473 iops : min= 1, max= 32, avg= 5.21, stdev= 3.49, samples=2473 cpu : usr=0.02%, sys=0.08%, ctx=21439, majf=30, minf=76 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=19865,6565,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_4m: (groupid=0, jobs=1): err= 0: pid=2120: Thu Jun 1 19:15:16 2023 cpu : usr=0.00%, sys=0.00%, ctx=0, majf=0, minf=0 IO depths : 1=0.6%, 2=1.2%, 4=2.3%, 8=4.7%, 16=9.4%, 32=18.7%, >=64=63.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=125,46,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 Run status group 0 (all jobs): READ: bw=110MiB/s (116MB/s), 66.3KiB/s-16.0MiB/s (67.9kB/s-16.8MB/s), io=195GiB (209GB), run=1800115-1805384msec WRITE: bw=90.1MiB/s (94.5MB/s), 21.8KiB/s-17.3MiB/s (22.3kB/s-18.1MB/s), io=159GiB (171GB), run=1801055-1805684msec Disk stats (read/write): vdb: ios=473195/326341, merge=980962/278566, ticks=148674260/304872811, in_queue=455232554, util=100.00%