read_4k: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 read_16k: (g=0): rw=read, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 read_128k: (g=0): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 read_256k: (g=0): rw=read, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 read_1m: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 read_4m: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 write_4k: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 write_16k: (g=0): rw=write, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 write_128k: (g=0): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 write_256k: (g=0): rw=write, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 write_1m: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 write_4m: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randread_4k: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randread_16k: (g=0): rw=randread, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randread_128k: (g=0): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randread_256k: (g=0): rw=randread, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randread_1m: (g=0): rw=randread, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randread_4m: (g=0): rw=randread, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randwrite_4k: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randwrite_16k: (g=0): rw=randwrite, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randwrite_128k: (g=0): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randwrite_256k: (g=0): rw=randwrite, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randwrite_1m: (g=0): rw=randwrite, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randwrite_4m: (g=0): rw=randwrite, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randrw_4k: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randrw_16k: (g=0): rw=randrw, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randrw_128k: (g=0): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randrw_256k: (g=0): rw=randrw, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randrw_1m: (g=0): rw=randrw, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randrw_4m: (g=0): rw=randrw, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 fio-3.28 Starting 30 processes fio: pid=756265, got signal=9(6),w(6),m(6)][3.4%][r=41.8MiB/s,w=17.2MiB/s][r=120,w=150 IOPS][eta 28m:58s] Jobs: 4 (f=4): [_(5),R(1),_(5),W(1),_(10),w(1),K(1),_(5),m(1)][41.0%][r=328MiB/s][r=82 IOPS][eta 43m:27s] read_4k: (groupid=0, jobs=1): err= 0: pid=756242: Tue May 30 03:33:42 2023 read: IOPS=65, BW=263KiB/s (269kB/s)(462MiB/1800096msec) bw ( KiB/s): min= 7, max= 3584, per=1.44%, avg=629.61, stdev=511.39, samples=1502 iops : min= 1, max= 896, avg=157.28, stdev=127.89, samples=1502 cpu : usr=0.01%, sys=0.04%, ctx=4265, majf=26, minf=98 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=118398,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_16k: (groupid=0, jobs=1): err= 0: pid=756243: Tue May 30 03:33:42 2023 read: IOPS=50, BW=807KiB/s (826kB/s)(1419MiB/1800308msec) bw ( KiB/s): min= 7, max=14912, per=4.52%, avg=1969.82, stdev=1675.22, samples=1473 iops : min= 0, max= 932, avg=122.94, stdev=104.75, samples=1473 cpu : usr=0.01%, sys=0.04%, ctx=5482, majf=10, minf=288 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=90814,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_128k: (groupid=0, jobs=1): err= 0: pid=756244: Tue May 30 03:33:42 2023 read: IOPS=26, BW=3362KiB/s (3443kB/s)(5912MiB/1800571msec) bw ( KiB/s): min= 253, max=49152, per=17.52%, avg=7634.65, stdev=7296.53, samples=1583 iops : min= 1, max= 384, avg=59.44, stdev=57.04, samples=1583 cpu : usr=0.01%, sys=0.04%, ctx=7758, majf=46, minf=2097 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=47294,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_256k: (groupid=0, jobs=1): err= 0: pid=756245: Tue May 30 03:33:42 2023 read: IOPS=16, BW=4321KiB/s (4425kB/s)(7632MiB/1808545msec) bw ( KiB/s): min= 507, max=52224, per=22.82%, avg=9945.47, stdev=9384.90, samples=1564 iops : min= 1, max= 204, avg=38.63, stdev=36.70, samples=1564 cpu : usr=0.01%, sys=0.04%, ctx=8034, majf=23, minf=4131 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=30526,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_1m: (groupid=0, jobs=1): err= 0: pid=756246: Tue May 30 03:33:42 2023 read: IOPS=5, BW=5394KiB/s (5523kB/s)(9534MiB/1809962msec) bw ( KiB/s): min= 1396, max=141029, per=28.19%, avg=12285.41, stdev=12128.27, samples=1578 iops : min= 1, max= 137, avg=11.78, stdev=11.88, samples=1578 cpu : usr=0.00%, sys=0.04%, ctx=7751, majf=9, minf=16431 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.3% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=9534,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_4m: (groupid=0, jobs=1): err= 0: pid=756247: Tue May 30 03:33:42 2023 read: IOPS=1, BW=6504KiB/s (6660kB/s)(11.2GiB/1812598msec) bw ( KiB/s): min= 8110, max=163840, per=38.04%, avg=16580.25, stdev=14393.58, samples=1387 iops : min= 1, max= 40, avg= 3.83, stdev= 3.57, samples=1387 cpu : usr=0.00%, sys=0.05%, ctx=8506, majf=8, minf=65572 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.3%, 16=0.6%, 32=1.1%, >=64=97.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=2878,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_4k: (groupid=0, jobs=1): err= 0: pid=756248: Tue May 30 03:33:42 2023 write: IOPS=17, BW=71.3KiB/s (73.0kB/s)(126MiB/1808702msec); 0 zone resets bw ( KiB/s): min= 7, max= 1026, per=1.49%, avg=373.95, stdev=240.66, samples=687 iops : min= 1, max= 256, avg=93.35, stdev=60.18, samples=687 cpu : usr=0.00%, sys=0.01%, ctx=1987, majf=17, minf=44 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,32254,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_16k: (groupid=0, jobs=1): err= 0: pid=756249: Tue May 30 03:33:42 2023 write: IOPS=18, BW=292KiB/s (299kB/s)(516MiB/1808257msec); 0 zone resets bw ( KiB/s): min= 31, max= 4096, per=4.18%, avg=1046.36, stdev=837.85, samples=1006 iops : min= 1, max= 256, avg=65.21, stdev=52.40, samples=1006 cpu : usr=0.01%, sys=0.02%, ctx=3506, majf=21, minf=51 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,33022,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_128k: (groupid=0, jobs=1): err= 0: pid=756250: Tue May 30 03:33:42 2023 write: IOPS=10, BW=1395KiB/s (1428kB/s)(2464MiB/1808578msec); 0 zone resets bw ( KiB/s): min= 253, max=18176, per=12.97%, avg=3246.56, stdev=3051.58, samples=1541 iops : min= 1, max= 142, avg=25.15, stdev=23.88, samples=1541 cpu : usr=0.01%, sys=0.03%, ctx=7318, majf=36, minf=42 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,19710,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_256k: (groupid=0, jobs=1): err= 0: pid=756251: Tue May 30 03:33:42 2023 write: IOPS=10, BW=2754KiB/s (2820kB/s)(4864MiB/1808581msec); 0 zone resets bw ( KiB/s): min= 507, max=33792, per=24.99%, avg=6252.23, stdev=5900.29, samples=1587 iops : min= 1, max= 132, avg=24.20, stdev=23.10, samples=1587 cpu : usr=0.01%, sys=0.04%, ctx=7050, majf=56, minf=57 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,19454,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_1m: (groupid=0, jobs=1): err= 0: pid=756252: Tue May 30 03:33:42 2023 write: IOPS=3, BW=3872KiB/s (3965kB/s)(6846MiB/1810705msec); 0 zone resets bw ( KiB/s): min= 535, max=96256, per=35.69%, avg=8929.57, stdev=8572.40, samples=1555 iops : min= 0, max= 94, avg= 8.50, stdev= 8.42, samples=1555 cpu : usr=0.02%, sys=0.03%, ctx=7901, majf=46, minf=52 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.5%, >=64=99.1% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,6846,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_4m: (groupid=0, jobs=1): err= 0: pid=756253: Tue May 30 03:33:42 2023 write: IOPS=1, BW=7227KiB/s (7401kB/s)(12.5GiB/1812386msec); 0 zone resets bw ( KiB/s): min= 3711, max=114688, per=72.65%, avg=18174.05, stdev=14213.44, samples=1412 iops : min= 0, max= 28, avg= 4.22, stdev= 3.52, samples=1412 cpu : usr=0.03%, sys=0.04%, ctx=8088, majf=16, minf=41 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.3%, 16=0.5%, 32=1.0%, >=64=98.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,3198,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_4k: (groupid=0, jobs=1): err= 0: pid=756254: Tue May 30 03:33:42 2023 read: IOPS=7, BW=28.9KiB/s (29.6kB/s)(51.0MiB/1809209msec) bw ( KiB/s): min= 3, max= 632, per=0.15%, avg=65.06, stdev=64.71, samples=1592 iops : min= 0, max= 158, avg=16.10, stdev=16.21, samples=1592 cpu : usr=0.01%, sys=0.02%, ctx=7728, majf=17, minf=101 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=13054,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_16k: (groupid=0, jobs=1): err= 0: pid=756255: Tue May 30 03:33:42 2023 read: IOPS=7, BW=117KiB/s (119kB/s)(206MiB/1808792msec) bw ( KiB/s): min= 31, max= 4800, per=0.60%, avg=263.39, stdev=294.02, samples=1592 iops : min= 1, max= 300, avg=16.26, stdev=18.42, samples=1592 cpu : usr=0.00%, sys=0.03%, ctx=8112, majf=22, minf=311 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=13182,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_128k: (groupid=0, jobs=1): err= 0: pid=756256: Tue May 30 03:33:42 2023 read: IOPS=6, BW=828KiB/s (848kB/s)(1464MiB/1809466msec) bw ( KiB/s): min= 58, max=14592, per=4.36%, avg=1898.24, stdev=1821.39, samples=1570 iops : min= 0, max= 114, avg=14.61, stdev=14.28, samples=1570 cpu : usr=0.01%, sys=0.03%, ctx=7709, majf=10, minf=2078 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=11710,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_256k: (groupid=0, jobs=1): err= 0: pid=756257: Tue May 30 03:33:42 2023 read: IOPS=5, BW=1513KiB/s (1549kB/s)(2672MiB/1808568msec) bw ( KiB/s): min= 507, max=45988, per=7.82%, avg=3407.72, stdev=3340.39, samples=1591 iops : min= 1, max= 179, avg=13.10, stdev=13.09, samples=1591 cpu : usr=0.00%, sys=0.03%, ctx=7750, majf=24, minf=4161 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=10686,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_1m: (groupid=0, jobs=1): err= 0: pid=756258: Tue May 30 03:33:42 2023 read: IOPS=4, BW=4632KiB/s (4743kB/s)(8190MiB/1810447msec) bw ( KiB/s): min= 1382, max=157696, per=24.58%, avg=10711.42, stdev=11671.19, samples=1553 iops : min= 1, max= 154, avg=10.25, stdev=11.44, samples=1553 cpu : usr=0.01%, sys=0.04%, ctx=7704, majf=16, minf=16424 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=8190,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_4m: (groupid=0, jobs=1): err= 0: pid=756259: Tue May 30 03:33:42 2023 read: IOPS=1, BW=6361KiB/s (6514kB/s)(11.0GiB/1812028msec) bw ( KiB/s): min= 8110, max=180224, per=37.41%, avg=16302.05, stdev=12620.91, samples=1382 iops : min= 1, max= 44, avg= 3.76, stdev= 3.13, samples=1382 cpu : usr=0.00%, sys=0.05%, ctx=8333, majf=13, minf=65574 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.3%, 16=0.6%, 32=1.1%, >=64=97.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=2814,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_4k: (groupid=0, jobs=1): err= 0: pid=756260: Tue May 30 03:33:42 2023 write: IOPS=6, BW=27.6KiB/s (28.2kB/s)(48.7MiB/1809467msec); 0 zone resets bw ( KiB/s): min= 7, max= 608, per=0.24%, avg=61.92, stdev=61.11, samples=1594 iops : min= 1, max= 152, avg=15.32, stdev=15.30, samples=1594 cpu : usr=0.01%, sys=0.03%, ctx=7901, majf=15, minf=39 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,12478,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_16k: (groupid=0, jobs=1): err= 0: pid=756261: Tue May 30 03:33:42 2023 write: IOPS=6, BW=111KiB/s (114kB/s)(196MiB/1809960msec); 0 zone resets bw ( KiB/s): min= 21, max= 2267, per=1.00%, avg=251.66, stdev=245.21, samples=1585 iops : min= 1, max= 141, avg=15.53, stdev=15.34, samples=1585 cpu : usr=0.01%, sys=0.02%, ctx=7774, majf=25, minf=49 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,12542,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_128k: (groupid=0, jobs=1): err= 0: pid=756262: Tue May 30 03:33:42 2023 write: IOPS=5, BW=747KiB/s (765kB/s)(1320MiB/1809910msec); 0 zone resets bw ( KiB/s): min= 250, max=16128, per=6.77%, avg=1693.82, stdev=1679.84, samples=1581 iops : min= 1, max= 126, avg=13.02, stdev=13.16, samples=1581 cpu : usr=0.01%, sys=0.03%, ctx=7877, majf=34, minf=47 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,10558,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_256k: (groupid=0, jobs=1): err= 0: pid=756263: Tue May 30 03:33:42 2023 write: IOPS=5, BW=1431KiB/s (1465kB/s)(2528MiB/1809156msec); 0 zone resets bw ( KiB/s): min= 507, max=43520, per=13.20%, avg=3302.83, stdev=3326.92, samples=1552 iops : min= 1, max= 170, avg=12.68, stdev=13.05, samples=1552 cpu : usr=0.01%, sys=0.03%, ctx=7596, majf=13, minf=44 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,10110,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_1m: (groupid=0, jobs=1): err= 0: pid=756264: Tue May 30 03:33:42 2023 write: IOPS=3, BW=3832KiB/s (3924kB/s)(6782MiB/1812336msec); 0 zone resets bw ( KiB/s): min= 1061, max=73580, per=36.12%, avg=9036.13, stdev=8080.70, samples=1522 iops : min= 1, max= 71, avg= 8.60, stdev= 7.92, samples=1522 cpu : usr=0.02%, sys=0.03%, ctx=7868, majf=22, minf=42 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.5%, >=64=99.1% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,6782,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_4m: (groupid=0, jobs=1): err= 0: pid=756265: Tue May 30 03:33:42 2023 cpu : usr=0.00%, sys=0.00%, ctx=0, majf=0, minf=0 IO depths : 1=1.0%, 2=2.0%, 4=3.9%, 8=7.8%, 16=15.7%, 32=31.4%, >=64=38.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,102,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_4k: (groupid=0, jobs=1): err= 0: pid=756266: Tue May 30 03:33:42 2023 read: IOPS=5, BW=21.5KiB/s (22.0kB/s)(38.0MiB/1809158msec) bw ( KiB/s): min= 3, max= 479, per=0.11%, avg=50.79, stdev=47.71, samples=1519 iops : min= 0, max= 119, avg=12.53, stdev=11.95, samples=1519 write: IOPS=1, BW=7231B/s (7231B/s)(12.5MiB/1809158msec); 0 zone resets bw ( KiB/s): min= 7, max= 159, per=0.09%, avg=23.00, stdev=19.54, samples=1095 iops : min= 1, max= 39, avg= 5.59, stdev= 4.91, samples=1095 cpu : usr=0.01%, sys=0.02%, ctx=7581, majf=11, minf=45 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=9732,3194,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_16k: (groupid=0, jobs=1): err= 0: pid=756267: Tue May 30 03:33:42 2023 read: IOPS=4, BW=76.7KiB/s (78.6kB/s)(136MiB/1809955msec) bw ( KiB/s): min= 31, max= 1248, per=0.43%, avg=187.56, stdev=169.74, samples=1466 iops : min= 1, max= 78, avg=11.52, stdev=10.64, samples=1466 write: IOPS=1, BW=25.6KiB/s (26.3kB/s)(45.3MiB/1809955msec); 0 zone resets bw ( KiB/s): min= 31, max= 480, per=0.34%, avg=86.62, stdev=66.49, samples=1056 iops : min= 1, max= 30, avg= 5.21, stdev= 4.19, samples=1056 cpu : usr=0.01%, sys=0.02%, ctx=7611, majf=21, minf=44 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=8682,2900,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_128k: (groupid=0, jobs=1): err= 0: pid=756268: Tue May 30 03:33:42 2023 read: IOPS=4, BW=568KiB/s (581kB/s)(1003MiB/1809913msec) bw ( KiB/s): min= 253, max=18944, per=3.23%, avg=1406.57, stdev=1326.40, samples=1453 iops : min= 1, max= 148, avg=10.78, stdev=10.41, samples=1453 write: IOPS=1, BW=184KiB/s (188kB/s)(325MiB/1809913msec); 0 zone resets bw ( KiB/s): min= 178, max= 4352, per=2.57%, avg=642.99, stdev=497.98, samples=1023 iops : min= 1, max= 34, avg= 4.83, stdev= 3.93, samples=1023 cpu : usr=0.01%, sys=0.03%, ctx=8583, majf=26, minf=51 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=8026,2596,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_256k: (groupid=0, jobs=1): err= 0: pid=756269: Tue May 30 03:33:42 2023 read: IOPS=4, BW=1099KiB/s (1125kB/s)(1942MiB/1809332msec) bw ( KiB/s): min= 500, max=25600, per=6.10%, avg=2657.77, stdev=2613.47, samples=1482 iops : min= 1, max= 100, avg=10.17, stdev=10.24, samples=1482 write: IOPS=1, BW=368KiB/s (376kB/s)(650MiB/1809332msec); 0 zone resets bw ( KiB/s): min= 509, max= 6656, per=5.13%, avg=1283.23, stdev=964.45, samples=1025 iops : min= 1, max= 26, avg= 4.83, stdev= 3.81, samples=1025 cpu : usr=0.01%, sys=0.03%, ctx=8312, majf=23, minf=47 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=7768,2598,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_1m: (groupid=0, jobs=1): err= 0: pid=756270: Tue May 30 03:33:42 2023 read: IOPS=2, BW=2411KiB/s (2469kB/s)(4259MiB/1809093msec) bw ( KiB/s): min= 485, max=49250, per=13.88%, avg=6049.13, stdev=5068.35, samples=1424 iops : min= 0, max= 48, avg= 5.70, stdev= 4.99, samples=1424 write: IOPS=0, BW=812KiB/s (832kB/s)(1435MiB/1809093msec); 0 zone resets bw ( KiB/s): min= 485, max=18395, per=13.84%, avg=3462.27, stdev=2356.84, samples=841 iops : min= 0, max= 17, avg= 3.18, stdev= 2.34, samples=841 cpu : usr=0.01%, sys=0.03%, ctx=7949, majf=12, minf=39 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.3%, 32=0.6%, >=64=98.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=4259,1435,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_4m: (groupid=0, jobs=1): err= 0: pid=756271: Tue May 30 03:33:42 2023 read: IOPS=1, BW=5342KiB/s (5470kB/s)(9456MiB/1812599msec) bw ( KiB/s): min= 2142, max=98304, per=34.44%, avg=15009.61, stdev=9898.29, samples=1267 iops : min= 0, max= 24, avg= 3.44, stdev= 2.48, samples=1267 write: IOPS=0, BW=1885KiB/s (1930kB/s)(3336MiB/1812599msec); 0 zone resets bw ( KiB/s): min= 8126, max=40960, per=42.19%, avg=10555.07, stdev=5176.39, samples=630 iops : min= 1, max= 10, avg= 2.39, stdev= 1.32, samples=630 cpu : usr=0.01%, sys=0.05%, ctx=8300, majf=18, minf=46 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.3%, 16=0.5%, 32=1.0%, >=64=98.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=2364,834,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 Run status group 0 (all jobs): READ: bw=42.6MiB/s (44.6MB/s), 21.5KiB/s-6504KiB/s (22.0kB/s-6660kB/s), io=75.3GiB (80.9GB), run=1800096-1812599msec WRITE: bw=24.4MiB/s (25.6MB/s), 7231B/s-7227KiB/s (7231B/s-7401kB/s), io=43.2GiB (46.4GB), run=1808257-1812599msec Disk stats (read/write): vdb: ios=169052/107683, merge=271593/92466, ticks=132401825/305200772, in_queue=439165369, util=100.00%