read_4k: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 read_16k: (g=0): rw=read, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 read_128k: (g=0): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 read_256k: (g=0): rw=read, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 read_1m: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 read_4m: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 write_4k: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 write_16k: (g=0): rw=write, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 write_128k: (g=0): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 write_256k: (g=0): rw=write, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 write_1m: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 write_4m: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randread_4k: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randread_16k: (g=0): rw=randread, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randread_128k: (g=0): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randread_256k: (g=0): rw=randread, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randread_1m: (g=0): rw=randread, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randread_4m: (g=0): rw=randread, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randwrite_4k: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randwrite_16k: (g=0): rw=randwrite, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randwrite_128k: (g=0): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randwrite_256k: (g=0): rw=randwrite, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randwrite_1m: (g=0): rw=randwrite, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randwrite_4m: (g=0): rw=randwrite, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 randrw_4k: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64 randrw_16k: (g=0): rw=randrw, bs=(R) 16.0KiB-16.0KiB, (W) 16.0KiB-16.0KiB, (T) 16.0KiB-16.0KiB, ioengine=libaio, iodepth=64 randrw_128k: (g=0): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64 randrw_256k: (g=0): rw=randrw, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64 randrw_1m: (g=0): rw=randrw, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=64 randrw_4m: (g=0): rw=randrw, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64 fio-3.28 Starting 30 processes read_4k: (groupid=0, jobs=1): err= 0: pid=11510: Thu Jun 1 21:46:21 2023 read: IOPS=64, BW=257KiB/s (264kB/s)(453MiB/1801446msec) bw ( KiB/s): min= 6, max=20392, per=1.69%, avg=686.62, stdev=752.90, samples=1344 iops : min= 1, max= 5098, avg=171.43, stdev=188.25, samples=1344 cpu : usr=0.01%, sys=0.06%, ctx=8036, majf=3508, minf=5743 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=115902,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_16k: (groupid=0, jobs=1): err= 0: pid=11511: Thu Jun 1 21:46:21 2023 read: IOPS=56, BW=912KiB/s (934kB/s)(1604MiB/1801136msec) bw ( KiB/s): min= 29, max=10208, per=6.35%, avg=2581.87, stdev=1737.05, samples=1265 iops : min= 1, max= 638, avg=161.15, stdev=108.65, samples=1265 cpu : usr=0.01%, sys=0.05%, ctx=6554, majf=3367, minf=5597 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=102654,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_128k: (groupid=0, jobs=1): err= 0: pid=11512: Thu Jun 1 21:46:21 2023 read: IOPS=30, BW=3842KiB/s (3935kB/s)(6760MiB/1801434msec) bw ( KiB/s): min= 235, max=54784, per=21.94%, avg=8915.32, stdev=7486.47, samples=1545 iops : min= 1, max= 428, avg=69.27, stdev=58.57, samples=1545 cpu : usr=0.01%, sys=0.08%, ctx=14807, majf=7135, minf=11687 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=54078,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_256k: (groupid=0, jobs=1): err= 0: pid=11513: Thu Jun 1 21:46:21 2023 read: IOPS=17, BW=4501KiB/s (4609kB/s)(7920MiB/1801615msec) bw ( KiB/s): min= 482, max=53248, per=26.21%, avg=10650.84, stdev=9345.01, samples=1512 iops : min= 1, max= 208, avg=41.18, stdev=36.59, samples=1512 cpu : usr=0.01%, sys=0.07%, ctx=15465, majf=7360, minf=14020 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=31678,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_1m: (groupid=0, jobs=1): err= 0: pid=11514: Thu Jun 1 21:46:21 2023 read: IOPS=4, BW=4619KiB/s (4730kB/s)(8126MiB/1801361msec) bw ( KiB/s): min= 1167, max=169984, per=27.14%, avg=11029.01, stdev=11256.22, samples=1490 iops : min= 1, max= 166, avg=10.27, stdev=11.05, samples=1490 cpu : usr=0.00%, sys=0.07%, ctx=14516, majf=6635, minf=25750 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=8126,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 read_4m: (groupid=0, jobs=1): err= 0: pid=11515: Thu Jun 1 21:46:21 2023 read: IOPS=1, BW=5745KiB/s (5883kB/s)(9.99GiB/1823735msec) bw ( KiB/s): min= 7557, max=195824, per=38.97%, avg=15838.96, stdev=12297.73, samples=1284 iops : min= 1, max= 47, avg= 3.34, stdev= 3.04, samples=1284 cpu : usr=0.00%, sys=0.07%, ctx=12196, majf=4395, minf=72258 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.3%, 16=0.6%, 32=1.3%, >=64=97.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=2558,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_4k: (groupid=0, jobs=1): err= 0: pid=11516: Thu Jun 1 21:46:21 2023 write: IOPS=16, BW=66.9KiB/s (68.5kB/s)(118MiB/1802346msec); 0 zone resets bw ( KiB/s): min= 6, max= 1024, per=1.61%, avg=417.90, stdev=213.51, samples=572 iops : min= 1, max= 256, avg=104.24, stdev=53.49, samples=572 cpu : usr=0.00%, sys=0.02%, ctx=2201, majf=1305, minf=2307 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,30142,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_16k: (groupid=0, jobs=1): err= 0: pid=11517: Thu Jun 1 21:46:21 2023 write: IOPS=17, BW=287KiB/s (294kB/s)(505MiB/1802217msec); 0 zone resets bw ( KiB/s): min= 6, max= 4096, per=6.33%, avg=1639.48, stdev=873.38, samples=627 iops : min= 0, max= 256, avg=102.15, stdev=54.71, samples=627 cpu : usr=0.01%, sys=0.02%, ctx=2805, majf=1210, minf=2298 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,32318,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_128k: (groupid=0, jobs=1): err= 0: pid=11518: Thu Jun 1 21:46:21 2023 write: IOPS=14, BW=1857KiB/s (1902kB/s)(3272MiB/1804191msec); 0 zone resets bw ( KiB/s): min= 50, max=21504, per=20.73%, avg=5368.63, stdev=4263.49, samples=1240 iops : min= 0, max= 168, avg=41.53, stdev=33.39, samples=1240 cpu : usr=0.01%, sys=0.05%, ctx=9517, majf=4384, minf=6348 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,26174,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_256k: (groupid=0, jobs=1): err= 0: pid=11519: Thu Jun 1 21:46:21 2023 write: IOPS=13, BW=3369KiB/s (3450kB/s)(6000MiB/1823310msec); 0 zone resets bw ( KiB/s): min= 100, max=49367, per=33.01%, avg=8548.03, stdev=7375.96, samples=1428 iops : min= 0, max= 192, avg=32.96, stdev=28.89, samples=1428 cpu : usr=0.02%, sys=0.05%, ctx=12084, majf=5395, minf=7930 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.7% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,23998,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_1m: (groupid=0, jobs=1): err= 0: pid=11520: Thu Jun 1 21:46:21 2023 write: IOPS=4, BW=5103KiB/s (5225kB/s)(9086MiB/1823366msec); 0 zone resets bw ( KiB/s): min= 1914, max=129774, per=46.97%, avg=12162.65, stdev=10976.06, samples=1507 iops : min= 1, max= 126, avg=11.38, stdev=10.77, samples=1507 cpu : usr=0.02%, sys=0.06%, ctx=14810, majf=6914, minf=9546 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.3% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,9086,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 write_4m: (groupid=0, jobs=1): err= 0: pid=11521: Thu Jun 1 21:46:21 2023 write: IOPS=1, BW=5598KiB/s (5732kB/s)(9976MiB/1824895msec); 0 zone resets bw ( KiB/s): min= 7557, max=147456, per=60.30%, avg=15614.93, stdev=11858.05, samples=1269 iops : min= 1, max= 36, avg= 3.27, stdev= 2.93, samples=1269 cpu : usr=0.02%, sys=0.05%, ctx=12696, majf=4535, minf=6648 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.3%, 16=0.6%, 32=1.3%, >=64=97.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,2494,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_4k: (groupid=0, jobs=1): err= 0: pid=11522: Thu Jun 1 21:46:21 2023 read: IOPS=6, BW=27.0KiB/s (27.6kB/s)(47.5MiB/1802163msec) bw ( KiB/s): min= 7, max= 936, per=0.15%, avg=61.82, stdev=62.72, samples=1550 iops : min= 1, max= 234, avg=15.06, stdev=15.79, samples=1550 cpu : usr=0.01%, sys=0.05%, ctx=15175, majf=7926, minf=10624 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=12158,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_16k: (groupid=0, jobs=1): err= 0: pid=11523: Thu Jun 1 21:46:21 2023 read: IOPS=6, BW=108KiB/s (111kB/s)(190MiB/1801438msec) bw ( KiB/s): min= 30, max= 2752, per=0.63%, avg=256.55, stdev=270.76, samples=1501 iops : min= 1, max= 172, avg=15.56, stdev=17.04, samples=1501 cpu : usr=0.01%, sys=0.05%, ctx=15119, majf=7784, minf=10640 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=12158,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_128k: (groupid=0, jobs=1): err= 0: pid=11524: Thu Jun 1 21:46:21 2023 read: IOPS=6, BW=787KiB/s (806kB/s)(1400MiB/1820283msec) bw ( KiB/s): min= 235, max=19200, per=4.57%, avg=1856.91, stdev=1905.21, samples=1521 iops : min= 1, max= 150, avg=14.01, stdev=15.01, samples=1521 cpu : usr=0.01%, sys=0.05%, ctx=15885, majf=8214, minf=12818 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=11198,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_256k: (groupid=0, jobs=1): err= 0: pid=11525: Thu Jun 1 21:46:21 2023 read: IOPS=5, BW=1503KiB/s (1539kB/s)(2656MiB/1808827msec) bw ( KiB/s): min= 472, max=41733, per=8.69%, avg=3533.33, stdev=4008.43, samples=1524 iops : min= 1, max= 163, avg=13.30, stdev=15.77, samples=1524 cpu : usr=0.01%, sys=0.06%, ctx=15122, majf=7802, minf=14622 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=10622,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_1m: (groupid=0, jobs=1): err= 0: pid=11526: Thu Jun 1 21:46:21 2023 read: IOPS=3, BW=3558KiB/s (3644kB/s)(6334MiB/1822799msec) bw ( KiB/s): min= 393, max=124678, per=21.09%, avg=8569.41, stdev=8609.68, samples=1493 iops : min= 0, max= 121, avg= 7.86, stdev= 8.50, samples=1493 cpu : usr=0.01%, sys=0.06%, ctx=14651, majf=7273, minf=26404 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.3%, 32=0.5%, >=64=99.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=6334,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randread_4m: (groupid=0, jobs=1): err= 0: pid=11527: Thu Jun 1 21:46:21 2023 read: IOPS=1, BW=5741KiB/s (5879kB/s)(9.99GiB/1824950msec) bw ( KiB/s): min= 7757, max=278528, per=38.87%, avg=15795.58, stdev=12875.08, samples=1288 iops : min= 1, max= 68, avg= 3.32, stdev= 3.19, samples=1288 cpu : usr=0.00%, sys=0.07%, ctx=13568, majf=5395, minf=73070 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.3%, 16=0.6%, 32=1.3%, >=64=97.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=2558,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_4k: (groupid=0, jobs=1): err= 0: pid=11528: Thu Jun 1 21:46:21 2023 write: IOPS=6, BW=25.0KiB/s (25.7kB/s)(44.2MiB/1808625msec); 0 zone resets bw ( KiB/s): min= 6, max= 520, per=0.22%, avg=57.99, stdev=54.85, samples=1538 iops : min= 1, max= 130, avg=14.11, stdev=13.81, samples=1538 cpu : usr=0.01%, sys=0.05%, ctx=15591, majf=8119, minf=10241 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,11326,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_16k: (groupid=0, jobs=1): err= 0: pid=11529: Thu Jun 1 21:46:21 2023 write: IOPS=6, BW=102KiB/s (105kB/s)(182MiB/1822800msec); 0 zone resets bw ( KiB/s): min= 18, max= 2007, per=0.92%, avg=239.54, stdev=227.81, samples=1540 iops : min= 1, max= 125, avg=14.49, stdev=14.36, samples=1540 cpu : usr=0.01%, sys=0.05%, ctx=15562, majf=8136, minf=10418 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,11646,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_128k: (groupid=0, jobs=1): err= 0: pid=11530: Thu Jun 1 21:46:21 2023 write: IOPS=5, BW=750KiB/s (768kB/s)(1320MiB/1802114msec); 0 zone resets bw ( KiB/s): min= 50, max=16063, per=6.77%, avg=1752.07, stdev=1676.05, samples=1528 iops : min= 0, max= 125, avg=13.20, stdev=13.21, samples=1528 cpu : usr=0.01%, sys=0.05%, ctx=15152, majf=7874, minf=10376 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,10558,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_256k: (groupid=0, jobs=1): err= 0: pid=11531: Thu Jun 1 21:46:21 2023 write: IOPS=5, BW=1432KiB/s (1466kB/s)(2544MiB/1819400msec); 0 zone resets bw ( KiB/s): min= 469, max=32507, per=13.28%, avg=3440.70, stdev=3477.98, samples=1499 iops : min= 1, max= 126, avg=12.93, stdev=13.71, samples=1499 cpu : usr=0.01%, sys=0.05%, ctx=14957, majf=7810, minf=10256 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,10174,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_1m: (groupid=0, jobs=1): err= 0: pid=11532: Thu Jun 1 21:46:21 2023 write: IOPS=4, BW=4207KiB/s (4308kB/s)(7486MiB/1822246msec); 0 zone resets bw ( KiB/s): min= 1939, max=110151, per=39.37%, avg=10194.03, stdev=9828.36, samples=1485 iops : min= 1, max= 107, avg= 9.44, stdev= 9.68, samples=1485 cpu : usr=0.02%, sys=0.06%, ctx=16421, majf=8350, minf=10763 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=0,7486,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randwrite_4m: (groupid=0, jobs=1): err= 0: pid=11533: Thu Jun 1 21:46:21 2023 cpu : usr=0.00%, sys=0.00%, ctx=0, majf=0, minf=0 IO depths : 1=1.2%, 2=2.4%, 4=4.9%, 8=9.8%, 16=19.5%, 32=39.0%, >=64=23.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,82,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_4k: (groupid=0, jobs=1): err= 0: pid=11534: Thu Jun 1 21:46:21 2023 read: IOPS=5, BW=20.3KiB/s (20.8kB/s)(36.0MiB/1811896msec) bw ( KiB/s): min= 1, max= 413, per=0.12%, avg=49.32, stdev=48.50, samples=1468 iops : min= 0, max= 103, avg=11.95, stdev=12.22, samples=1468 write: IOPS=1, BW=6818B/s (6818B/s)(11.8MiB/1811897msec); 0 zone resets bw ( KiB/s): min= 7, max= 173, per=0.08%, avg=21.79, stdev=19.04, samples=1077 iops : min= 1, max= 43, avg= 5.09, stdev= 4.83, samples=1077 cpu : usr=0.01%, sys=0.05%, ctx=15475, majf=8287, minf=10673 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=9206,3016,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_16k: (groupid=0, jobs=1): err= 0: pid=11535: Thu Jun 1 21:46:21 2023 read: IOPS=4, BW=79.1KiB/s (81.0kB/s)(140MiB/1815732msec) bw ( KiB/s): min= 28, max= 1920, per=0.47%, avg=191.75, stdev=183.01, samples=1476 iops : min= 1, max= 120, avg=11.51, stdev=11.56, samples=1476 write: IOPS=1, BW=26.3KiB/s (27.0kB/s)(46.7MiB/1815732msec); 0 zone resets bw ( KiB/s): min= 28, max= 604, per=0.34%, avg=87.64, stdev=73.85, samples=1075 iops : min= 1, max= 37, avg= 5.04, stdev= 4.70, samples=1075 cpu : usr=0.01%, sys=0.05%, ctx=15430, majf=7972, minf=10741 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=8978,2988,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_128k: (groupid=0, jobs=1): err= 0: pid=11536: Thu Jun 1 21:46:21 2023 read: IOPS=4, BW=567KiB/s (580kB/s)(1009MiB/1822492msec) bw ( KiB/s): min= 50, max=13154, per=3.47%, avg=1409.32, stdev=1297.60, samples=1453 iops : min= 0, max= 102, avg=10.50, stdev=10.24, samples=1453 write: IOPS=1, BW=184KiB/s (188kB/s)(327MiB/1822492msec); 0 zone resets bw ( KiB/s): min= 50, max= 4806, per=2.51%, avg=649.59, stdev=528.79, samples=1019 iops : min= 0, max= 37, avg= 4.58, stdev= 4.22, samples=1019 cpu : usr=0.01%, sys=0.05%, ctx=15847, majf=8288, minf=10833 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=8069,2617,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_256k: (groupid=0, jobs=1): err= 0: pid=11537: Thu Jun 1 21:46:21 2023 read: IOPS=4, BW=1073KiB/s (1098kB/s)(1907MiB/1820362msec) bw ( KiB/s): min= 470, max=25037, per=6.65%, avg=2702.58, stdev=2538.65, samples=1431 iops : min= 1, max= 97, avg=10.06, stdev=10.02, samples=1431 write: IOPS=1, BW=358KiB/s (367kB/s)(637MiB/1820362msec); 0 zone resets bw ( KiB/s): min= 100, max=10219, per=4.95%, avg=1282.16, stdev=1037.72, samples=1008 iops : min= 0, max= 39, avg= 4.54, stdev= 4.14, samples=1008 cpu : usr=0.01%, sys=0.05%, ctx=14745, majf=7704, minf=10655 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.3%, >=64=99.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=7627,2547,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_1m: (groupid=0, jobs=1): err= 0: pid=11538: Thu Jun 1 21:46:21 2023 read: IOPS=3, BW=3488KiB/s (3571kB/s)(6196MiB/1819227msec) bw ( KiB/s): min= 354, max=100352, per=21.87%, avg=8886.18, stdev=8241.44, samples=1412 iops : min= 0, max= 98, avg= 8.17, stdev= 8.16, samples=1412 write: IOPS=1, BW=1158KiB/s (1186kB/s)(2058MiB/1819227msec); 0 zone resets bw ( KiB/s): min= 354, max=36790, per=17.33%, avg=4487.99, stdev=3566.92, samples=929 iops : min= 0, max= 35, avg= 3.88, stdev= 3.60, samples=929 cpu : usr=0.01%, sys=0.06%, ctx=15514, majf=8040, minf=10618 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=6196,2058,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 randrw_4m: (groupid=0, jobs=1): err= 0: pid=11539: Thu Jun 1 21:46:21 2023 read: IOPS=0, BW=4031KiB/s (4128kB/s)(7184MiB/1824803msec) bw ( KiB/s): min= 7135, max=81920, per=31.80%, avg=12922.27, stdev=7871.86, samples=1108 iops : min= 1, max= 20, avg= 2.62, stdev= 2.01, samples=1108 write: IOPS=0, BW=1423KiB/s (1457kB/s)(2536MiB/1824803msec); 0 zone resets bw ( KiB/s): min= 7380, max=57344, per=38.41%, avg=9947.34, stdev=4644.14, samples=501 iops : min= 1, max= 14, avg= 1.88, stdev= 1.24, samples=501 cpu : usr=0.01%, sys=0.06%, ctx=12823, majf=5279, minf=7469 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.3%, 16=0.7%, 32=1.3%, >=64=97.4% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% issued rwts: total=1796,634,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=64 Run status group 0 (all jobs): READ: bw=39.7MiB/s (41.6MB/s), 20.3KiB/s-5745KiB/s (20.8kB/s-5883kB/s), io=70.7GiB (75.9GB), run=1801136-1824950msec WRITE: bw=25.3MiB/s (26.5MB/s), 6818B/s-5598KiB/s (6818B/s-5732kB/s), io=45.1GiB (48.4GB), run=1802114-1824895msec Disk stats (read/write): vdb: ios=167743/102288, merge=292581/103979, ticks=139513630/289117964, in_queue=430228378, util=99.58%