Hi,
Thanks for reading my question!
I have setup the pool this week and its been in full use for couple of days now.
It was performing fine with the speed you could expect from the underlying harddisks
But today I noticed the write speed of the pool got real bad without understanding the reason.
When i do zpool iostat -n 1 is see more reads than write when copying data to the zfs volume
capacity operations bandwidth
pool alloc free read write read write
vm_server_storage_raidz 8.00T 2.91T 278 126 174M 85.5M
vm_server_storage_raidz 8.00T 2.91T 291 84 184M 56.9M
vm_server_storage_raidz 8.00T 2.91T 298 86 188M 53.4M
vm_server_storage_raidz 8.00T 2.91T 328 78 210M 51.4M
vm_server_storage_raidz 8.00T 2.91T 352 97 234M 70.9M
vm_server_storage_raidz 8.00T 2.91T 242 153 145M 101M
vm_server_storage_raidz 8.00T 2.91T 256 111 145M 67.9M
vm_server_storage_raidz 8.00T 2.91T 269 104 170M 66.4M
The pool consists of 3x toshiba 4tb sas disks (2019 512n) in raidz1.
Blocksize is 1m because its 99% used for big media files.
Changes to zfs pool I made are,
logbias=throughput (did it for performance but was using the wrong blocksize for my usage after all)
sync=disabled (also for performance, everything is always in 2 places and have backups)
I’m using zfs (v2.1.12-pve1) in proxmox (8.0.4 fully up to date)
root@pve:~# zpool list
NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
bedrijf_share_480gb_mirror 444G 174G 270G - - 0% 39% 1.00x ONLINE -
vm_server_storage_raidz 10.9T 8.00T 2.91T - - 0% 73% 1.00x ONLINE -
root@pve:~# zpool status
pool: vm_server_storage_raidz
state: ONLINE
scan: scrub repaired 0B in 08:00:14 with 0 errors on Sun Aug 13 08:24:16 2023
config:
NAME STATE READ WRITE CKSUM
vm_server_storage_raidz ONLINE 0 0 0
raidz1-0 ONLINE 0 0 0
scsi-350000399b8298ff5 ONLINE 0 0 0
scsi-350000399b8297c35 ONLINE 0 0 0
scsi-350000399a832a841 ONLINE 0 0 0
errors: No known data errors
root@pve:~# zfs get all vm_server_storage_raidz
NAME PROPERTY VALUE SOURCE
vm_server_storage_raidz type filesystem -
vm_server_storage_raidz creation Fri Aug 11 16:49 2023 -
vm_server_storage_raidz used 7.14T -
vm_server_storage_raidz available 264M -
vm_server_storage_raidz referenced 128K -
vm_server_storage_raidz compressratio 1.01x -
vm_server_storage_raidz mounted yes -
vm_server_storage_raidz quota none default
vm_server_storage_raidz reservation none default
vm_server_storage_raidz recordsize 128K default
vm_server_storage_raidz mountpoint /vm_server_storage_raidz default
vm_server_storage_raidz sharenfs off default
vm_server_storage_raidz checksum on default
vm_server_storage_raidz compression on local
vm_server_storage_raidz atime off local
vm_server_storage_raidz devices on default
vm_server_storage_raidz exec on default
vm_server_storage_raidz setuid on default
vm_server_storage_raidz readonly off default
vm_server_storage_raidz zoned off default
vm_server_storage_raidz snapdir hidden default
vm_server_storage_raidz aclmode discard default
vm_server_storage_raidz aclinherit restricted default
vm_server_storage_raidz createtxg 1 -
vm_server_storage_raidz canmount on default
vm_server_storage_raidz xattr on default
vm_server_storage_raidz copies 1 default
vm_server_storage_raidz version 5 -
vm_server_storage_raidz utf8only off -
vm_server_storage_raidz normalization none -
vm_server_storage_raidz casesensitivity sensitive -
vm_server_storage_raidz vscan off default
vm_server_storage_raidz nbmand off default
vm_server_storage_raidz sharesmb off default
vm_server_storage_raidz refquota none default
vm_server_storage_raidz refreservation none default
vm_server_storage_raidz guid 8487269358916759388 -
vm_server_storage_raidz primarycache all default
vm_server_storage_raidz secondarycache all default
vm_server_storage_raidz usedbysnapshots 0B -
vm_server_storage_raidz usedbydataset 128K -
vm_server_storage_raidz usedbychildren 7.14T -
vm_server_storage_raidz usedbyrefreservation 0B -
vm_server_storage_raidz logbias throughput local
vm_server_storage_raidz objsetid 54 -
vm_server_storage_raidz dedup off default
vm_server_storage_raidz mlslabel none default
vm_server_storage_raidz sync disabled local
vm_server_storage_raidz dnodesize legacy default
vm_server_storage_raidz refcompressratio 1.00x -
vm_server_storage_raidz written 128K -
vm_server_storage_raidz logicalused 5.41T -
vm_server_storage_raidz logicalreferenced 42K -
vm_server_storage_raidz volmode default default
vm_server_storage_raidz filesystem_limit none default
vm_server_storage_raidz snapshot_limit none default
vm_server_storage_raidz filesystem_count none default
vm_server_storage_raidz snapshot_count none default
vm_server_storage_raidz snapdev hidden default
vm_server_storage_raidz acltype off default
vm_server_storage_raidz context none default
vm_server_storage_raidz fscontext none default
vm_server_storage_raidz defcontext none default
vm_server_storage_raidz rootcontext none default
vm_server_storage_raidz relatime off default
vm_server_storage_raidz redundant_metadata all default
vm_server_storage_raidz overlay on default
vm_server_storage_raidz encryption off default
vm_server_storage_raidz keylocation none default
vm_server_storage_raidz keyformat none default
vm_server_storage_raidz pbkdf2iters 0 default
vm_server_storage_raidz special_small_blocks 0 default
root@pve:~# zpool get all vm_server_storage_raidz
NAME PROPERTY VALUE SOURCE
vm_server_storage_raidz size 10.9T -
vm_server_storage_raidz capacity 73% -
vm_server_storage_raidz altroot - default
vm_server_storage_raidz health ONLINE -
vm_server_storage_raidz guid 7167498555092595164 -
vm_server_storage_raidz version - default
vm_server_storage_raidz bootfs - default
vm_server_storage_raidz delegation on default
vm_server_storage_raidz autoreplace off default
vm_server_storage_raidz cachefile - default
vm_server_storage_raidz failmode wait default
vm_server_storage_raidz listsnapshots off default
vm_server_storage_raidz autoexpand off default
vm_server_storage_raidz dedupratio 1.00x -
vm_server_storage_raidz free 2.91T -
vm_server_storage_raidz allocated 8.00T -
vm_server_storage_raidz readonly off -
vm_server_storage_raidz ashift 12 local
vm_server_storage_raidz comment - default
vm_server_storage_raidz expandsize - -
vm_server_storage_raidz freeing 0 -
vm_server_storage_raidz fragmentation 0% -
vm_server_storage_raidz leaked 0 -
vm_server_storage_raidz multihost off default
vm_server_storage_raidz checkpoint - -
vm_server_storage_raidz load_guid 14822383124988917419 -
vm_server_storage_raidz autotrim off default
vm_server_storage_raidz compatibility off default
vm_server_storage_raidz feature@async_destroy enabled local
vm_server_storage_raidz feature@empty_bpobj active local
vm_server_storage_raidz feature@lz4_compress active local
vm_server_storage_raidz feature@multi_vdev_crash_dump enabled local
vm_server_storage_raidz feature@spacemap_histogram active local
vm_server_storage_raidz feature@enabled_txg active local
vm_server_storage_raidz feature@hole_birth active local
vm_server_storage_raidz feature@extensible_dataset active local
vm_server_storage_raidz feature@embedded_data active local
vm_server_storage_raidz feature@bookmarks enabled local
vm_server_storage_raidz feature@filesystem_limits enabled local
vm_server_storage_raidz feature@large_blocks active local
vm_server_storage_raidz feature@large_dnode enabled local
vm_server_storage_raidz feature@sha512 enabled local
vm_server_storage_raidz feature@skein enabled local
vm_server_storage_raidz feature@edonr enabled local
vm_server_storage_raidz feature@userobj_accounting active local
vm_server_storage_raidz feature@encryption enabled local
vm_server_storage_raidz feature@project_quota active local
vm_server_storage_raidz feature@device_removal enabled local
vm_server_storage_raidz feature@obsolete_counts enabled local
vm_server_storage_raidz feature@zpool_checkpoint enabled local
vm_server_storage_raidz feature@spacemap_v2 active local
vm_server_storage_raidz feature@allocation_classes enabled local
vm_server_storage_raidz feature@resilver_defer enabled local
vm_server_storage_raidz feature@bookmark_v2 enabled local
vm_server_storage_raidz feature@redaction_bookmarks enabled local
vm_server_storage_raidz feature@redacted_datasets enabled local
vm_server_storage_raidz feature@bookmark_written enabled local
vm_server_storage_raidz feature@log_spacemap active local
vm_server_storage_raidz feature@livelist enabled local
vm_server_storage_raidz feature@device_rebuild enabled local
vm_server_storage_raidz feature@zstd_compress enabled local
vm_server_storage_raidz feature@draid enabled local
System is a hp ml150 g9 xeon 2650 v4/128gb