Test NVMe
This commit is contained in:
Sergey Mazein
+33
-54
@@ -12,7 +12,7 @@
|
||||
```bash
|
||||
qm list
|
||||
pct list
|
||||
systemctl stop pvedaemon pve-cluster pve-storage pveproxy
|
||||
systemctl stop pvedaemon pve-cluster pveproxy
|
||||
```
|
||||
|
||||
---
|
||||
@@ -28,63 +28,40 @@ lsof +D /var/lib/vz
|
||||
```
|
||||
и заверши эти процессы.
|
||||
|
||||
|
||||
# создать том заново на всей свободной области
|
||||
# создать том заново на всей свободной области
|
||||
lvcreate -n data -l 100%FREE vg0
|
||||
|
||||
mkfs.xfs -f -s size=4096 -l size=256m -n size=4096 /dev/vg0/data
|
||||
|
||||
# подключить
|
||||
mkdir -p /var/lib/vz
|
||||
echo '/dev/vg0/data /var/lib/vz xfs defaults,noatime,nodiratime,logbufs=8,logbsize=256k 0 2' >> /etc/fstab
|
||||
mount -a
|
||||
|
||||
|
||||
# XFS с адекватными настройками
|
||||
mkfs.xfs -f -s size=4096 -l size=256m -n size=4096 /dev/vg0/data
|
||||
|
||||
# подключить
|
||||
mkdir -p /var/lib/vz
|
||||
echo '/dev/vg0/data /var/lib/vz xfs defaults,noatime,nodiratime,logbufs=8,logbsize=256k 0 2' >> /etc/fstab
|
||||
mount -a
|
||||
|
||||
mkfs.xfs -f -m reflink=1,crc=1 -l size=1536m,version=2,lazy-count=1 -n size=4096 -d su=512k,sw=8,agcount=64 /dev/vg0/data
|
||||
|
||||
|
||||
---
|
||||
|
||||
### 3. Пересоздай файловую систему XFS
|
||||
Оптимальные параметры для RAID0 с chunk = 1 Mb и 6 дисков для блокчейнов:
|
||||
```bash
|
||||
mkfs.xfs -f -s size=4096 -l size=256m -n size=4096 -d su=1m,sw=6 /dev/vg0/data
|
||||
mkfs.xfs -f -s size=4096 -l size=1g -n size=4096 -d su=1m,sw=6 /dev/md3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. Смонтируй том обратно
|
||||
Проверка, что XFS «увидел» выравнивание:
|
||||
```bash
|
||||
mount /dev/vg0/data /var/lib/vz
|
||||
xfs_info /dev/md3 | egrep 'sunit|swidth|sectsz'
|
||||
# Ожидаем: sunit=256 (256 * 4K = 1MiB), swidth=1536 (768 * 4K = 3MiB), sectsz=4096
|
||||
= sectsz=4096 attr=2, projid32bit=1
|
||||
= sunit=256 swidth=1536 blks
|
||||
= sectsz=4096 sunit=1 blks, lazy-count=1
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 5. Обнови `/etc/fstab`
|
||||
Добавь (или замени существующую) строку:
|
||||
```
|
||||
/dev/vg0/data /var/lib/vz xfs defaults,noatime,nodiratime,logbufs=8,logbsize=256k 0 2
|
||||
```
|
||||
добавить автоматически и применить
|
||||
```
|
||||
```bash
|
||||
mkdir -p /var/lib/vz
|
||||
UUID=$(blkid -s UUID -o value /dev/vg0/data)
|
||||
echo "UUID=$UUID /var/lib/vz xfs defaults,noatime,nodiratime,logbufs=8,logbsize=256k 0 2" >> /etc/fstab
|
||||
UUID=$(blkid -s UUID -o value /dev/md3)
|
||||
echo "UUID=$UUID /var/lib/vz xfs defaults,noatime,inode64,logbufs=8,logbsize=256k 0 2" >> /etc/fstab
|
||||
```
|
||||
```bash
|
||||
nano /etc/fstab
|
||||
```
|
||||
```bash
|
||||
mount -a
|
||||
```
|
||||
|
||||
|
||||
### 6. Перезапусти службы Proxmox
|
||||
```bash
|
||||
systemctl start pve-cluster pvedaemon pve-storage pveproxy
|
||||
systemctl start pve-cluster pvedaemon pveproxy
|
||||
```
|
||||
---
|
||||
|
||||
@@ -109,8 +86,8 @@ df -h /var/lib/vz
|
||||
```
|
||||
= sunit=256 swidth=1536 blks
|
||||
= sectsz=4096 sunit=1 blks, lazy-count=1
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/mapper/vg0-data 21T 410G 21T 2% /var/lib/vz
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/md3 21T 410G 21T 2% /var/lib/vz
|
||||
```
|
||||
|
||||
-----
|
||||
@@ -141,9 +118,8 @@ systemctl status irqbalance
|
||||
```
|
||||
|
||||
```bash
|
||||
echo 4096 | sudo tee /sys/block/md1/queue/read_ahead_kb
|
||||
echo 4096 | sudo tee /sys/block/md3/queue/read_ahead_kb
|
||||
for d in /sys/block/nvme*n1/queue/read_ahead_kb; do echo 2048 | sudo tee "$d"; done
|
||||
systemctl enable --now irqbalance
|
||||
```
|
||||
|
||||
💾 3. Применение постоянных значений
|
||||
@@ -154,18 +130,18 @@ cat >/usr/local/sbin/raid-tuning.sh <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# RAID-масив(ы): ставим 4 MiB
|
||||
for f in /sys/block/md*/queue/read_ahead_kb; do
|
||||
[ -e "$f" ] && echo 4096 > "$f"
|
||||
done
|
||||
# RAID массив(ы): 4 MiB readahead
|
||||
[ -e /sys/block/md3/queue/read_ahead_kb ] && echo 4096 > /sys/block/md3/queue/read_ahead_kb
|
||||
|
||||
# NVMe-диски: ставим 2 MiB
|
||||
for f in /sys/block/nvme*n1/queue/read_ahead_kb; do
|
||||
[ -e "$f" ] && echo 2048 > "$f"
|
||||
# NVMe диски: 2 MiB readahead + scheduler=none
|
||||
for d in /sys/block/nvme*n1; do
|
||||
[ -e "$d/queue/read_ahead_kb" ] && echo 2048 > "$d/queue/read_ahead_kb"
|
||||
[ -e "$d/queue/scheduler" ] && echo none > "$d/queue/scheduler"
|
||||
done
|
||||
EOF
|
||||
|
||||
chmod +x /usr/local/sbin/raid-tuning.sh
|
||||
|
||||
```
|
||||
|
||||
|
||||
@@ -173,7 +149,8 @@ chmod +x /usr/local/sbin/raid-tuning.sh
|
||||
cat >/etc/systemd/system/raid-tuning.service <<'EOF'
|
||||
[Unit]
|
||||
Description=RAID/NVMe tuning
|
||||
After=local-fs.target
|
||||
After=mdadm-raid.service local-fs.target
|
||||
Wants=mdadm-raid.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
@@ -183,11 +160,13 @@ RemainAfterExit=yes
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
```
|
||||
|
||||
```bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now raid-tuning.service
|
||||
systemctl enable --now fstrim.timer
|
||||
systemctl status raid-tuning.service --no-pager
|
||||
```
|
||||
```bash
|
||||
|
||||
Reference in New Issue
Block a user