Linux服務器的網卡的優化

通常情況下，Linux的網卡中斷是由一個CPU核心來處理的，當承擔高流量的場景下，會出現一些詭異的情況（網卡尚未達到瓶頸，但是卻出現丟包的情況）

這種時候，我們最好看下網卡中斷是不是缺少調優。

優化3要點：網卡多隊列+irq affinity親緣性設置+關閉irqbalance

目前阿里云官方提供的centos和ubuntu鏡像里面，已經自帶了優化腳本，內容如下:

centos7的腳本路徑在? /usr/sbin/ecs_mq_rps_rfs? 具體內容如下：

#!/bin/bash
#?This?is?the?default?setting?of?networking?multiqueue?and?irq?affinity
#?1.?enable?multiqueue?if?available
#?2.?irq?affinity?optimization
#?3.?stop?irqbalance?service
#?set?and?check?multiqueue

function?set_check_multiqueue()
{
????eth=$1
????log_file=$2
????queue_num=$(ethtool?-l?$eth?|?grep?-ia5?'pre-set'?|?grep?-i?combined?|?awk?{'print?$2'})
????if?[?$queue_num?-gt?1?];?then
????????#?set?multiqueue
????????ethtool?-L?$eth?combined?$queue_num
????????#?check?multiqueue?setting
????????cur_q_num=$(ethtool?-l?$eth?|?grep?-iA5?current?|?grep?-i?combined?|?awk?{'print?$2'})
????????if?[?"X$queue_num"?!=?"X$cur_q_num"?];?then
????????????echo?"Failed?to?set?$eth?queue?size?to?$queue_num"?>>?$log_file
????????????echo?"after?setting,?pre-set?queue?num:?$queue_num?,?current:?$cur_q_num"?>>?$log_file
????????????return?1
????????else
????????????echo?"OK.?set?$eth?queue?size?to?$queue_num"?>>?$log_file
????????fi
????else
????????echo?"only?support?$queue_num?queue;?no?need?to?enable?multiqueue?on?$eth"?>>?$log_file
????fi
}

#set?irq?affinity
function?set_irq_smpaffinity()
{
????log_file=$1
????node_dir=/sys/devices/system/node
????for?i?in?$(ls?-d?$node_dir/node*);?do
????????i=${i/*node/}
????done
????
????echo?"max?node?:$i"?>>?$log_file
????node_cpumax=$(cat?/sys/devices/system/node/node${i}/cpulist?|awk?-F-?'{print?$NF}')
????irqs=($(cat?/proc/interrupts?|grep?virtio?|grep?put?|?awk?-F:?'{print?$1}'))
????core=0
????for?irq?in?${irqs[@]};do
????????VEC=$core
????????if?[?$VEC?-ge?32?];then
????????????let?"IDX?=?$VEC?/?32"
????????????MASK_FILL=""
????????????MASK_ZERO="00000000"
????????????for?((i=1;?i<=$IDX;i++))
????????????????do
????????????????????MASK_FILL="${MASK_FILL},${MASK_ZERO}"
????????????????done
????????????let?"VEC?-=?32?*?$IDX"
????????????MASK_TMP=$((1<<$VEC))
????????????MASK=$(printf?"%X%s"?$MASK_TMP?$MASK_FILL)
????????else
????????????MASK_TMP=$((1<<$VEC))
????????????MASK=$(printf?"%X"?$MASK_TMP)
????????fi
????????echo?$MASK?>?/proc/irq/$irq/smp_affinity
????????echo?"mask:$MASK,?irq:$irq"?>>?$log_file
????????core=$(((core+1)%(node_cpumax+1)))
????done
}

#?stop?irqbalance?service
function?stop_irqblance()
{
????log_file=$1
????ret=0
????if?[?"X"?!=?"X$(ps?-ef?|?grep?irqbalance?|?grep?-v?grep)"?];?then
????????if?which?systemctl;then
????????????systemctl?stop?irqbalance
????????else
????????????service?irqbalance?stop
????????fi
????????if?[?$??-ne?0?];?then
????????????echo?"Failed?to?stop?irqbalance"?>>?$log_file
????????????ret=1
????????fi
????else
???????echo?"OK.?irqbalance?stoped."?>>?$log_file
????fi
????return?$ret
}
#?main?logic
function?main()
{
????ecs_network_log=/var/log/ecs_network_optimization.log
????ret_value=0
????echo?"running?$0"?>?$ecs_network_log
????echo?"========??ECS?network?setting?starts?$(date?+'%Y-%m-%d?%H:%M:%S')?========"?>>?$ecs_network_log
????#?we?assume?your?NIC?interface(s)?is/are?like?eth*
????eth_dirs=$(ls?-d?/sys/class/net/eth*)
????if?[?"X$eth_dirs"?=?"X"?];?then
????????echo?"ERROR!?can?not?find?any?ethX?in?/sys/class/net/?dir."?>>?$ecs_network_log
????????ret_value=1
????fi
????for?i?in?$eth_dirs
????do
????????cur_eth=$(basename?$i)
????????echo?"optimize?network?performance:?current?device?$cur_eth"?>>?$ecs_network_log
????????#?only?optimize?virtio_net?device
????????driver=$(basename?$(readlink?$i/device/driver))
????????if?!?echo?$driver?|?grep?-q?virtio;?then
????????????echo?"ignore?device?$cur_eth?with?driver?$driver"?>>?$ecs_network_log
????????????continue
????????fi
????????echo?"set?and?check?multiqueue?on?$cur_eth"?>>?$ecs_network_log
????????set_check_multiqueue?$cur_eth?$ecs_network_log
????????if?[?$??-ne?0?];?then
????????????echo?"Failed?to?set?multiqueue?on?$cur_eth"?>>?$ecs_network_log
????????????ret_value=1
????????fi
????done
????stop_irqblance??$ecs_network_log
????set_irq_smpaffinity?$ecs_network_log
????echo?"========??ECS?network?setting?END?$(date?+'%Y-%m-%d?%H:%M:%S')??========"?>>?$ecs_network_log
????return?$ret_value
}


#?program?starts?here
main
exit?$?

查詢的rps綁定情況的腳本 get_rps.sh

#!/bin/bash
#?獲取當前rps情況
for?i?in?$(ls?/sys/class/net/eth0/queues/rx-*/rps_cpus);?do?
??echo?$i
??cat?$i
done

參考文檔:?

https://help.aliyun.com/knowledge_detail/52559.html? 【推薦閱讀】

https://www.jianshu.com/p/09bb5d5a72ba

https://help.aliyun.com/document_detail/25378.html#g6

https://tech.meituan.com/2018/03/16/redis-high-concurrency-optimization.html? 【推薦閱讀】