- 配置大页内存
1.修改grub默认配置文件
[root@compute-10e8e74e95 ~]# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL="serial console"
GRUB_SERIAL_COMMAND="serial --speed=115200"
GRUB_CMDLINE_LINUX="console=tty0 pci=realloc biosdevname=0 net.ifnames=0 console=ttyS0,115200n8 crashkernel=512M iommu=pt intel_iommu=on isolcpus=92,93,94,95,124,125,126,127 default_hugepagesz=1G hugepagesz=1G hugepages=900 irqaffinity=0-91,96-123 nohz_full=92,93,94,95,124,125,126,127"
GRUB_DISABLE_RECOVERY="true"
备注：添加红色字体部分，注意需要配置numactl –H，并且预留相应的cpu号给ovs-dpdk使用
[root@compute-10e8e74e95 ~]# numactl -H
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
node 0 size: 515100 MB
node 0 free: 47792 MB
node 1 cpus: 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
node 1 size: 515929 MB
node 1 free: 47739 MB
node distances:
node 0 1
0: 10 20
1: 20 10
2.生成启动grub.conf文件
执行命令：grub2-mkconfig -o /boot/grub2/grub.cfg
3.重启物理服务器
执行命令reboot
- 清理环境
- 删除kernel ovs相关的配置
删除相关的port（根据实际情况进行配置），执行命令：
ovs-vsctl del-port br-ex bond1
ovs-vsctl del-port phy-br-ex
ovs-vsctl del-port br-tun patch-int
ovs-vsctl del-port br-int int-br-ex
ovs-vsctl del-port br-int patch-tun
删除相关的（bridge根据实际情况进行配置）,执行命令：
ovs-vsctl del-br br-tun
ovs-vsctl del-br br-ex
ovs-vsctl del-br br-int
- 安装dpdk包
- 查看dpdk支持的版本
[root@compute-10e8e74e95 ~]# ovs-vsctl list open_vswitch
_uuid : f4fee592-ee08-4bc4-8024-1885b4c447ef
bridges : [2c7d97c6-bab9-418c-ab97-f11facfa3356, d2905fcd-742f-4d20-80e6-be920ccd4a9b, e3db88e8-62df-4d47-93ca-460e5ea8c213]
cur_cfg : 2801
datapath_types : [netdev, system]
datapaths : {}
db_version : "8.2.0"
dpdk_initialized : true
dpdk_version : "DPDK 20.11.0"
external_ids : {hostname=compute-10e8e74e95, rundir="/var/run/openvswitch", system-id="09b01c8a-b4f5-436e-a491-d7fa51623ef2"}
iface_types : [dpdk, dpdkvhostuser, dpdkvhostuserclient, erspan, geneve, gre, gtpu, internal, ip6erspan, ip6gre, lisp, patch, stt, system, tap, vxlan]
manager_options : [d9bd573f-0096-4d74-837c-e432e7b9b45b]
next_cfg : 2801
other_config : {dpdk-init="true", dpdk-socket-mem="4096,4096", pmd-cpu-mask=f0000000f00000000000000000000000, pmd-rxq-assign=roundrobin}
ovs_version : "2.14.1.4"
ssl : []
statistics : {}
system_type : ctyunos
system_version : "2.0.1"
- 安装对应的dpdk包
yum install dpdk-20.11-8.ctl2.x86_64 –y
- 配置dpdk-ovs
- 配置open_vswitch
执行命令：
ovs-vsctl set open_vswitch . dpdk_initialized=true
ovs-vsctl set open_vswitch . other_config:dpdk-init=true
ovs-vsctl set open_vswitch . other_config:dpdk-socket-mem=4096,4096
ovs-vsctl set open_vswitch . other_config:pmd-cpu-mask=f0000000f00000000000000000000000 (需要根据具体情况进行配置)
ovs-vsctl set open_vswitch . other_config:pmd-rxq-assign=roundrobin
- 配置netdev桥以及之间patch
执行命令：
ovs-vsctl add-br br-int -- set bridge br-int datapath_type=netdev
ovs-vsctl add-br br-ex -- set bridge br-ex datapath_type=netdev
ovs-vsctl add-br br-tun -- set bridge br-tun datapath_type=netdev
ovs-vsctl add-port br-int int-br-ex -- set Interface int-br-ex type=patch -- set Interface int-br-ex options:peer=phy-br-ex
ovs-vsctl add-port br-ex phy-br-ex -- set Interface phy-br-ex type=patch -- set Interface phy-br-ex options:peer=int-br-ex
ovs-vsctl add-port br-tun patch-tun -- set Interface patch-tun type=patch -- set Interface patch-tun options:peer=patch-int
ovs-vsctl add-port br-int patch-int -- set Interface patch-int type=patch -- set Interface patch-int options:peer=patch-tun
- 配置网卡dpdk口
modprobe vfio-pci
dpdk-devbind.py --bind=vfio-pci eth0
dpdk-devbind.py --bind=vfio-pci eth3
备注：eth0和eth3需要在交换机那里配置聚合链路口
查看dpdk接管的网卡
[root@compute-10e8e74e95 ~]# dpdk-devbind.py --status
Network devices using DPDK-compatible driver
============================================
0000:4b:00.0 'Ethernet Controller E810-XXV for SFP 159b' drv=vfio-pci unused=ice
0000:ca:00.1 'Ethernet Controller E810-XXV for SFP 159b' drv=vfio-pci unused=ice
Network devices using kernel driver
===================================
0000:4b:00.1 'Ethernet Controller E810-XXV for SFP 159b' if=eth1 drv=ice unused=vfio-pci
0000:b1:00.0 'I350 Gigabit Network Connection 1521' if=ens0 drv=igb unused=vfio-pci
0000:b1:00.1 'I350 Gigabit Network Connection 1521' if=ens1 drv=igb unused=vfio-pci
0000:ca:00.0 'Ethernet Controller E810-XXV for SFP 159b' if=eth2 drv=ice unused=vfio-pci
No 'Baseband' devices detected
==============================
No 'Crypto' devices detected
============================
No 'Eventdev' devices detected
==============================
No 'Mempool' devices detected
=============================
No 'Compress' devices detected
==============================
Misc (rawdev) devices using kernel driver
=========================================
0000:00:01.0 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.1 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.2 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.3 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.4 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.5 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.6 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:00:01.7 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.0 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.1 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.2 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.3 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.4 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.5 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.6 'Device 0b00' drv=ioatdma unused=vfio-pci
0000:80:01.7 'Device 0b00' drv=ioatdma unused=vfio-pci
No 'Regex' devices detected
===========================
配置dpdkbond0口，执行命令：
ovs-vsctl add-bond br-ex dpdkbond0 eth0 eth3 lacp=active bond_mode=balance-tcp -- set interface eth0 type=dpdk -- set interface eth0 options:dpdk-devargs=0000:4b:00.0 -- set interface eth3 type=dpdk -- set interface eth3 options:dpdk-devargs=0000:ca:00.1
配置mtu值9000
ovs-vsctl set interface eth0 mtu_request=9000
ovs-vsctl set interface eth3 mtu_request=9000
配置网卡的接收队列12
ovs-vsctl set interface eth0 options:n_rxq=12
ovs-vsctl set interface eth3 options:n_rxq=12
- 重启openvswitch
修改openvswitch配置
如果是kolla部署的环境，libvirtd服务是用容器，则如下修改：
vim /etc/sysconfig/openvswitch
#OVS_USER_ID="openvswitch:hugetlbfs"
OVS_USER_ID="root:root"
执行命令：
systemctl restart openvswitch.service
5.查看ovs信息
[root@compute-10e8e74e95 ~]# ovs-vsctl show
f4fee592-ee08-4bc4-8024-1885b4c447ef
Manager "ptcp:6640:127.0.0.1"
is_connected: true
Bridge br-tun
Controller "tcp:127.0.0.1:6633"
is_connected: true
fail_mode: secure
datapath_type: netdev
Port br-tun
Interface br-tun
type: internal
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Bridge br-ex
Controller "tcp:127.0.0.1:6633"
is_connected: true
fail_mode: secure
datapath_type: netdev
Port phy-br-ex
Interface phy-br-ex
type: patch
options: {peer=int-br-ex}
Port dpdkbond0
Interface eth0
type: dpdk
options: {dpdk-devargs="0000:4b:00.0", n_rxq="12"}
Interface eth3
type: dpdk
options: {dpdk-devargs="0000:ca:00.1", n_rxq="12"}
备注：这种情况表示配置成功了
Port br-ex
Interface br-ex
type: internal
6.查看聚合链路是否正常
[root@compute-10e8e74e95 ~]# ovs-appctl lacp/show
---- dpdkbond0 ----
status: active negotiated
sys_id: b4:05:5d:d0:72:58
sys_priority: 65534
aggregation key: 1
lacp_time: slow
slave: eth0: current attached
port_id: 1
port_priority: 65535
may_enable: true
actor sys_id: b4:05:5d:d0:72:58
actor sys_priority: 65534
actor port_id: 1
actor port_priority: 65535
actor key: 1
actor state: activity aggregation synchronized collecting distributing
partner sys_id: b8:45:f4:82:f6:dc
partner sys_priority: 123
partner port_id: 32769
partner port_priority: 32768
partner key: 40011
partner state: activity timeout aggregation synchronized collecting distributing
slave: eth3: current attached
port_id: 2
port_priority: 65535
may_enable: true
actor sys_id: b4:05:5d:d0:72:58
actor sys_priority: 65534
actor port_id: 2
actor port_priority: 65535
actor key: 1
actor state: activity aggregation synchronized collecting distributing
partner sys_id: b8:45:f4:82:f6:dc
partner sys_priority: 123
partner port_id: 16385
partner port_priority: 32768
partner key: 40011
partner state: activity timeout aggregation synchronized collecting distributing
如果有问题，请查看交换机的配置是否正确
7.查看bond口
[root@compute-10e8e74e95 ~]# ovs-appctl bond/show dpdkbond0
---- dpdkbond0 ----
bond_mode: balance-tcp
bond may use recirculation: yes, Recirc-ID : 1
bond_arp_copy: false
bond-hash-basis: 0
lb_output action: disabled, bond-id: -1
updelay: 0 ms
downdelay: 0 ms
next rebalance: 5736 ms
lacp_status: negotiated
lacp_fallback_ab: false
active-backup primary: <none>
active slave mac: b4:05:5d:d0:72:58(eth0)
slave eth0: enabled
active slave
may_enable: true
slave eth3: enabled
active slave
may_enable: true
查看ovs使用cpu情况
[root@compute-10e8e74e95 ~]# ovs-appctl dpif-netdev/pmd-perf-show
Time: 01:46:58.382
Measurement duration: 156612.175 s
pmd thread numa_id 0 core_id 92:
Iterations: 687901846489 (0.23 us/it)
- Used TSC cycles: 468721445401888 ( 99.8 % of total cycles)
- idle iterations: 685138121634 ( 93.4 % of used cycles)
- busy iterations: 2763724855 ( 6.6 % of used cycles)
Rx packets: 4365121188 (28 Kpps, 7081 cycles/pkt)
Datapath passes: 4370882589 (1.00 passes/pkt)
- EMC hits: 3852919131 ( 88.1 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 516309786 ( 11.8 %, 1.01 subtbl lookups/hit)
- Upcalls: 1653672 ( 0.0 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 4365488327 (28 Kpps)
Tx batches: 4016841843 (1.09 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.185 s
pmd thread numa_id 0 core_id 93:
Iterations: 590242374877 (0.26 us/it)
- Used TSC cycles: 468721442384642 ( 99.8 % of total cycles)
- idle iterations: 587466990004 ( 93.4 % of used cycles)
- busy iterations: 2775384873 ( 6.6 % of used cycles)
Rx packets: 4368011851 (28 Kpps, 7035 cycles/pkt)
Datapath passes: 4373789983 (1.00 passes/pkt)
- EMC hits: 3848751863 ( 88.0 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 523395907 ( 12.0 %, 1.01 subtbl lookups/hit)
- Upcalls: 1642213 ( 0.0 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 4368067315 (28 Kpps)
Tx batches: 4010582972 (1.09 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.173 s
pmd thread numa_id 0 core_id 94:
Iterations: 688925161221 (0.23 us/it)
- Used TSC cycles: 468721441450464 ( 99.8 % of total cycles)
- idle iterations: 686127752507 ( 93.5 % of used cycles)
- busy iterations: 2797408714 ( 6.5 % of used cycles)
Rx packets: 4306887781 (28 Kpps, 7034 cycles/pkt)
Datapath passes: 4313267300 (1.00 passes/pkt)
- EMC hits: 3773910753 ( 87.5 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 537658041 ( 12.5 %, 1.01 subtbl lookups/hit)
- Upcalls: 1698506 ( 0.0 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 4306926564 (28 Kpps)
Tx batches: 4023369698 (1.07 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.166 s
pmd thread numa_id 0 core_id 95:
Iterations: 600696286327 (0.26 us/it)
- Used TSC cycles: 468721443403624 ( 99.8 % of total cycles)
- idle iterations: 597921539416 ( 93.5 % of used cycles)
- busy iterations: 2774746911 ( 6.5 % of used cycles)
Rx packets: 4307936500 (28 Kpps, 7041 cycles/pkt)
Datapath passes: 4314021464 (1.00 passes/pkt)
- EMC hits: 3793785586 ( 87.9 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 518551709 ( 12.0 %, 1.01 subtbl lookups/hit)
- Upcalls: 1684169 ( 0.0 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 4308018484 (28 Kpps)
Tx batches: 3995217997 (1.08 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.171 s
pmd thread numa_id 1 core_id 124:
Iterations: 1067364952889 (0.15 us/it)
- Used TSC cycles: 468720996929166 ( 99.8 % of total cycles)
- idle iterations: 1066025018080 ( 97.1 % of used cycles)
- busy iterations: 1339934809 ( 2.9 % of used cycles)
Rx packets: 1874342424 (12 Kpps, 7153 cycles/pkt)
Datapath passes: 1879581564 (1.00 passes/pkt)
- EMC hits: 1623868611 ( 86.4 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 254295797 ( 13.5 %, 1.01 subtbl lookups/hit)
- Upcalls: 1417153 ( 0.1 %, 0.0 us/upcall)
- Lost upcalls: 3 ( 0.0 %)
Tx packets: 1874530666 (12 Kpps)
Tx batches: 1750862409 (1.07 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.184 s
pmd thread numa_id 1 core_id 125:
Iterations: 1078397846281 (0.14 us/it)
- Used TSC cycles: 468721001105724 ( 99.8 % of total cycles)
- idle iterations: 1077040535555 ( 97.2 % of used cycles)
- busy iterations: 1357310726 ( 2.8 % of used cycles)
Rx packets: 1846415002 (12 Kpps, 7022 cycles/pkt)
Datapath passes: 1851233119 (1.00 passes/pkt)
- EMC hits: 1614542364 ( 87.2 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 235243950 ( 12.7 %, 1.02 subtbl lookups/hit)
- Upcalls: 1446805 ( 0.1 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 1848099243 (12 Kpps)
Tx batches: 1748242620 (1.06 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.169 s
pmd thread numa_id 1 core_id 126:
Iterations: 1032599992429 (0.15 us/it)
- Used TSC cycles: 468720998337698 ( 99.8 % of total cycles)
- idle iterations: 1031236093859 ( 97.3 % of used cycles)
- busy iterations: 1363898570 ( 2.7 % of used cycles)
Rx packets: 1839851583 (12 Kpps, 6917 cycles/pkt)
Datapath passes: 1844521384 (1.00 passes/pkt)
- EMC hits: 1618153312 ( 87.7 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 225017169 ( 12.2 %, 1.02 subtbl lookups/hit)
- Upcalls: 1350903 ( 0.1 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 1839966499 (12 Kpps)
Tx batches: 1745505805 (1.05 pkts/batch)
Time: 01:46:58.382
Measurement duration: 156612.172 s
pmd thread numa_id 1 core_id 127:
Iterations: 1056055251335 (0.15 us/it)
- Used TSC cycles: 468720998410240 ( 99.8 % of total cycles)
- idle iterations: 1054678217267 ( 97.2 % of used cycles)
- busy iterations: 1377034068 ( 2.8 % of used cycles)
Rx packets: 1904366345 (12 Kpps, 6932 cycles/pkt)
Datapath passes: 1909025115 (1.00 passes/pkt)
- EMC hits: 1668131643 ( 87.4 %)
- SMC hits: 0 ( 0.0 %)
- Megaflow hits: 239545762 ( 12.5 %, 1.01 subtbl lookups/hit)
- Upcalls: 1347710 ( 0.1 %, 0.0 us/upcall)
- Lost upcalls: 0 ( 0.0 %)
Tx packets: 1904401441 (12 Kpps)
Tx batches: 1781037852 (1.07 pkts/batch)
- Nova-compute配置修改
vim /etc/kolla/nova-compute/nova.conf
vcpu_pin_set = 0-91,96-123 (根据实际情况进行配置)
reserved_huge_pages=node:0,size:1048576,count:50 (根据实际情况进行配置)
reserved_huge_pages=node:1,size:1048576,count:50 (根据实际情况进行配置)
添加到Default组后面，如果前面有此字段的配置，就进行相应的修改
然后重启nova-compute
docker restart nova_compute
- Neutron-openvswitch-agent配置修改
vim /etc/kolla/neutron-openvswitch-agent/ml2_conf.ini
[ovs]
bridge_mappings = physnet1:br-ex
datapath_type = netdev
ovsdb_connection = tcp:127.0.0.1:6640
local_ip = 10.8.74.95
然后重启neutron-openvswitch-agent
docker restart neutron_openvswitch_agent
- Nova-libvirt配置修改
如果是容器启动的libvirt，修改如下：
vim /etc/kolla/nova-libvirt/qemu.conf
user = "nova"
group = "root"
然后重启nova-libvirt
Docker restart nova_libvirt
- 创建云主机
- 修改flavor属性，执行命令
openstack flavor set 4c16g --property hw:mem_page_size='large'
- 创建云主机，执行命令
openstack server create --network 5021390a-40af-4743-88e6-512e87037e82 --image de97f4c6-6af4-4179-b791-de95350b250d --flavor 4c16g --availability-zone nova:compute-10e8e74e95 --min 0 --max 1 test1
