Date: Sun, 06 Sep 2015 22:15:01 -0300 From: Marcelo Gondim <gondim@bsdinfo.com.br> To: FreeBSD Net <freebsd-net@freebsd.org> Subject: Tuning for Intel X520-SR2 and Intel i350-T2 Message-ID: <55ECE515.3080104@bsdinfo.com.br>
next in thread | raw e-mail | index | archive | help
Hi all,
I have a router FreeBSD 10.2-RELEASE-p2 with one network card Intel
X520-SR2 and two cards Intel i350-T2.
My hardware:
- Motherboard Intel S2600COE [1]
- Dual Hexa Xeon: Intel(R) Xeon(R) CPU E5-2630 v2 @ 2.60GHz
- 18Gb ram
- Downstream traffic 4Gbps
Is there any tuning to reduce the use of network outages?
My top below:
last pid: 8196; load averages: 17.98, 17.52, 17.21 up 1+23:05:25 21:45:05
32 processes: 1 running, 31 sleeping
CPU 0: 0.4% user, 0.0% nice, 0.4% system, 37.0% interrupt, 62.2% idle
CPU 1: 0.0% user, 0.0% nice, 0.0% system, 50.4% interrupt, 49.6% idle
CPU 2: 0.8% user, 0.0% nice, 0.0% system, 46.9% interrupt, 52.4% idle
CPU 3: 0.0% user, 0.0% nice, 0.0% system, 65.7% interrupt, 34.3% idle
CPU 4: 0.0% user, 0.0% nice, 0.0% system, 62.6% interrupt, 37.4% idle
CPU 5: 0.0% user, 0.0% nice, 0.4% system, 63.0% interrupt, 36.6% idle
CPU 6: 0.0% user, 0.0% nice, 0.0% system, 63.8% interrupt, 36.2% idle
CPU 7: 0.4% user, 0.0% nice, 0.0% system, 33.1% interrupt, 66.5% idle
CPU 8: 0.0% user, 0.0% nice, 0.0% system, 64.6% interrupt, 35.4% idle
CPU 9: 0.4% user, 0.0% nice, 0.0% system, 33.5% interrupt, 66.1% idle
CPU 10: 0.4% user, 0.0% nice, 0.8% system, 36.6% interrupt, 62.2% idle
CPU 11: 0.8% user, 0.0% nice, 0.4% system, 31.9% interrupt, 66.9% idle
My sysctl.conf:
net.inet.ip.forwarding=1
net.inet.ip.fastforwarding=1
net.inet6.ip6.forwarding=1
kern.ipc.somaxconn=4096
net.inet.tcp.syncookies=1
net.inet.ip.redirect=1
net.inet.ip.accept_sourceroute=0
net.inet.ip.sourceroute=0
net.inet.tcp.drop_synfin=1
net.inet.udp.blackhole=1
net.inet.tcp.blackhole=2
security.bsd.see_other_uids=0
net.inet.ip.fw.dyn_buckets=65536
net.inet.ip.fw.dyn_max=65536
hw.intr_storm_threshold=9000
net.inet.ip.dummynet.pipe_slot_limit=800
net.inet.icmp.icmplim=2000
My loader.conf:
loader_logo="beastie"
if_lagg_load="YES"
speaker_load="YES"
aio_load="YES"
autoboot_delay="5"
net.fibs=1
kern.hz=2000
My firewall rules:
(root@rt01)[~]# ipfw show
00100 400 134880 allow ip from any to any via lo0
00200 0 0 deny ip from 127.0.0.0/8 to any
00300 0 0 deny ip from ::1 to any
00400 0 0 deny ip from any to 127.0.0.0/8
00500 0 0 deny ip from any to ::1
00600 7381651 499415987 deny ip from not table(3) to any in
recv vlan0
00700 4011900 222326864 deny ip from not table(3) to any in
recv vlan1
00800 1287607 84935306 deny ip from not table(3) to any in
recv vlan2
00900 2152212 128056358 deny ip from not table(3) to any in
recv vlan3
01000 0 0 deny ip from not table(3) to any in
recv vlan7
01100 0 0 deny ip from not table(3) to any in
recv vlan5
01200 0 0 deny ip from not table(3) to any in
recv vlan6
01300 0 0 deny ip from not table(3) to any in
recv vlan8
01400 0 0 check-state
01500 1543 96723 deny tcp from any to any in tcpflags
fin,psh,urg
01600 785 265717 deny tcp from any to any in tcpflags
!syn,!fin,!ack,!psh,!rst,!urg
01700 633 49047 deny tcp from any to any in tcpflags
syn,fin
01800 20620 828603 deny tcp from any to any in tcpflags
fin,rst
01900 0 0 deny ip from any to any in ipoptions
ssrr,lsrr,rr,ts
02000 0 0 deny ip from table(99) to any in via igb4
02100 0 0 deny ip from table(99) to any in via igb5
02200 75031 11544750 deny ip from table(99) to any in via igb1
02300 340077 17057941 deny ip from table(1) to any
02400 0 0 deny log logamount 100 ip from any to
table(2)
02500 21858 1661959 allow udp from 186.xxx.xxx.3 to me
dst-port 161 in via vlan0 keep-state
02600 0 0 allow tcp from 186.xxx.xxx.3 to me
dst-port 199 in via vlan0 setup keep-state
02700 100 8128 deny udp from any to me dst-port 161
02800 0 0 deny tcp from any to me dst-port 199
65535 158071018941 121039458717515 allow ip from any to any
My /etc/rc.local (cpu affinity):
#!/bin/sh
echo "SO4L16G>L8C." > /dev/speaker
/usr/bin/cpuset -l 0 /usr/local/sbin/bgpd
/usr/bin/cpuset -l 5 -x 300
/usr/bin/cpuset -l 4 -x 301
/usr/bin/cpuset -l 3 -x 302
/usr/bin/cpuset -l 2 -x 303
/usr/bin/cpuset -l 1 -x 304
/usr/bin/cpuset -l 0 -x 305
/usr/bin/cpuset -l 0 -x 306
/usr/bin/cpuset -l 1 -x 307
/usr/bin/cpuset -l 2 -x 308
/usr/bin/cpuset -l 3 -x 309
/usr/bin/cpuset -l 4 -x 310
/usr/bin/cpuset -l 5 -x 311
/usr/bin/cpuset -l 5 -x 312
/usr/bin/cpuset -l 4 -x 313
/usr/bin/cpuset -l 3 -x 314
/usr/bin/cpuset -l 2 -x 315
/usr/bin/cpuset -l 1 -x 316
/usr/bin/cpuset -l 0 -x 317
/usr/bin/cpuset -l 0 -x 318
/usr/bin/cpuset -l 1 -x 319
/usr/bin/cpuset -l 2 -x 320
/usr/bin/cpuset -l 3 -x 321
/usr/bin/cpuset -l 4 -x 322
/usr/bin/cpuset -l 5 -x 323
/usr/bin/cpuset -l 5 -x 324
/usr/bin/cpuset -l 4 -x 325
/usr/bin/cpuset -l 3 -x 326
/usr/bin/cpuset -l 2 -x 327
/usr/bin/cpuset -l 1 -x 328
/usr/bin/cpuset -l 0 -x 329
/usr/bin/cpuset -l 0 -x 330
/usr/bin/cpuset -l 1 -x 331
/usr/bin/cpuset -l 2 -x 332
/usr/bin/cpuset -l 3 -x 333
/usr/bin/cpuset -l 4 -x 334
/usr/bin/cpuset -l 5 -x 335
/usr/bin/cpuset -l 5 -x 264
/usr/bin/cpuset -l 4 -x 265
/usr/bin/cpuset -l 3 -x 266
/usr/bin/cpuset -l 2 -x 267
/usr/bin/cpuset -l 1 -x 268
/usr/bin/cpuset -l 0 -x 269
/usr/bin/cpuset -l 0 -x 270
/usr/bin/cpuset -l 1 -x 271
/usr/bin/cpuset -l 2 -x 272
/usr/bin/cpuset -l 3 -x 273
/usr/bin/cpuset -l 4 -x 274
/usr/bin/cpuset -l 5 -x 275
/usr/bin/cpuset -l 5 -x 276
/usr/bin/cpuset -l 4 -x 277
/usr/bin/cpuset -l 3 -x 278
/usr/bin/cpuset -l 2 -x 279
/usr/bin/cpuset -l 1 -x 280
/usr/bin/cpuset -l 0 -x 281
/usr/bin/cpuset -l 0 -x 282
/usr/bin/cpuset -l 1 -x 283
/usr/bin/cpuset -l 2 -x 284
/usr/bin/cpuset -l 3 -x 285
/usr/bin/cpuset -l 4 -x 286
/usr/bin/cpuset -l 5 -x 287
/usr/bin/cpuset -l 5 -x 288
/usr/bin/cpuset -l 4 -x 289
/usr/bin/cpuset -l 3 -x 290
/usr/bin/cpuset -l 2 -x 291
/usr/bin/cpuset -l 1 -x 292
/usr/bin/cpuset -l 0 -x 293
/usr/bin/cpuset -l 0 -x 294
/usr/bin/cpuset -l 1 -x 295
/usr/bin/cpuset -l 2 -x 296
/usr/bin/cpuset -l 3 -x 297
/usr/bin/cpuset -l 4 -x 298
/usr/bin/cpuset -l 5 -x 299
/usr/bin/cpuset -l 11 -x 337
/usr/bin/cpuset -l 10 -x 338
/usr/bin/cpuset -l 9 -x 339
/usr/bin/cpuset -l 8 -x 340
/usr/bin/cpuset -l 7 -x 341
/usr/bin/cpuset -l 6 -x 342
/usr/bin/cpuset -l 6 -x 343
/usr/bin/cpuset -l 8 -x 344
/usr/bin/cpuset -l 8 -x 345
/usr/bin/cpuset -l 9 -x 346
/usr/bin/cpuset -l 10 -x 347
/usr/bin/cpuset -l 11 -x 348
/usr/bin/cpuset -l 11 -x 349
/usr/bin/cpuset -l 10 -x 350
/usr/bin/cpuset -l 9 -x 351
/usr/bin/cpuset -l 8 -x 352
/usr/bin/cpuset -l 7 -x 353
/usr/bin/cpuset -l 6 -x 354
ix0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=8407bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWTSO>
ether a0:36:9f:2a:6d:ac
inet6 fe80::a236:9fff:fe2a:6dac%ix0 prefixlen 64 scopeid 0x9
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet autoselect (10Gbase-LR
<full-duplex,rxpause,txpause>)
status: active
ix1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=8407bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWTSO>
ether a0:36:9f:2a:6d:ae
inet6 fe80::a236:9fff:fe2a:6dae%ix1 prefixlen 64 scopeid 0xa
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet autoselect (10Gbase-SR
<full-duplex,rxpause,txpause>)
status: active
igb4: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=403bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,VLAN_HWTSO>
ether 00:1b:21:7b:ee:6c
inet6 fe80::21e:67ff:fe9b:3e1%igb4 prefixlen 64 scopeid 0x5
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
igb5: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=403bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,VLAN_HWTSO>
ether 00:1b:21:7b:ee:6c
inet6 fe80::21e:67ff:fe9b:3e1%igb5 prefixlen 64 scopeid 0x6
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
igb6: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=403bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,VLAN_HWTSO>
ether 00:1b:21:7b:ee:98
inet6 fe80::21e:67ff:fe9b:3e1%igb6 prefixlen 64 scopeid 0x7
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
igb7: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=403bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,VLAN_HWTSO>
ether 00:1b:21:7b:ee:98
inet6 fe80::21e:67ff:fe9b:3e1%igb7 prefixlen 64 scopeid 0x8
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
[1]
http://download.intel.com/support/motherboards/server/s2600co/sb/g42278004_s2600co_tps_rev171.pdf
Thanks and best regards,
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?55ECE515.3080104>