Date: Thu, 11 Jul 2013 20:32:23 +0300 From: "isp" <mline@ukr.net> To: freebsd-net@freebsd.org Subject: FreeBSD router problems Message-ID: <16706.1373563943.8842093075575209984@ffe17.ukr.net>
next in thread | raw e-mail | index | archive | help
Hi! I have a problem with my FreeBSD router, I can't get more than 1 Gbps
throught it, but I have 2 Gbps LAGG on it. There are only 27 IPFW rules
(NAT+Shaping). IPoE only.
lagg0 (VLAN's + shaping) - two 'igb' adapters
lagg1 (NAT, tso if off) - two 'em' adapters
I tried to switch off dummynet, but it doesn't helps.
# uname -a
[code]FreeBSD router 9.1-RELEASE-p3 FreeBSD 9.1-RELEASE-p3 #0: Tue Apr 30
20:02:00 EEST 2013 root@south:/usr/obj/usr/src/sys/ROUTER amd64
# top -aSPHI
last pid: 91712; load averages: 2.18, 2.06,
1.97
up 20+22:28:36 17:40:22
120 processes: 7 running, 87 sleeping, 26 waiting
CPU 0: 0.0% user, 0.0% nice, 1.6% system, 38.6% interrupt, 59.8% idle
CPU 1: 0.0% user, 0.0% nice, 7.1% system, 37.0% interrupt, 55.9% idle
CPU 2: 0.0% user, 0.0% nice, 3.9% system, 38.6% interrupt, 57.5% idle
CPU 3: 0.0% user, 0.0% nice, 15.7% system, 26.8% interrupt, 57.5% idle
Mem: 59M Active, 1102M Inact, 942M Wired, 800M Buf, 5529M Free
Swap: 16G Total, 16G Free
PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAND
12 root -72 - 0K 448K RUN 1 153:39 72.22% [intr{swi1:
netisr 0}]
11 root 155 ki31 0K 64K RUN 1 494.2H 65.19% [idle{idle:
cpu1}]
11 root 155 ki31 0K 64K CPU2 2 494.3H 64.65% [idle{idle:
cpu2}]
11 root 155 ki31 0K 64K RUN 0 493.3H 63.38% [idle{idle:
cpu0}]
11 root 155 ki31 0K 64K CPU3 3 496.4H 62.55% [idle{idle:
cpu3}]
12 root -92 - 0K 448K WAIT 2 58:49 9.38% [intr{irq266:
igb0:que}]
12 root -92 - 0K 448K WAIT 2 59:32 9.03% [intr{irq271:
igb1:que}]
12 root -92 - 0K 448K CPU1 1 59:09 8.94% [intr{irq265:
igb0:que}]
12 root -92 - 0K 448K WAIT 3 57:52 8.01% [intr{irq272:
igb1:que}]
12 root -92 - 0K 448K WAIT 1 59:32 7.96% [intr{irq270:
igb1:que}]
12 root -92 - 0K 448K WAIT 3 55:47 7.81% [intr{irq267:
igb0:que}]
12 root -92 - 0K 448K WAIT 0 55:24 7.23% [intr{irq264:
igb0:que}]
12 root -92 - 0K 448K WAIT 0 56:57 6.69% [intr{irq269:
igb1:que}]
12 root -92 - 0K 448K WAIT 3 203:34 4.74% [intr{irq275:
em1:rx 0}]
0 root -92 0 0K 336K - 2 427:03 2.64%
[kernel{dummynet}]
0 root -92 0 0K 336K - 3 206:57 2.54% [kernel{em0
que}]
86278 root 20 0 33348K 8588K select 0 8:35 0.54%
/usr/local/sbin/snmpd -p /var/run/net_snmpd.pid -r
12 root -92 - 0K 448K WAIT 2 7:56 0.20% [intr{irq276:
em1:tx 0}]
# cat /etc/sysctl.conf
dev.igb.0.rx_processing_limit=4096
dev.igb.1.rx_processing_limit=4096
dev.em.0.rx_int_delay=200
dev.em.0.tx_int_delay=200
dev.em.0.rx_abs_int_delay=4000
dev.em.0.tx_abs_int_delay=4000
dev.em.0.rx_processing_limit=4096
dev.em.1.rx_int_delay=200
dev.em.1.tx_int_delay=200
dev.em.1.rx_abs_int_delay=4000
dev.em.1.tx_abs_int_delay=4000
dev.em.1.rx_processing_limit=4096
net.inet.ip.forwarding=1
net.inet.ip.fastforwarding=1
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=0
net.inet.ip.redirect=0
net.inet.tcp.delayed_ack=0
net.inet.tcp.recvbuf_max=4194304
net.inet.tcp.sendbuf_max=4194304
net.inet.tcp.sack.enable=0
net.inet.tcp.drop_synfin=1
net.inet.tcp.nolocaltimewait=1
net.inet.ip.ttl=255
net.inet.ip.sourceroute=0
net.inet.ip.accept_sourceroute=0
net.inet.udp.recvspace=64080
net.inet.ip.rtmaxcache=1024
net.inet.ip.intr_queue_maxlen=5120
kern.ipc.nmbclusters=824288
kern.ipc.maxsockbuf=83886080
kern.ipc.maxsockets=102400
net.inet.tcp.recvspace=95536
net.inet.tcp.sendspace=95536
net.local.stream.recvspace=32768
net.local.stream.sendspace=32768
kern.ipc.somaxconn=32768
net.inet.tcp.maxtcptw=65535
net.inet.ip.fw.one_pass=1
net.inet.ip.fw.dyn_max=65535
net.inet.ip.fw.dyn_buckets=2048
net.inet.ip.fw.dyn_syn_lifetime=10
net.inet.ip.fw.dyn_ack_lifetime=120
net.inet.ip.fw.verbose=0
net.inet.ip.dummynet.io_fast=1
net.inet.ip.dummynet.hash_size=65536
net.inet.ip.dummynet.pipe_slot_limit=1000
net.inet.icmp.icmplim=3000
net.inet.icmp.drop_redirect=1
net.inet.icmp.log_redirect=0
net.inet.icmp.bmcastecho=0
net.inet.icmp.maskrepl=0
kern.random.sys.harvest.ethernet=0
kern.random.sys.harvest.point_to_point=0
kern.random.sys.harvest.interrupt=0
net.inet.raw.maxdgram=16384
net.inet.raw.recvspace=16384
net.route.netisr_maxqlen=8192
net.inet.ip.intr_queue_maxlen=10240
net.isr.dispatch=deferred
# cat /boot/loader.conf
loader_logo="beastie"
autoboot_delay=3
geom_mirror_load="YES"
hw.igb.rxd=4096
hw.igb.txd=4096
hw.igb.rx_process_limit=4096
hw.igb.max_interrupt_rate=32000
hw.igb.num_queues=4
hw.igb.fc_setting=0
hw.igb.lro=0
hw.em.rxd=4096
hw.em.txd=4096
hw.em.rx_process_limit=4096
hw.em.fc_setting=0
dev.em.0.rx_int_delay=200
dev.em.0.tx_int_delay=200
dev.em.0.rx_abs_int_delay=4000
dev.em.0.tx_abs_int_delay=4000
dev.em.1.rx_int_delay=200
dev.em.1.tx_int_delay=200
dev.em.1.rx_abs_int_delay=4000
dev.em.1.tx_abs_int_delay=4000
net.isr.maxthreads=4
net.isr.bindthreads=0
net.inet.tcp.tcbhashsize=32000
net.link.ifqmaxlen=10240
net.isr.defaultqlimit=8192
# vmstat -i
interrupt total rate
irq20: ehci1 4171628 2
irq21: atapci0 1561194 0
irq22: ehci0+ 2713150 1
cpu0:timer 14622957598 8082
irq264: igb0:que 0 515616328 284
irq265: igb0:que 1 738456087 408
irq266: igb0:que 2 711371660 393
irq267: igb0:que 3 462738813 255
irq268: igb0:link 3 0
irq269: igb1:que 0 656044816 362
irq270: igb1:que 1 546931002 302
irq271: igb1:que 2 617173223 341
irq272: igb1:que 3 644295672 356
irq273: igb1:link 4 0
irq274: em0 557400132 308
irq275: em1:rx 0 424252744 234
irq276: em1:tx 0 708469817 391
irq277: em1:link 2 0
cpu3:timer 678408141 374
cpu1:timer 674674076 372
cpu2:timer 621495291 343
Total 23188731381 12816
# netstat -w1
input (Total) output
packets errs idrops bytes packets errs bytes colls
442k 0 0 304M 457k 0 393M 0
449k 0 0 308M 463k 0 395M 0
445k 0 0 304M 461k 0 393M 0
439k 0 0 303M 456k 0 393M 0
434k 0 0 297M 450k 0 387M 0
440k 0 0 301M 456k 0 392M 0
438k 0 0 300M 455k 0 391M 0
# ifconfig lagg0 (internal, 500 VLAN's)
lagg0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=401bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,VLAN_HWTSO>
ether a0:36:9f:16:d0:9c
media: Ethernet autoselect
status: active
laggproto lacp lagghash l2,l3,l4
laggport: igb1 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
laggport: igb0 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
# ifconfig lagg1 - (external, NAT)
lagg1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=4209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWTSO>
ether 00:1e:67:59:ea:89
inet ХХХ.ХХХ.ХХХ.14 netmask 0xffffffe0 broadcast
ХХХ.ХХХ.ХХХ.31
inet ХХХ.ХХХ.ХХХ.70 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.70
inet ХХХ.ХХХ.ХХХ.71 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.71
inet ХХХ.ХХХ.ХХХ.72 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.72
inet ХХХ.ХХХ.ХХХ.73 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.73
inet ХХХ.ХХХ.ХХХ.74 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.74
inet ХХХ.ХХХ.ХХХ.75 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.75
inet ХХХ.ХХХ.ХХХ.76 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.76
inet ХХХ.ХХХ.ХХХ.77 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.77
inet ХХХ.ХХХ.ХХХ.78 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.78
inet ХХХ.ХХХ.ХХХ.79 netmask 0xffffffff broadcast
ХХХ.ХХХ.ХХХ.79
inet ХХХ.ХХХ.ХХХ.33 netmask 0xfffffff0 broadcast
ХХХ.ХХХ.ХХХ.47
media: Ethernet autoselect
status: active
laggproto lacp lagghash l2,l3,l4
laggport: em1 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
laggport: em0 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
# netstat -w1 -I em0
input (em0) output
packets errs idrops bytes packets errs bytes colls
101k 0 0 111M 36k 0 13M 0
101k 0 0 112M 36k 0 13M 0
100k 0 0 112M 37k 0 13M 0
# netstat -w1 -I em1
[code] input (em1) output
packets errs idrops bytes packets errs bytes colls
100k 0 0 111M 37k 0 9.1M 0
102k 0 0 113M 39k 0 10M 0
91k 0 0 101M 38k 0 9.7M 0
# netstat -w1 -I igb0
input (igb0) output
packets errs idrops bytes packets errs bytes colls
39k 0 0 9.1M 51k 0 57M 0
38k 0 0 9.1M 49k 0 54M 0
39k 0 0 9.4M 51k 0 56M 0
# netstat -w1 -I igb1
input (igb1) output
packets errs idrops bytes packets errs bytes colls
36k 0 0 14M 48k 0 56M 0
35k 0 0 14M 50k 0 59M 0
34k 0 0 13M 48k 0 57M 0
# netstat -w1 -I lagg0
input (lagg0) output
packets errs idrops bytes packets errs bytes colls
75k 0 0 23M 98k 0 113M 0
73k 0 0 21M 98k 0 113M 0
73k 0 0 23M 98k 0 112M 0
# netstat -w1 -I lagg1
input (lagg1) output
packets errs idrops bytes packets errs bytes colls
100k 0 0 112M 74k 0 24M 0
101k 0 0 113M 73k 0 24M 0
102k 0 0 114M 74k 0 24M 0
>
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?16706.1373563943.8842093075575209984>