Date: Thu, 11 Jul 2013 21:11:41 +0300 From: "isp" <mline@ukr.net> To: "Alan Somers" <asomers@freebsd.org> Cc: freebsd-net@freebsd.org Subject: Re[2]: FreeBSD router problems Message-ID: <3798.1373566301.16193856550018220032@ffe15.ukr.net> In-Reply-To: <CAOtMX2iSWQbmm4rZT0ATAH3QsK8C9HxWcUm57JgJBmW0qnQquA@mail.gmail.com> References: <16706.1373563943.8842093075575209984@ffe17.ukr.net> <CAOtMX2iSWQbmm4rZT0ATAH3QsK8C9HxWcUm57JgJBmW0qnQquA@mail.gmail.com>
next in thread | previous in thread | raw e-mail | index | archive | help
I have a real network with more than 4 000 users. In normal case, when I
have two 1Gbps routers, and I split VLAN's between them total bandwidth
if growing up to 1.7 Gbps.
--- Incoming mail ---
From: "Alan Somers" <asomers@freebsd.org>
Date: 11 July 2013, 21:00:41
How are you benchmarking it? Each TCP connection only uses one member
of a lagg port. So if you want to see > 1 Gbps, you'll need to
benchmark with multiple TCP connections. You may also need multiple
systems; I don't know the full details of LACP.
On Thu, Jul 11, 2013 at 11:32 AM, isp < mline@ukr.net > wrote:
>
>
>
> Hi! I have a problem with my FreeBSD router, I can't get more than 1 Gbps
> throught it, but I have 2 Gbps LAGG on it. There are only 27 IPFW rules
> (NAT+Shaping). IPoE only.
> lagg0 (VLAN's + shaping) - two 'igb' adapters
> lagg1 (NAT, tso if off) - two 'em' adapters
>
> I tried to switch off dummynet, but it doesn't helps.
>
> # uname -a
> [code]FreeBSD router 9.1-RELEASE-p3 FreeBSD 9.1-RELEASE-p3 #0: Tue Apr 30
> 20:02:00 EEST 2013 root@south:/usr/obj/usr/src/sys/ROUTER amd64
>
> # top -aSPHI
> last pid: 91712; load averages: 2.18, 2.06,
> 1.97
> up 20+22:28:36 17:40:22
> 120 processes: 7 running, 87 sleeping, 26 waiting
> CPU 0: 0.0% user, 0.0% nice, 1.6% system, 38.6% interrupt, 59.8% idle
> CPU 1: 0.0% user, 0.0% nice, 7.1% system, 37.0% interrupt, 55.9% idle
> CPU 2: 0.0% user, 0.0% nice, 3.9% system, 38.6% interrupt, 57.5% idle
> CPU 3: 0.0% user, 0.0% nice, 15.7% system, 26.8% interrupt, 57.5% idle
> Mem: 59M Active, 1102M Inact, 942M Wired, 800M Buf, 5529M Free
> Swap: 16G Total, 16G Free
>
> PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAND
> 12 root -72 - 0K 448K RUN 1 153:39 72.22% [intr{swi1:
> netisr 0}]
> 11 root 155 ki31 0K 64K RUN 1 494.2H 65.19% [idle{idle:
> cpu1}]
> 11 root 155 ki31 0K 64K CPU2 2 494.3H 64.65% [idle{idle:
> cpu2}]
> 11 root 155 ki31 0K 64K RUN 0 493.3H 63.38% [idle{idle:
> cpu0}]
> 11 root 155 ki31 0K 64K CPU3 3 496.4H 62.55% [idle{idle:
> cpu3}]
> 12 root -92 - 0K 448K WAIT 2 58:49 9.38% [intr{irq266:
> igb0:que}]
> 12 root -92 - 0K 448K WAIT 2 59:32 9.03% [intr{irq271:
> igb1:que}]
> 12 root -92 - 0K 448K CPU1 1 59:09 8.94% [intr{irq265:
> igb0:que}]
> 12 root -92 - 0K 448K WAIT 3 57:52 8.01% [intr{irq272:
> igb1:que}]
> 12 root -92 - 0K 448K WAIT 1 59:32 7.96% [intr{irq270:
> igb1:que}]
> 12 root -92 - 0K 448K WAIT 3 55:47 7.81% [intr{irq267:
> igb0:que}]
> 12 root -92 - 0K 448K WAIT 0 55:24 7.23% [intr{irq264:
> igb0:que}]
> 12 root -92 - 0K 448K WAIT 0 56:57 6.69% [intr{irq269:
> igb1:que}]
> 12 root -92 - 0K 448K WAIT 3 203:34 4.74% [intr{irq275:
> em1:rx 0}]
> 0 root -92 0 0K 336K - 2 427:03 2.64%
> [kernel{dummynet}]
> 0 root -92 0 0K 336K - 3 206:57 2.54% [kernel{em0
> que}]
> 86278 root 20 0 33348K 8588K select 0 8:35 0.54%
> /usr/local/sbin/snmpd -p /var/run/net_snmpd.pid -r
> 12 root -92 - 0K 448K WAIT 2 7:56 0.20% [intr{irq276:
> em1:tx 0}]
>
> # cat /etc/sysctl.conf
> dev.igb.0.rx_processing_limit=4096
> dev.igb.1.rx_processing_limit=4096
> dev.em.0.rx_int_delay=200
> dev.em.0.tx_int_delay=200
> dev.em.0.rx_abs_int_delay=4000
> dev.em.0.tx_abs_int_delay=4000
> dev.em.0.rx_processing_limit=4096
> dev.em.1.rx_int_delay=200
> dev.em.1.tx_int_delay=200
> dev.em.1.rx_abs_int_delay=4000
> dev.em.1.tx_abs_int_delay=4000
> dev.em.1.rx_processing_limit=4096
> net.inet.ip.forwarding=1
> net.inet.ip.fastforwarding=1
> net.inet.tcp.blackhole=2
> net.inet.udp.blackhole=0
> net.inet.ip.redirect=0
> net.inet.tcp.delayed_ack=0
> net.inet.tcp.recvbuf_max=4194304
> net.inet.tcp.sendbuf_max=4194304
> net.inet.tcp.sack.enable=0
> net.inet.tcp.drop_synfin=1
> net.inet.tcp.nolocaltimewait=1
> net.inet.ip.ttl=255
> net.inet.ip.sourceroute=0
> net.inet.ip.accept_sourceroute=0
> net.inet.udp.recvspace=64080
> net.inet.ip.rtmaxcache=1024
> net.inet.ip.intr_queue_maxlen=5120
> kern.ipc.nmbclusters=824288
> kern.ipc.maxsockbuf=83886080
> kern.ipc.maxsockets=102400
> net.inet.tcp.recvspace=95536
> net.inet.tcp.sendspace=95536
> net.local.stream.recvspace=32768
> net.local.stream.sendspace=32768
> kern.ipc.somaxconn=32768
> net.inet.tcp.maxtcptw=65535
> net.inet.ip.fw.one_pass=1
> net.inet.ip.fw.dyn_max=65535
> net.inet.ip.fw.dyn_buckets=2048
> net.inet.ip.fw.dyn_syn_lifetime=10
> net.inet.ip.fw.dyn_ack_lifetime=120
> net.inet.ip.fw.verbose=0
> net.inet.ip.dummynet.io_fast=1
> net.inet.ip.dummynet.hash_size=65536
> net.inet.ip.dummynet.pipe_slot_limit=1000
> net.inet.icmp.icmplim=3000
> net.inet.icmp.drop_redirect=1
> net.inet.icmp.log_redirect=0
> net.inet.icmp.bmcastecho=0
> net.inet.icmp.maskrepl=0
> kern.random.sys.harvest.ethernet=0
> kern.random.sys.harvest.point_to_point=0
> kern.random.sys.harvest.interrupt=0
> net.inet.raw.maxdgram=16384
> net.inet.raw.recvspace=16384
> net.route.netisr_maxqlen=8192
> net.inet.ip.intr_queue_maxlen=10240
> net.isr.dispatch=deferred
>
> # cat /boot/loader.conf
> loader_logo="beastie"
> autoboot_delay=3
> geom_mirror_load="YES"
> hw.igb.rxd=4096
> hw.igb.txd=4096
> hw.igb.rx_process_limit=4096
> hw.igb.max_interrupt_rate=32000
> hw.igb.num_queues=4
> hw.igb.fc_setting=0
> hw.igb.lro=0
> hw.em.rxd=4096
> hw.em.txd=4096
> hw.em.rx_process_limit=4096
> hw.em.fc_setting=0
> dev.em.0.rx_int_delay=200
> dev.em.0.tx_int_delay=200
> dev.em.0.rx_abs_int_delay=4000
> dev.em.0.tx_abs_int_delay=4000
> dev.em.1.rx_int_delay=200
> dev.em.1.tx_int_delay=200
> dev.em.1.rx_abs_int_delay=4000
> dev.em.1.tx_abs_int_delay=4000
> net.isr.maxthreads=4
> net.isr.bindthreads=0
> net.inet.tcp.tcbhashsize=32000
> net.link.ifqmaxlen=10240
> net.isr.defaultqlimit=8192
>
> # vmstat -i
> interrupt total rate
> irq20: ehci1 4171628 2
> irq21: atapci0 1561194 0
> irq22: ehci0+ 2713150 1
> cpu0:timer 14622957598 8082
> irq264: igb0:que 0 515616328 284
> irq265: igb0:que 1 738456087 408
> irq266: igb0:que 2 711371660 393
> irq267: igb0:que 3 462738813 255
> irq268: igb0:link 3 0
> irq269: igb1:que 0 656044816 362
> irq270: igb1:que 1 546931002 302
> irq271: igb1:que 2 617173223 341
> irq272: igb1:que 3 644295672 356
> irq273: igb1:link 4 0
> irq274: em0 557400132 308
> irq275: em1:rx 0 424252744 234
> irq276: em1:tx 0 708469817 391
> irq277: em1:link 2 0
> cpu3:timer 678408141 374
> cpu1:timer 674674076 372
> cpu2:timer 621495291 343
> Total 23188731381 12816
>
> # netstat -w1
> input (Total) output
> packets errs idrops bytes packets errs bytes colls
> 442k 0 0 304M 457k 0 393M 0
> 449k 0 0 308M 463k 0 395M 0
> 445k 0 0 304M 461k 0 393M 0
> 439k 0 0 303M 456k 0 393M 0
> 434k 0 0 297M 450k 0 387M 0
> 440k 0 0 301M 456k 0 392M 0
> 438k 0 0 300M 455k 0 391M 0
>
> # ifconfig lagg0 (internal, 500 VLAN's)
> lagg0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
> 1500
> options=401bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,VLAN_HWTSO>
> ether a0:36:9f:16:d0:9c
> media: Ethernet autoselect
> status: active
> laggproto lacp lagghash l2,l3,l4
> laggport: igb1 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
> laggport: igb0 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
>
> # ifconfig lagg1 - (external, NAT)
> lagg1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
> 1500
> options=4209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWTSO>
> ether 00:1e:67:59:ea:89
> inet ХХХ.ХХХ.ХХХ.14 netmask 0xffffffe0 broadcast
> ХХХ.ХХХ.ХХХ.31
> inet ХХХ.ХХХ.ХХХ.70 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.70
> inet ХХХ.ХХХ.ХХХ.71 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.71
> inet ХХХ.ХХХ.ХХХ.72 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.72
> inet ХХХ.ХХХ.ХХХ.73 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.73
> inet ХХХ.ХХХ.ХХХ.74 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.74
> inet ХХХ.ХХХ.ХХХ.75 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.75
> inet ХХХ.ХХХ.ХХХ.76 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.76
> inet ХХХ.ХХХ.ХХХ.77 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.77
> inet ХХХ.ХХХ.ХХХ.78 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.78
> inet ХХХ.ХХХ.ХХХ.79 netmask 0xffffffff broadcast
> ХХХ.ХХХ.ХХХ.79
> inet ХХХ.ХХХ.ХХХ.33 netmask 0xfffffff0 broadcast
> ХХХ.ХХХ.ХХХ.47
> media: Ethernet autoselect
> status: active
> laggproto lacp lagghash l2,l3,l4
> laggport: em1 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
> laggport: em0 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
>
> # netstat -w1 -I em0
> input (em0) output
> packets errs idrops bytes packets errs bytes colls
> 101k 0 0 111M 36k 0 13M 0
> 101k 0 0 112M 36k 0 13M 0
> 100k 0 0 112M 37k 0 13M 0
>
> # netstat -w1 -I em1
> [code] input (em1) output
> packets errs idrops bytes packets errs bytes colls
> 100k 0 0 111M 37k 0 9.1M 0
> 102k 0 0 113M 39k 0 10M 0
> 91k 0 0 101M 38k 0 9.7M 0
>
> # netstat -w1 -I igb0
> input (igb0) output
> packets errs idrops bytes packets errs bytes colls
> 39k 0 0 9.1M 51k 0 57M 0
> 38k 0 0 9.1M 49k 0 54M 0
> 39k 0 0 9.4M 51k 0 56M 0
>
> # netstat -w1 -I igb1
> input (igb1) output
> packets errs idrops bytes packets errs bytes colls
> 36k 0 0 14M 48k 0 56M 0
> 35k 0 0 14M 50k 0 59M 0
> 34k 0 0 13M 48k 0 57M 0
>
> # netstat -w1 -I lagg0
> input (lagg0) output
> packets errs idrops bytes packets errs bytes colls
> 75k 0 0 23M 98k 0 113M 0
> 73k 0 0 21M 98k 0 113M 0
> 73k 0 0 23M 98k 0 112M 0
>
> # netstat -w1 -I lagg1
> input (lagg1) output
> packets errs idrops bytes packets errs bytes colls
> 100k 0 0 112M 74k 0 24M 0
> 101k 0 0 113M 73k 0 24M 0
> 102k 0 0 114M 74k 0 24M 0
>
>>
> _______________________________________________
> freebsd-net@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-net > To unsubscribe, send any mail to "freebsd-net-unsubscribe@freebsd.org"
_______________________________________________ freebsd-net@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-net To unsubscribe, send any mail to "freebsd-net-unsubscribe@freebsd.org"
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?3798.1373566301.16193856550018220032>