Skip to content

evpn can not work as expected(or not stable) #20437

New issue
New issue
Open
Open
evpn can not work as expected(or not stable)#20437
Labels
triageNeeds further investigation
@BurlyLuo

Description

@BurlyLuo
BurlyLuo
opened on Jan 12, 2026

Description

1. Containerlab topo:
[root@rowan> 02-evpn-l2-l3]# cat 1.yaml 
name: evpn
prefix: ""

topology:
  nodes:
    leaf1:
      kind: linux
      image: quay.io/weiluo/frrouting/frr:10.5.0
      cmd: sh -c "sed -i 's/bgpd=no/bgpd=yes/;s/zebra=no/zebra=yes/' /etc/frr/daemons && /usr/lib/frr/docker-start"
      binds:
        - /lib/modules:/lib/modules
        - ./frr/leaf1.conf:/etc/frr/frr.conf
      exec:
        - bash -c "echo 'PS1=\"[\\\\u@\\\\h]\\\\$ \"' > /root/.bashrc"
        - touch /etc/frr/vtysh.conf

        # add vtep ip
        - ip addr add 100.64.0.1/32 dev lo
         
        # leaf1 to spine
        - ip addr add 192.168.1.1/24 dev eth1
         
        # add vrf[spec vrf]
        - ip link add red type vrf table 1100
        - ip link set red up

        # For subnet 10.1.5.0/24
        - ip link add br5 type bridge
        - ip link set br5 master red
        - ip link set br5 addr aa:bb:cc:05:01:05
        - ip link add vni5000 type vxlan local 100.64.0.1 dstport 4789 id 5000 nolearning
        - ip link set vni5000 master br5 addrgenmode none
        - ip link set vni5000 type bridge_slave neigh_suppress on learning off
        - ip link set br5 up
        - ip link set vni5000 up
        - ip addr add 10.1.5.254/24 dev br5

        # For subnet 10.1.8.0/24
        - ip link add br8 type bridge
        - ip link set br8 master red
        - ip link set br8 addr aa:bb:cc:08:01:08
        - ip link add vni8000 type vxlan local 100.64.0.1 dstport 4789 id 8000 nolearning
        - ip link set vni8000 master br8 addrgenmode none
        - ip link set vni8000 type bridge_slave neigh_suppress on learning off
        - ip link set br8 up
        - ip link set vni8000 up
        - ip addr add 10.1.8.254/24 dev br8

        # For subnet 10.1.9.0/24 and 10.1.10.0/24
        - ip link add br100 type bridge
        - ip link set br100 master red addrgenmode none
        - ip link set br100 addr aa:bb:cc:09:01:09
        - ip link add vni100 type vxlan local 100.64.0.1 dstport 4789 id 100 nolearning
        - ip link set vni100 master br100 addrgenmode none
        - ip link set vni100 type bridge_slave neigh_suppress on learning off
        - ip link set vni100 up
        - ip link set br100 up
        
        # For leaf1 to vm:10.1.5.x and vm:10.1.8.x and vm:10.1.9.x
        - ip link set eth2 master br5
        - ip link set eth3 master br8
        - ip link set eth4 master red
        - ip addr add 10.1.9.254/24 dev eth4


    leaf2:
      kind: linux
      image: quay.io/weiluo/frrouting/frr:10.5.0
      cmd: sh -c "sed -i 's/bgpd=no/bgpd=yes/;s/zebra=no/zebra=yes/' /etc/frr/daemons && /usr/lib/frr/docker-start"
      binds:
        - /lib/modules:/lib/modules
        - ./frr/leaf2.conf:/etc/frr/frr.conf
      exec:
        - bash -c "echo 'PS1=\"[\\\\u@\\\\h]\\\\$ \"' > /root/.bashrc"
        - touch /etc/frr/vtysh.conf

        # add vtep ip 
        - ip addr add 100.65.0.2/32 dev lo

        # leaf2 to spine
        - ip addr add 192.168.1.3/24 dev eth1

        # add vrf[spec vrf] 
        - ip link add red type vrf table 1100
        - ip link set red up

        # For subnet 10.1.5.0/24 
        - ip link add br5 type bridge
        - ip link set br5 master red
        - ip link set br5 addr aa:bb:cc:05:02:05
        - ip addr add 10.1.5.254/24 dev br5
        - ip link add vni5000 type vxlan local 100.65.0.2 dstport 4789 id 5000 nolearning
        - ip link set vni5000 master br5 addrgenmode none
        - ip link set vni5000 type bridge_slave neigh_suppress on learning off
        - ip link set vni5000 up
        - ip link set br5 up

        # For subnet 10.1.8.0/24
        - ip link add br8 type bridge
        - ip link set br8 master red
        - ip link set br8 addr aa:bb:cc:08:02:08
        - ip addr add 10.1.8.254/24 dev br8
        - ip link add vni8000 type vxlan local 100.65.0.2 dstport 4789 id 8000 nolearning
        - ip link set vni8000 master br8 addrgenmode none
        - ip link set vni8000 type bridge_slave neigh_suppress on learning off
        - ip link set vni8000 up
        - ip link set br8 up

        # For subnet 10.1.9.0/24 and 10.1.10.0/24
        - ip link add br100 type bridge
        - ip link set br100 master red addrgenmode none
        - ip link set br100 addr aa:bb:cc:09:02:09
        - ip link add vni100 type vxlan local 100.65.0.2 dstport 4789 id 100 nolearning
        - ip link set vni100 master br100 addrgenmode none
        - ip link set vni100 type bridge_slave neigh_suppress on learning off
        - ip link set vni100 up
        - ip link set br100 up
        
        - ip link set eth2 master br5
        - ip link set eth3 master br8
        - ip link set eth4 master red
        - ip addr add 10.1.10.254/24 dev eth4

    spine:
      kind: linux
      image: quay.io/weiluo/frrouting/frr:10.5.0
      cmd: sh -c "sed -i 's/bgpd=no/bgpd=yes/;s/zebra=no/zebra=yes/' /etc/frr/daemons && /usr/lib/frr/docker-start"
      binds:
        - /lib/modules:/lib/modules
        - ./frr/spine.conf:/etc/frr/frr.conf
      exec:
        - bash -c "echo 'PS1=\"[\\\\u@\\\\h]\\\\$ \"' > /root/.bashrc"
        - touch /etc/frr/vtysh.conf
        # spine to leaf1 and leaf2
        - ip addr add 192.168.1.0/31 dev eth1
        - ip addr add 192.168.1.2/31 dev eth2

    vm1:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.5.10/24 dev eth1
        - ip r r default via 10.1.5.254

    vm2:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.8.10/24 dev eth1
        - ip r r default via 10.1.8.254

    vm3:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.5.11/24 dev eth1
        - ip r r default via 10.1.5.254

    vm4:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.8.11/24 dev eth1
        - ip r r default via 10.1.8.254

    vm5:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.9.10/24 dev eth1
        - ip r r default via 10.1.9.254

    vm6:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.10.10/24 dev eth1
        - ip r r default via 10.1.10.254

  links:
    # 192.168.1.0/31 <> 192.168.1.1/31
    - endpoints: ["spine:eth1", "leaf1:eth1"]
    # 192.168.1.2/31 <> 192.168.1.3/31
    - endpoints: ["spine:eth2", "leaf2:eth1"]
   
    # 10.1.5.10/24 <> 10.1.5.11/24
    - endpoints: ["leaf1:eth2", "vm1:eth1"]
    - endpoints: ["leaf2:eth2", "vm3:eth1"]
    # 10.1.8.10/24 <> 10.1.8.11/24
    - endpoints: ["leaf1:eth3", "vm2:eth1"]
    - endpoints: ["leaf2:eth3", "vm4:eth1"]

    # 10.1.9.10/24 <> 10.1.10.10/24
    - endpoints: ["leaf1:eth4", "vm5:eth1"]
    - endpoints: ["leaf2:eth4", "vm6:eth1"]
[root@rowan> 02-evpn-l2-l3]#

and the evpn route show

[root@rowan> 02-evpn-l2-l3]# lo leaf1 bash 
[root@leaf1]# vtysh 

Hello, this is FRRouting (version 10.5.0_git).
Copyright 1996-2005 Kunihiro Ishiguro, et al.

leaf1# show bgp l2vpn evpn route 
BGP table version is 27, local router ID is 100.64.0.1
Status codes: s suppressed, d damped, h history, * valid, > best, i - internal
Origin codes: i - IGP, e - EGP, ? - incomplete
EVPN type-1 prefix: [1]:[EthTag]:[ESI]:[IPlen]:[VTEP-IP]:[Frag-id]
EVPN type-2 prefix: [2]:[EthTag]:[MAClen]:[MAC]:[IPlen]:[IP]
EVPN type-3 prefix: [3]:[EthTag]:[IPlen]:[OrigIP]
EVPN type-4 prefix: [4]:[ESI]:[IPlen]:[OrigIP]
EVPN type-5 prefix: [5]:[EthTag]:[IPlen]:[IP]

   Network          Next Hop            Metric LocPrf Weight Path
                    Extended Community
Route Distinguisher: 100.64.0.1:2
 *>  [5]:[0]:[24]:[10.1.5.0]
                    100.64.0.1               0         32768 ?
                    ET:8 RT:64512:100 Rmac:aa:bb:cc:09:01:09
 *>  [5]:[0]:[24]:[10.1.8.0]
                    100.64.0.1               0         32768 ?
                    ET:8 RT:64512:100 Rmac:aa:bb:cc:09:01:09
 *>  [5]:[0]:[24]:[10.1.9.0]
                    100.64.0.1               0         32768 ?
                    ET:8 RT:64512:100 Rmac:aa:bb:cc:09:01:09
Route Distinguisher: 100.64.0.1:3
 *>  [2]:[0]:[48]:[aa:bb:cc:05:01:05]:[32]:[10.1.5.254]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:5000 RT:64512:100 Rmac:aa:bb:cc:09:01:09
 *>  [2]:[0]:[48]:[aa:bb:cc:05:01:05]:[128]:[fe80::a8bb:ccff:fe05:105]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:5000
 *>  [2]:[0]:[48]:[aa:c1:ab:84:ce:2b]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:5000
 *>  [2]:[0]:[48]:[aa:c1:ab:84:ce:2b]:[32]:[10.1.5.10]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:5000 RT:64512:100 Rmac:aa:bb:cc:09:01:09
 *>  [3]:[0]:[32]:[100.64.0.1]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:5000
Route Distinguisher: 100.64.0.1:4
 *>  [2]:[0]:[48]:[aa:bb:cc:08:01:08]:[32]:[10.1.8.254]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:8000 RT:64512:100 Rmac:aa:bb:cc:09:01:09
 *>  [2]:[0]:[48]:[aa:bb:cc:08:01:08]:[128]:[fe80::a8bb:ccff:fe08:108]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:8000
 *>  [3]:[0]:[32]:[100.64.0.1]
                    100.64.0.1                         32768 i
                    ET:8 RT:64512:8000
Route Distinguisher: 100.65.0.2:2
 *>  [5]:[0]:[24]:[10.1.5.0]
                    100.65.0.2                             0 64612 64512 ?
                    RT:64512:100 ET:8 Rmac:aa:bb:cc:09:02:09
 *>  [5]:[0]:[24]:[10.1.8.0]
                    100.65.0.2                             0 64612 64512 ?
                    RT:64512:100 ET:8 Rmac:aa:bb:cc:09:02:09
 *>  [5]:[0]:[24]:[10.1.10.0]
                    100.65.0.2                             0 64612 64512 ?
                    RT:64512:100 ET:8 Rmac:aa:bb:cc:09:02:09
Route Distinguisher: 100.65.0.2:3
 *>  [2]:[0]:[48]:[aa:bb:cc:05:02:05]:[128]:[fe80::a8bb:ccff:fe05:205]
                    100.65.0.2                             0 64612 64512 i
                    RT:64512:5000 ET:8
 *>  [3]:[0]:[32]:[100.65.0.2]
                    100.65.0.2                             0 64612 64512 i
                    RT:64512:5000 ET:8
Route Distinguisher: 100.65.0.2:4
 *>  [2]:[0]:[48]:[aa:bb:cc:08:02:08]:[128]:[fe80::a8bb:ccff:fe08:208]
                    100.65.0.2                             0 64612 64512 i
                    RT:64512:8000 ET:8
 *>  [3]:[0]:[32]:[100.65.0.2]
                    100.65.0.2                             0 64612 64512 i
                    RT:64512:8000 ET:8

Displayed 18 prefixes (18 paths)
leaf1#

there should be 10.1.5.11 and 10.1.8.11. but not there. if clear bgp *, some time is fine, but not always.

Version

leaf1# show version 
FRRouting 10.5.0_git (leaf1) on Linux(6.8.0-90-generic).
Copyright 1996-2005 Kunihiro Ishiguro, et al.
configured with:
    '--prefix=/usr' '--sysconfdir=/etc' '--localstatedir=/var' '--sbindir=/usr/lib/frr' '--libdir=/usr/lib' '--enable-rpki' '--enable-vtysh' '--enable-multipath=64' '--enable-vty-group=frrvty' '--enable-user=frr' '--enable-group=frr' '--enable-pcre2posix' '--enable-scripting' 'CC=cc' 'CXX=c++'
leaf1#


[root@rowan> 1231]# cat /etc/os-release 
PRETTY_NAME="Ubuntu 22.04.3 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.3 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy
[root@rowan> 1231]# uname -a
Linux rowan 6.8.0-90-generic #91~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Thu Nov 20 15:20:45 UTC 2 x86_64 x86_64 x86_64 GNU/Linux
[root@rowan> 1231]#

How to reproduce

1.topo file:
[root@rowan> 02-evpn-l2-l3]# cat 1.yaml 
name: evpn
prefix: ""

topology:
  nodes:
    leaf1:
      kind: linux
      image: quay.io/weiluo/frrouting/frr:10.5.0
      cmd: sh -c "sed -i 's/bgpd=no/bgpd=yes/;s/zebra=no/zebra=yes/' /etc/frr/daemons && /usr/lib/frr/docker-start"
      binds:
        - /lib/modules:/lib/modules
        - ./frr/leaf1.conf:/etc/frr/frr.conf
      exec:
        - bash -c "echo 'PS1=\"[\\\\u@\\\\h]\\\\$ \"' > /root/.bashrc"
        - touch /etc/frr/vtysh.conf

        # add vtep ip
        - ip addr add 100.64.0.1/32 dev lo
         
        # leaf1 to spine
        - ip addr add 192.168.1.1/24 dev eth1
         
        # add vrf[spec vrf]
        - ip link add red type vrf table 1100
        - ip link set red up

        # For subnet 10.1.5.0/24
        - ip link add br5 type bridge
        - ip link set br5 master red
        - ip link set br5 addr aa:bb:cc:05:01:05
        - ip link add vni5000 type vxlan local 100.64.0.1 dstport 4789 id 5000 nolearning
        - ip link set vni5000 master br5 addrgenmode none
        - ip link set vni5000 type bridge_slave neigh_suppress on learning off
        - ip link set br5 up
        - ip link set vni5000 up
        - ip addr add 10.1.5.254/24 dev br5

        # For subnet 10.1.8.0/24
        - ip link add br8 type bridge
        - ip link set br8 master red
        - ip link set br8 addr aa:bb:cc:08:01:08
        - ip link add vni8000 type vxlan local 100.64.0.1 dstport 4789 id 8000 nolearning
        - ip link set vni8000 master br8 addrgenmode none
        - ip link set vni8000 type bridge_slave neigh_suppress on learning off
        - ip link set br8 up
        - ip link set vni8000 up
        - ip addr add 10.1.8.254/24 dev br8

        # For subnet 10.1.9.0/24 and 10.1.10.0/24
        - ip link add br100 type bridge
        - ip link set br100 master red addrgenmode none
        - ip link set br100 addr aa:bb:cc:09:01:09
        - ip link add vni100 type vxlan local 100.64.0.1 dstport 4789 id 100 nolearning
        - ip link set vni100 master br100 addrgenmode none
        - ip link set vni100 type bridge_slave neigh_suppress on learning off
        - ip link set vni100 up
        - ip link set br100 up
        
        # For leaf1 to vm:10.1.5.x and vm:10.1.8.x and vm:10.1.9.x
        - ip link set eth2 master br5
        - ip link set eth3 master br8
        - ip link set eth4 master red
        - ip addr add 10.1.9.254/24 dev eth4


    leaf2:
      kind: linux
      image: quay.io/weiluo/frrouting/frr:10.5.0
      cmd: sh -c "sed -i 's/bgpd=no/bgpd=yes/;s/zebra=no/zebra=yes/' /etc/frr/daemons && /usr/lib/frr/docker-start"
      binds:
        - /lib/modules:/lib/modules
        - ./frr/leaf2.conf:/etc/frr/frr.conf
      exec:
        - bash -c "echo 'PS1=\"[\\\\u@\\\\h]\\\\$ \"' > /root/.bashrc"
        - touch /etc/frr/vtysh.conf

        # add vtep ip 
        - ip addr add 100.65.0.2/32 dev lo

        # leaf2 to spine
        - ip addr add 192.168.1.3/24 dev eth1

        # add vrf[spec vrf] 
        - ip link add red type vrf table 1100
        - ip link set red up

        # For subnet 10.1.5.0/24 
        - ip link add br5 type bridge
        - ip link set br5 master red
        - ip link set br5 addr aa:bb:cc:05:02:05
        - ip addr add 10.1.5.254/24 dev br5
        - ip link add vni5000 type vxlan local 100.65.0.2 dstport 4789 id 5000 nolearning
        - ip link set vni5000 master br5 addrgenmode none
        - ip link set vni5000 type bridge_slave neigh_suppress on learning off
        - ip link set vni5000 up
        - ip link set br5 up

        # For subnet 10.1.8.0/24
        - ip link add br8 type bridge
        - ip link set br8 master red
        - ip link set br8 addr aa:bb:cc:08:02:08
        - ip addr add 10.1.8.254/24 dev br8
        - ip link add vni8000 type vxlan local 100.65.0.2 dstport 4789 id 8000 nolearning
        - ip link set vni8000 master br8 addrgenmode none
        - ip link set vni8000 type bridge_slave neigh_suppress on learning off
        - ip link set vni8000 up
        - ip link set br8 up

        # For subnet 10.1.9.0/24 and 10.1.10.0/24
        - ip link add br100 type bridge
        - ip link set br100 master red addrgenmode none
        - ip link set br100 addr aa:bb:cc:09:02:09
        - ip link add vni100 type vxlan local 100.65.0.2 dstport 4789 id 100 nolearning
        - ip link set vni100 master br100 addrgenmode none
        - ip link set vni100 type bridge_slave neigh_suppress on learning off
        - ip link set vni100 up
        - ip link set br100 up
        
        - ip link set eth2 master br5
        - ip link set eth3 master br8
        - ip link set eth4 master red
        - ip addr add 10.1.10.254/24 dev eth4

    spine:
      kind: linux
      image: quay.io/weiluo/frrouting/frr:10.5.0
      cmd: sh -c "sed -i 's/bgpd=no/bgpd=yes/;s/zebra=no/zebra=yes/' /etc/frr/daemons && /usr/lib/frr/docker-start"
      binds:
        - /lib/modules:/lib/modules
        - ./frr/spine.conf:/etc/frr/frr.conf
      exec:
        - bash -c "echo 'PS1=\"[\\\\u@\\\\h]\\\\$ \"' > /root/.bashrc"
        - touch /etc/frr/vtysh.conf
        # spine to leaf1 and leaf2
        - ip addr add 192.168.1.0/31 dev eth1
        - ip addr add 192.168.1.2/31 dev eth2

    vm1:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.5.10/24 dev eth1
        - ip r r default via 10.1.5.254

    vm2:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.8.10/24 dev eth1
        - ip r r default via 10.1.8.254

    vm3:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.5.11/24 dev eth1
        - ip r r default via 10.1.5.254

    vm4:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.8.11/24 dev eth1
        - ip r r default via 10.1.8.254

    vm5:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.9.10/24 dev eth1
        - ip r r default via 10.1.9.254

    vm6:
      kind: linux
      image: 192.168.2.100:5000/nettool
      exec:
        - ip addr add 10.1.10.10/24 dev eth1
        - ip r r default via 10.1.10.254

  links:
    # 192.168.1.0/31 <> 192.168.1.1/31
    - endpoints: ["spine:eth1", "leaf1:eth1"]
    # 192.168.1.2/31 <> 192.168.1.3/31
    - endpoints: ["spine:eth2", "leaf2:eth1"]
   
    # 10.1.5.10/24 <> 10.1.5.11/24
    - endpoints: ["leaf1:eth2", "vm1:eth1"]
    - endpoints: ["leaf2:eth2", "vm3:eth1"]
    # 10.1.8.10/24 <> 10.1.8.11/24
    - endpoints: ["leaf1:eth3", "vm2:eth1"]
    - endpoints: ["leaf2:eth3", "vm4:eth1"]

    # 10.1.9.10/24 <> 10.1.10.10/24
    - endpoints: ["leaf1:eth4", "vm5:eth1"]
    - endpoints: ["leaf2:eth4", "vm6:eth1"]
[root@rowan> 02-evpn-l2-l3]#
  1. deploy with containerlab
[root@rowan> 02-evpn-l2-l3]# alias clc
alias clc='rm -rf clab-* >/dev/null 2>&1 && clab deploy -t $(grep -l "topology:" *.yaml *.yml 2>/dev/null | head -1)'
[root@rowan> 02-evpn-l2-l3]#

topo:

Image

Expected behavior

10.1.5.11 and 10.1.8.11 with BGP route. vrf red Vrf.

Actual behavior

unstable
and always not missed 10.1.5.11 and 10.1.8.11 at the vrf rotting table.

Additional context

evpn.zip

Attach all the files for reproduce.

Checklist

  • I have searched the open issues for this bug.
  • I have not included sensitive information in this report.

Metadata

Metadata

Assignees

No one assigned

    Labels

    triageNeeds further investigation

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions