電子發燒友App
前分析過linux kernel 2.6.32的bridge轉發邏輯，下面分析一下linux kernel 3.10的bridge轉發邏輯。這樣正是CentOS 5和CentOS 7對應的內核。3.10 kernel中bridge邏輯的最大改變就是增加了vlan處理邏輯以及brdige入口函數的設置。
1. netdev_rx_handler_register
在分析之前首先要介紹一個重要函數：netdev_rx_handler_register，這個函數是2.6內核所沒有的。
netdev_rx_handler_register
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/*
* dev: 要注冊接收函數的dev
* rx_handler: 要注冊的接收函數
* rx_handler_data: 指向rx_handler_data使用的數據
*/
int netdev_rx_handler_register(struct net_device *dev,
rx_handler_func_t *rx_handler,
void *rx_handler_data)
{
ASSERT_RTNL();
if (dev->rx_handler)
return -EBUSY;
/* Note: rx_handler_data must be set before rx_handler */
rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
rcu_assign_pointer(dev->rx_handler, rx_handler);
return 0;
}
這個函數可以給設備（net_device）注冊接收函數，然后在__netif_receive_skb函數中根據接收skb的設備接口，再調用這個被注冊的接收函數。比如為網橋下的接口注冊br_handle_frame函數，為bonding接口注冊bond_handle_frame函數。這相對于老式的網橋處理更靈活，有了這個機制也可以在模塊中自行注冊處理函數。比如3.10中的openvswitch（OpenvSwitch在3.10已經合入了內核）創建netdev vport的函數netdev_create。
netdev_create
1
2
3
4
5
6
7
static struct vport *netdev_create(const struct vport_parms *parms)
{
struct vport *vport;
/....../
err = netdev_rx_handler_register(netdev_vport->dev, netdev_frame_hook,vport);
/....../
}
這個函數在創建netdev vport時將設備的接收函數設置為netdev_frame_hook函數，這也是整個openvswitch的入口函數，如果查看OpenvSwitch的源碼可以看到當安裝于2.6內核時這里是替換掉bridge的br_handle_frame_hook函數，從而由bridge邏輯進入OpenvSwitch邏輯。
2. Bridge轉發邏輯分析
還是先從netif_receive_skb函數分析，這個函數算是進入協議棧的入口。
netif_receive_skb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
int netif_receive_skb(struct sk_buff *skb)
{
int ret;
if (skb_defer_rx_timestamp(skb))
return NET_RX_SUCCESS;
rcu_read_lock();
/*RPS邏輯處理，現在內核中使用了RPS機制, 將報文分散到各個cpu的接收隊列中進行負載均衡處理*/
#ifdef CONFIG_RPS
if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu = get_rps_cpu(skb->dev, skb, &rflow);
if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
return ret;
}
}
#endif
ret = __netif_receive_skb(skb);
rcu_read_unlock();
return ret;
}
netif_receive_skb只是對數據包進行了RPS的處理，然后調用__netif_receive_skb。
__netif_receive_skb并沒有其他多余的處理邏輯，主要調用 __netif_receive_skb_core，這個函數才真正相當于2.6內核的netif_receive_skb。以下代碼省略了和bridge無關的邏輯。
__netif_receive_skb_core
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
{
struct packet_type *ptype, *pt_prev;
rx_handler_func_t *rx_handler;
struct net_device *orig_dev;
struct net_device *null_or_dev;
bool deliver_exact = false;
int ret = NET_RX_DROP;
__be16 type;
/*......*/
orig_dev = skb->dev;
skb_reset_network_header(skb);
pt_prev = NULL;
skb->skb_iif = skb->dev->ifindex;
/*ptype_all協議處理，tcpdump抓包就在這里*/
list_for_each_entry_rcu(ptype, &ptype_all, list) {
if (!ptype->dev || ptype->dev == skb->dev) {
if (pt_prev)
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = ptype;
}
}
/*調用接收設備的rx_handler*/
rx_handler = rcu_dereference(skb->dev->rx_handler);
if (rx_handler) {
if (pt_prev) {
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = NULL;
}
switch (rx_handler(&skb)) {
case RX_HANDLER_CONSUMED:
ret = NET_RX_SUCCESS;
goto out;
case RX_HANDLER_ANOTHER:
goto another_round;
case RX_HANDLER_EXACT:
deliver_exact = true;
case RX_HANDLER_PASS:
break;
default:
BUG();
}
}
/*根據 skb->protocol傳遞給上層協議*/
type = skb->protocol;
list_for_each_entry_rcu(ptype,&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
if (ptype->type == type && (ptype->dev == null_or_dev || ptype->dev == skb->dev ||ptype->dev == orig_dev)) {
if (pt_prev)
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = ptype;
}
}
if (pt_prev) {
if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
goto drop;
else
ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
} else {
drop:
atomic_long_inc(&skb->dev->rx_dropped);
kfree_skb(skb);
ret = NET_RX_DROP;
}
out:
return ret;
}
如果一個dev被添加到一個bridge（做為bridge的一個接口），的這個接口設備的rx_handler被設置為br_handle_frame函數，這是在br_add_if函數中設置的，而br_add_if (net/bridge/br_if.c)是在向網橋設備上添加接口時設置的。進入br_handle_frame也就進入了bridge的邏輯代碼。
br_add_if
1
2
3
4
5
6
int br_add_if(struct net_bridge *br, struct net_device *dev)
{
/*......*/
err = netdev_rx_handler_register(dev, br_handle_frame, p);
/*......*/
}
br_handle_frame
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
rx_handler_result_t br_handle_frame(struct sk_buff **pskb)
{
struct net_bridge_port *p;
struct sk_buff *skb = *pskb;
const unsigned char *dest = eth_hdr(skb)->h_dest;
br_should_route_hook_t *rhook;
if (unlikely(skb->pkt_type == PACKET_LOOPBACK))
return RX_HANDLER_PASS;
if (!is_valid_ether_addr(eth_hdr(skb)->h_source))
goto drop;
skb = skb_share_check(skb, GFP_ATOMIC);
if (!skb)
return RX_HANDLER_CONSUMED;
/*獲取dev對應的bridge port*/
p = br_port_get_rcu(skb->dev);
/*特殊目的mac地址的處理*/
if (unlikely(is_link_local_ether_addr(dest))) {
/*
* See IEEE 802.1D Table 7-10 Reserved addresses
*
* Assignment Value
* Bridge Group Address 01-80-C2-00-00-00
* (MAC Control) 802.3 01-80-C2-00-00-01
* (Link Aggregation) 802.3 01-80-C2-00-00-02
* 802.1X PAE address 01-80-C2-00-00-03
*
* 802.1AB LLDP 01-80-C2-00-00-0E
*
* Others reserved for future standardization
*/
switch (dest[5]) {
case 0x00: /* Bridge Group Address */
/* If STP is turned off,then must forward to keep loop detection */
if (p->br->stp_enabled == BR_NO_STP)
goto forward;
break;
case 0x01: /* IEEE MAC (Pause) */
goto drop;
default:
/* Allow selective forwarding for most other protocols */
if (p->br->group_fwd_mask & (1u << dest[5]))
goto forward;
}
/* LOCAL_IN hook點，注意經過這個hook點并不代表發送到主機協議棧（只有特殊目的mac 01-80-C2才會走到這里）*/
if (NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, skb, skb->dev,
NULL, br_handle_local_finish)) {
return RX_HANDLER_CONSUMED; /* consumed by filter */
} else {
*pskb = skb;
return RX_HANDLER_PASS; /* continue processing */
}
}
/*轉發邏輯*/
forward:
switch (p->state) {
case BR_STATE_FORWARDING:
rhook = rcu_dereference(br_should_route_hook);
if (rhook) {
if ((*rhook)(skb)) {
*pskb = skb;
return RX_HANDLER_PASS;
}
dest = eth_hdr(skb)->h_dest;
}
/* fall through */
case BR_STATE_LEARNING:
/*skb的目的mac和bridge的mac一樣，則將skb發往本機協議棧*/
if (ether_addr_equal(p->br->dev->dev_addr, dest))
skb->pkt_type = PACKET_HOST;
/*NF_BR_PRE_ROUTING hook點*/
NF_HOOK(NFPROTO_BRIDGE, NF_BR_PRE_ROUTING, skb, skb->dev, NULL,br_handle_frame_finish);
break;
default:
drop:
kfree_skb(skb);
}
return RX_HANDLER_CONSUMED;
}
經過NF_BR_LOCAL_IN hook點會執行br_handle_local_finish函數。
br_handle_local_finish
1
2
3
4
5
6
7
8
9
10
static int br_handle_local_finish(struct sk_buff *skb)
{
struct net_bridge_port *p = br_port_get_rcu(skb->dev);
u16 vid = 0;
/*獲取skb的vlan id(3.10的bridge支持vlan)*/
br_vlan_get_tag(skb, &vid);
/*更新bridge的mac表，注意vlan id也是參數，說明每個vlan有一個獨立的mac表*/
br_fdb_update(p->br, p, eth_hdr(skb)->h_source, vid);
return 0; /* process further */
}
經過NF_BR_PRE_ROUTING hook點會執行br_handle_frame_finish函數。
br_handle_frame_finish
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
int br_handle_frame_finish(struct sk_buff *skb)
{
const unsigned char *dest = eth_hdr(skb)->h_dest;
struct net_bridge_port *p = br_port_get_rcu(skb->dev);
struct net_bridge *br;
struct net_bridge_fdb_entry *dst;
struct net_bridge_mdb_entry *mdst;
struct sk_buff *skb2;
u16 vid = 0;
if (!p || p->state == BR_STATE_DISABLED)
goto drop;
/*這個判斷主要是vlan的相關檢查，如是否和接收接口配置的vlan相同*/
if (!br_allowed_ingress(p->br, nbp_get_vlan_info(p), skb, &vid))
goto out;
/* insert into forwarding database after filtering to avoid spoofing */
br = p->br;
/*更新轉發數據庫*/
br_fdb_update(br, p, eth_hdr(skb)->h_source, vid);
/*多播mac的處理*/
if (!is_broadcast_ether_addr(dest) && is_multicast_ether_addr(dest) &&
br_multicast_rcv(br, p, skb))
goto drop;
if (p->state == BR_STATE_LEARNING)
goto drop;
BR_INPUT_SKB_CB(skb)->brdev = br->dev;
/* The packet skb2 goes to the local host (NULL to skip). */
skb2 = NULL;
/*如果網橋被設置為混雜模式*/
if (br->dev->flags & IFF_PROMISC)
skb2 = skb;
dst = NULL;
/*如果skb的目的mac是廣播*/
if (is_broadcast_ether_addr(dest))
skb2 = skb;
else if (is_multicast_ether_addr(dest)) { /*多播*/
mdst = br_mdb_get(br, skb, vid);
if (mdst || BR_INPUT_SKB_CB_MROUTERS_ONLY(skb)) {
if ((mdst && mdst->mglist) ||
br_multicast_is_router(br))
skb2 = skb;
br_multicast_forward(mdst, skb, skb2);
skb = NULL;
if (!skb2)
goto out;
} else
skb2 = skb;
br->dev->stats.multicast++;
} else if ((dst = __br_fdb_get(br, dest, vid)) && dst->is_local) {/*目的地址是本機mac，則發往本機協議棧*/
skb2 = skb;
/* Do not forward the packet since it's local. */
skb = NULL;
}
if (skb) {
if (dst) {
dst->used = jiffies;
br_forward(dst->dst, skb, skb2); //轉發給目的接口
} else
br_flood_forward(br, skb, skb2); //找不到目的接口則廣播
}
if (skb2)
return br_pass_frame_up(skb2); //發往本機協議棧
out:
return 0;
drop:
kfree_skb(skb);
goto out;
}
{C}
我們先看發往本機協議棧的函數br_pass_frame_up。
br_pass_frame_up
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
static int br_pass_frame_up(struct sk_buff *skb)
{
struct net_device *indev, *brdev = BR_INPUT_SKB_CB(skb)->brdev;
struct net_bridge *br = netdev_priv(brdev);
//更新統計計數(略)
/* Bridge is just like any other port. Make sure the
* packet is allowed except in promisc modue when someone
* may be running packet capture.
*/
if (!(brdev->flags & IFF_PROMISC) && !br_allowed_egress(br, br_get_vlan_info(br), skb)) {
kfree_skb(skb); //如果不是混雜模式且vlan處理不合要求則丟棄
return NET_RX_DROP;
}
//vlan處理邏輯
skb = br_handle_vlan(br, br_get_vlan_info(br), skb);
if (!skb)
return NET_RX_DROP;
indev = skb->dev;
skb->dev = brdev; //重點，這里修改了skb->dev為bridge
//經過NF_BR_LOCAL_IN再次進入協議棧
return NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, skb, indev, NULL,
netif_receive_skb);
}
{C}
再次進入netif_receive_skb，由于skb-dev被設置成了bridge，而bridge設備的rx_handler函數是沒有被設置的，所以就不會再次進入bridge邏輯，而直接進入了主機上層協議棧。
下面看轉發邏輯，轉發邏輯主要在br_forward函數中，而br_forward主要調用__br_forward函數。
__br_forward
1
2
3
4
5
6
7
8
9
10
11
12
13
14
static void __br_forward(const struct net_bridge_port *to, struct sk_buff *skb)
{
struct net_device *indev;
//vlan處理
skb = br_handle_vlan(to->br, nbp_get_vlan_info(to), skb);
if (!skb)
return;
indev = skb->dev;
skb->dev = to->dev; //skb->dev設置為出口設備dev
skb_forward_csum(skb);
//經過NF_BR_FORWARD hook點，調用br_forward_finish
NF_HOOK(NFPROTO_BRIDGE, NF_BR_FORWARD, skb, indev, skb->dev,
br_forward_finish);
}
{C}
br_forward_finish
1
2
3
4
5
int br_forward_finish(struct sk_buff *skb)
{
//經過NF_BR_POST_ROUTING hook點,調用br_dev_queue_push_xmit
return NF_HOOK(NFPROTO_BRIDGE, NF_BR_POST_ROUTING, skb, NULL, skb->dev, br_dev_queue_push_xmit);
}
{C}
br_dev_queue_push_xmit
1
2
3
4
5
6
7
8
9
10
11
12
int br_dev_queue_push_xmit(struct sk_buff *skb)
{
/* ip_fragment doesn't copy the MAC header */
if (nf_bridge_maybe_copy_header(skb) || (packet_length(skb) > skb->dev->mtu && !skb_is_gso(skb))) {
kfree_skb(skb);
} else {
skb_push(skb, ETH_HLEN);
br_drop_fake_rtable(skb);
dev_queue_xmit(skb); //發送到鏈路層
}
return 0;
}
Skb進入dev_queue_xmit就會調用相應設備驅動的發送函數。也就出了bridge邏輯。所以整個3.10kernel的bridge轉發邏輯如下圖所示：
注意，和2.6kernel一樣，bridge的OUTPUT hook點在bridge dev的發送函數中，這里不再分析列出。
?
工商網監
評論