diff options
| author | Eric Dumazet <edumazet@google.com> | 2023-03-17 16:20:02 +0000 |
|---|---|---|
| committer | David S. Miller <davem@davemloft.net> | 2023-03-19 10:57:54 +0000 |
| commit | 105a201ebf3312990b96c4fbaade22e31402f8cc (patch) | |
| tree | 2a80ad1832af132b1c1b6555718abaf1750cc30f /net/packet | |
| parent | 72abf2179969a54c16c03b1649b922ec179b364a (diff) | |
net/packet: remove po->xmit
Use PACKET_SOCK_QDISC_BYPASS atomic bit instead of a pointer.
This removes one indirect call in fast path,
and READ_ONCE()/WRITE_ONCE() annotations as well.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Suggested-by: Willem de Bruijn <willemb@google.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/packet')
| -rw-r--r-- | net/packet/af_packet.c | 24 | ||||
| -rw-r--r-- | net/packet/internal.h | 2 |
2 files changed, 10 insertions, 16 deletions
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 7b9367b233d3..497193f73030 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -270,8 +270,11 @@ static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb) } #endif -static int packet_direct_xmit(struct sk_buff *skb) +static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb) { + if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS)) + return dev_queue_xmit(skb); + #ifdef CONFIG_NETFILTER_EGRESS if (nf_hook_egress_active()) { skb = nf_hook_direct_egress(skb); @@ -305,12 +308,6 @@ static void packet_cached_dev_reset(struct packet_sock *po) RCU_INIT_POINTER(po->cached_dev, NULL); } -static bool packet_use_direct_xmit(const struct packet_sock *po) -{ - /* Paired with WRITE_ONCE() in packet_setsockopt() */ - return READ_ONCE(po->xmit) == packet_direct_xmit; -} - static u16 packet_pick_tx_queue(struct sk_buff *skb) { struct net_device *dev = skb->dev; @@ -2872,8 +2869,7 @@ tpacket_error: packet_inc_pending(&po->tx_ring); status = TP_STATUS_SEND_REQUEST; - /* Paired with WRITE_ONCE() in packet_setsockopt() */ - err = READ_ONCE(po->xmit)(skb); + err = packet_xmit(po, skb); if (unlikely(err != 0)) { if (err > 0) err = net_xmit_errno(err); @@ -3076,8 +3072,8 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) virtio_net_hdr_set_proto(skb, &vnet_hdr); } - /* Paired with WRITE_ONCE() in packet_setsockopt() */ - err = READ_ONCE(po->xmit)(skb); + err = packet_xmit(po, skb); + if (unlikely(err != 0)) { if (err > 0) err = net_xmit_errno(err); @@ -3359,7 +3355,6 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, init_completion(&po->skb_completion); sk->sk_family = PF_PACKET; po->num = proto; - po->xmit = dev_queue_xmit; err = packet_alloc_pending(po); if (err) @@ -4010,8 +4005,7 @@ packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, if (copy_from_sockptr(&val, optval, sizeof(val))) return -EFAULT; - /* Paired with all lockless reads of po->xmit */ - WRITE_ONCE(po->xmit, val ? packet_direct_xmit : dev_queue_xmit); + packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val); return 0; } default: @@ -4126,7 +4120,7 @@ static int packet_getsockopt(struct socket *sock, int level, int optname, val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF); break; case PACKET_QDISC_BYPASS: - val = packet_use_direct_xmit(po); + val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS); break; default: return -ENOPROTOOPT; diff --git a/net/packet/internal.h b/net/packet/internal.h index e793e99646f1..27930f69f368 100644 --- a/net/packet/internal.h +++ b/net/packet/internal.h @@ -128,7 +128,6 @@ struct packet_sock { unsigned int tp_tstamp; struct completion skb_completion; struct net_device __rcu *cached_dev; - int (*xmit)(struct sk_buff *skb); struct packet_type prot_hook ____cacheline_aligned_in_smp; atomic_t tp_drops ____cacheline_aligned_in_smp; }; @@ -143,6 +142,7 @@ enum packet_sock_flags { PACKET_SOCK_HAS_VNET_HDR, PACKET_SOCK_RUNNING, PACKET_SOCK_PRESSURE, + PACKET_SOCK_QDISC_BYPASS, }; static inline void packet_sock_flag_set(struct packet_sock *po, |