commit ee6f2dceac6c0bb7bed2f4321111b82cce298f22 Author: Thomas Gschwantner Date: Thu Jul 5 23:33:35 2018 +0200 Test diff --git a/src/device.c b/src/device.c index 89944ca..3e99110 100644 --- a/src/device.c +++ b/src/device.c @@ -326,6 +326,15 @@ static int newlink(struct net *src_net, struct net_device *dev, struct nlattr *t if (ret < 0) goto error_8; + /* + void netif_napi_add(struct net_device *dev, + struct napi_struct *napi, + int (*poll)(struct napi_struct *, int), + int weight); + */ + struct napi_struct *napi = kzalloc(sizeof *napi, GFP_KERNEL); + netif_napi_add(dev, napi, wg_poll, 64); + ret = register_netdevice(dev); if (ret < 0) goto error_9; diff --git a/src/peer.c b/src/peer.c index 9669158..ae931ca 100644 --- a/src/peer.c +++ b/src/peer.c @@ -85,6 +85,7 @@ void peer_remove(struct wireguard_peer *peer) if (unlikely(!peer)) return; lockdep_assert_held(&peer->device->device_update_lock); + netif_napi_del(&peer->napi); allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, &peer->device->device_update_lock); pubkey_hashtable_remove(&peer->device->peer_hashtable, peer); skb_queue_purge(&peer->staged_packet_queue); diff --git a/src/peer.h b/src/peer.h index 867b9c3..088a6ee 100644 --- a/src/peer.h +++ b/src/peer.h @@ -57,6 +57,7 @@ struct wireguard_peer { struct rcu_head rcu; struct list_head peer_list; u64 internal_id; + struct napi_struct napi; }; struct wireguard_peer *peer_create(struct wireguard_device *wg, const u8 public_key[NOISE_PUBLIC_KEY_LEN], const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); diff --git a/src/queueing.h b/src/queueing.h index 0057cfa..a493f1f 100644 --- a/src/queueing.h +++ b/src/queueing.h @@ -24,6 +24,7 @@ void packet_queue_free(struct crypt_queue *queue, bool multicore); struct multicore_worker __percpu *packet_alloc_percpu_multicore_worker(work_func_t function, void *ptr); /* receive.c APIs: */ +int wg_poll(struct napi_struct *napi, int budget); void packet_receive(struct wireguard_device *wg, struct sk_buff *skb); void packet_handshake_receive_worker(struct work_struct *work); /* Workqueue workers: */ @@ -138,6 +139,15 @@ static inline void queue_enqueue_per_peer(struct crypt_queue *queue, struct sk_b peer_put(peer); } +static inline void queue_enqueue_per_peer_napi(struct sk_buff *skb, enum packet_state state) +{ + struct wireguard_peer *peer = peer_rcu_get(PACKET_PEER(skb)); + + atomic_set(&PACKET_CB(skb)->state, state); + napi_schedule(&peer->napi); + peer_put(peer); +} + #ifdef DEBUG bool packet_counter_selftest(void); #endif diff --git a/src/receive.c b/src/receive.c index 7dc3a61..557d0af 100644 --- a/src/receive.c +++ b/src/receive.c @@ -368,6 +368,52 @@ packet_processed: dev_kfree_skb(skb); } +int wg_poll(struct napi_struct *napi, int budget) { + struct wireguard_peer *peer; + struct noise_keypair *keypair; + struct sk_buff *skb; + struct endpoint endpoint; + enum packet_state state; + bool free; + int i; + + for (i = 1; i <= budget; ++i) { + skb = napi->skb; + state = atomic_read(&PACKET_CB(skb)->state); + peer = PACKET_PEER(skb); + keypair = PACKET_CB(skb)->keypair; + free = true; + + if (unlikely(state != PACKET_STATE_CRYPTED)) + goto next; + + if (unlikely(!counter_validate(&keypair->receiving.counter, + PACKET_CB(skb)->nonce))) { + net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", + peer->device->dev->name, PACKET_CB(skb)->nonce, + keypair->receiving.counter.receive.counter); + goto next; + } + + if (unlikely(socket_endpoint_from_skb(&endpoint, skb))) + goto next; + + skb_reset(skb); + packet_consume_data_done(skb, &endpoint); + free = false; + +next: + noise_keypair_put(keypair); + peer_put(peer); + if (unlikely(free)) + dev_kfree_skb(skb); + + napi = container_of(napi->poll_list.next, struct napi_struct, poll_list); + } + + return i; +} + void packet_rx_worker(struct work_struct *work) { struct crypt_queue *queue = container_of(work, struct crypt_queue, work); @@ -445,9 +491,10 @@ static void packet_consume_data(struct wireguard_device *wg, struct sk_buff *skb if (likely(!ret)) return; /* Successful. No need to drop references below. */ - if (ret == -EPIPE) - queue_enqueue_per_peer(&peer->rx_queue, skb, PACKET_STATE_DEAD); - else { + if (ret == -EPIPE) { + /*queue_enqueue_per_peer(&peer->rx_queue, skb, PACKET_STATE_DEAD);*/ + queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); + } else { peer_put(peer); noise_keypair_put(PACKET_CB(skb)->keypair); dev_kfree_skb(skb);