From e4efea3b2e7661098e38ca9a7608b5c1dd1211db Mon Sep 17 00:00:00 2001
From: Simon Rozman <simon@rozman.si>
Date: Thu, 21 Oct 2021 13:52:27 +0200
Subject: driver: accumulate up to 16 receive NBLs before indicating

NdisMIndicateReceiveNetBufferLists() allows batch processing of NBLs and
we can see in the ring whether there are more packets to follow or not.

Signed-off-by: Simon Rozman <simon@rozman.si>
---
 driver/wintun.c | 83 ++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 65 insertions(+), 18 deletions(-)

diff --git a/driver/wintun.c b/driver/wintun.c
index f3eea05..69225ee 100644
--- a/driver/wintun.c
+++ b/driver/wintun.c
@@ -167,6 +167,11 @@ typedef struct _TUN_CTX
             HANDLE Thread;
             KSPIN_LOCK Lock;
             struct
+            {
+                NET_BUFFER_LIST *Head, *Tail;
+                ULONG Count;
+            } PendingNbls;
+            struct
             {
                 NET_BUFFER_LIST *Head, *Tail;
                 KEVENT Empty;
@@ -438,6 +443,46 @@ TunReturnNetBufferLists(NDIS_HANDLE MiniportAdapterContext, PNET_BUFFER_LIST Net
     InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifInErrors, ErrorPacketsCount);
 }
 
+_IRQL_requires_(DISPATCH_LEVEL)
+static VOID
+TunFlushPendingReceiveData(_Inout_ TUN_CTX *Ctx)
+{
+    KLOCK_QUEUE_HANDLE LockHandle;
+    KeAcquireInStackQueuedSpinLock(&Ctx->Device.Receive.Lock, &LockHandle);
+    if (Ctx->Device.Receive.ActiveNbls.Head)
+        NET_BUFFER_LIST_NEXT_NBL_EX(Ctx->Device.Receive.ActiveNbls.Tail) = Ctx->Device.Receive.PendingNbls.Head;
+    else
+    {
+        KeClearEvent(&Ctx->Device.Receive.ActiveNbls.Empty);
+        Ctx->Device.Receive.ActiveNbls.Head = Ctx->Device.Receive.PendingNbls.Head;
+    }
+    Ctx->Device.Receive.ActiveNbls.Tail = Ctx->Device.Receive.PendingNbls.Tail;
+    KeReleaseInStackQueuedSpinLock(&LockHandle);
+
+    NdisMIndicateReceiveNetBufferLists(
+        Ctx->MiniportAdapterHandle,
+        Ctx->Device.Receive.PendingNbls.Head,
+        NDIS_DEFAULT_PORT_NUMBER,
+        Ctx->Device.Receive.PendingNbls.Count,
+        NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL | NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE);
+    Ctx->Device.Receive.PendingNbls.Head = NULL;
+    Ctx->Device.Receive.PendingNbls.Count = 0;
+}
+
+_IRQL_requires_max_(DISPATCH_LEVEL)
+static VOID
+TunDropPendingReceiveData(_Inout_ TUN_CTX *Ctx)
+{
+    while (Ctx->Device.Receive.PendingNbls.Head)
+    {
+        NET_BUFFER_LIST *NblNext = NET_BUFFER_LIST_NEXT_NBL(Ctx->Device.Receive.PendingNbls.Head);
+        NdisFreeNetBufferList(Ctx->Device.Receive.PendingNbls.Head);
+        Ctx->Device.Receive.PendingNbls.Head = NblNext;
+    }
+    InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifInDiscards, Ctx->Device.Receive.PendingNbls.Count);
+    Ctx->Device.Receive.PendingNbls.Count = 0;
+}
+
 _IRQL_requires_max_(PASSIVE_LEVEL)
 _Function_class_(KSTART_ROUTINE)
 static VOID
@@ -491,6 +536,7 @@ TunProcessReceiveData(_Inout_ TUN_CTX *Ctx)
                 KeClearEvent(Ctx->Device.Receive.TailMoved);
             }
         }
+    nextNbl:
         if (RingTail >= RingCapacity)
             break;
 
@@ -542,31 +588,32 @@ TunProcessReceiveData(_Inout_ TUN_CTX *Ctx)
         KIRQL Irql = ExAcquireSpinLockShared(&Ctx->TransitionLock);
         if (!ReadAcquire(&Ctx->Running))
             goto cleanupNbl;
+        if (Ctx->Device.Receive.PendingNbls.Head &&
+            NET_BUFFER_LIST_INFO(Ctx->Device.Receive.PendingNbls.Head, NetBufferListFrameType) != (PVOID)NblProto)
+            TunFlushPendingReceiveData(Ctx);
 
-        KLOCK_QUEUE_HANDLE LockHandle;
-        KeAcquireInStackQueuedSpinLock(&Ctx->Device.Receive.Lock, &LockHandle);
-        if (Ctx->Device.Receive.ActiveNbls.Head)
-            NET_BUFFER_LIST_NEXT_NBL_EX(Ctx->Device.Receive.ActiveNbls.Tail) = Nbl;
+        if (Ctx->Device.Receive.PendingNbls.Head)
+            NET_BUFFER_LIST_NEXT_NBL(Ctx->Device.Receive.PendingNbls.Tail) =
+                NET_BUFFER_LIST_NEXT_NBL_EX(Ctx->Device.Receive.PendingNbls.Tail) = Nbl;
         else
+            Ctx->Device.Receive.PendingNbls.Head = Nbl;
+        Ctx->Device.Receive.PendingNbls.Tail = Nbl;
+        Ctx->Device.Receive.PendingNbls.Count++;
+        if (Ctx->Device.Receive.PendingNbls.Count >= 16 || RingHead == (RingTail = ReadULongAcquire(&Ring->Tail)))
         {
-            KeClearEvent(&Ctx->Device.Receive.ActiveNbls.Empty);
-            Ctx->Device.Receive.ActiveNbls.Head = Nbl;
+            TunFlushPendingReceiveData(Ctx);
+            ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql);
+            continue;
+        }
+        else
+        {
+            ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql);
+            goto nextNbl;
         }
-        Ctx->Device.Receive.ActiveNbls.Tail = Nbl;
-        KeReleaseInStackQueuedSpinLock(&LockHandle);
-
-        NdisMIndicateReceiveNetBufferLists(
-            Ctx->MiniportAdapterHandle,
-            Nbl,
-            NDIS_DEFAULT_PORT_NUMBER,
-            1,
-            NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL | NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE);
-
-        ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql);
-        continue;
 
     cleanupNbl:
         ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql);
+        TunDropPendingReceiveData(Ctx);
         NdisFreeNetBufferList(Nbl);
     cleanupMdl:
         IoFreeMdl(Mdl);
-- 
cgit v1.2.3-59-g8ed1b