aboutsummaryrefslogblamecommitdiffstats
path: root/drivers/net/virtio_net.c
blob: 4d7d5434cc5ded2eb7ed18ebf96af7cc9b4cb657 (plain) (tree)
1
2
3
4
5
6
7
8
9
                                            
                                 

                                                                       



                              
                          


                             
                      
                            
                              
                          
                       
                      
                          
                         
                         
                      
                    
                             
 
                                          

                                     
                                                    

                               
                                  
 
                           
                                                             
                           
 

                                                   


                                                                             



                                      

                              



                                                                               
   
                            
 
                                      
 



                                               

                               
  
 


                                   

  



                                    

                         
                  

  

                                    

                    
                  



                          
                  


                                                                    
                                                                    

                                                                 



                                                                
                                                         


                                                                 






                                                                 
                                                         




                                                                 






                                                         


                                                   
 

                                      
                                






                                                          

                                

                                        

                                      


                                             
                                                                  
                                            
 


                                                     

                                                         
 


                                                                

                                                      

                                    

  






                                                    
                       
                            

  



                                   

                                 

                            





                                                           


                                                               


                                                     


                                                                              


                                   


                                                                              


                                       


                                                                


                                                  

                                                           
 


                                                     
 
                                 



                              

                                     
                                             


                                                   

  
                        
                                            
          


                                                                              
           
                        

  














                                                                           




                                                              
                                   








                                       
                             






                          
                                                                                
 
                                                          

 



                                                                
                                                                   
 
                         
 
                                                                  
                                                                          

                                                

 
                                                                        
 
                                   
 
                
                                                      


                                                                   



                                         














                                                                        





                                                          

 
                                               
 
                                                 
                                                            
 
                                          
                                 
 




                                                                       

 
















                                                                                      
                                     

                                                            
                                                                          

                                                                           

                            
                                             
                                                   
                
 
                                        
 
                                                                          
                                                       

                            
 
                                
 

                                  
                                              
            
                                                                
 

                                        
 
                       

                                 
 


                                         
                                   
 

                       
 







                                                                             






                                                                   
                                                                           


                                   
                                    
                     



                                                                                


                                                    
 
                 
                                     
 

                   
 


                                                          
 
                                             

                


                                                                
 




                                                                       
                                                         
                         
                                    
                                   
 
                                                   
 

                                                                       
                          
                                                               
 
                 

 







                                                                             
                                                   
                                                                        

                                                   

                                          

                              

                        
                      
                      
                     
                  

              



                                                                   






                                                     


                          
 
                                                                       
                                                                 











                                                                  
         









                                                           
                        
 



                                                                               

                                                 

                                     

                                        
                                 
                                               
 
                   

 




                                                                 





























                                                                              
                                                                              













                                                                              
                                                                 

















                                                           


                                                              
                                                          
                                                      
                                                            
                                                                    
 
                            
                                  
                                                       



                                                                             
                                                   
                               
                              

                
                           
                            
 


                                                 
                                                                           
                                       
                                    
                                

                        
                                                
                                     
 




















                                                                                

                                                                         
                                                
                                              
                                       
                                     
                                                       
                                     
 

                              
                                                                               
                                                     
                                                      

                              
                                        


                                                          

                                                                 
                                                                            

                                             
                                                   


                                          
                                               
                                                                   

                                             
                                                      

                                          
                        
                                                         
                                          


                                                                    




                                     

                                     
                               


                                           
                          





                                                                    
                   


                          

                           
                       

                    


                                                          
                                                           

                                                            
                                                    
                                                                  

                                

                                                               
 
                                          





                           
                       



                             
                                                                
                                                                 
                                                                  

                                                   
                                                          
                                                                
                                                                        
 

                                                                  

                                                   


                                            
                                                               
                
 
                        
                                          
 


                                                 
                                       
                                      
                                    
                           

                        






                                                                      





                                                                     

                                                                    
                                                    
                                                                   


                                                                          

                                             
                                                     

                                        

                 


                                                                            
                                                       
                                                                               
                                              
                                                
                                                              

                                       
                                                       
                                     
 

                              






                                                                             


                                                                           
                                                                    




                                                                        

                                                                         

                                                

                              
                                        


                                                          

                                                                 
                                                                            



                                                               
                                                   
                                                       
                                               

                                          
                                  
                                               





                                                                   
                                                      
                                                       
                                               

                                          
                        
                                                         
                                          

                                                                    
                                          
                              

                                                          
                                     
                 

                          
 

                                                  
                                                                       



                                                             
 
                                                                               
                            
 

                                
                           

                                  
                                                                
                                     
                                                                                
                                                    

                                                                    

                                                      
                 
 
                                    
                                              


                                                          
                                                                               



                                                                     

                                                               

                                                                        


                                             










                                                                       
                                                       
                 
                                                  


                                                                              
                                                            

                                                                      
                                                               
                 

         
                                                              

                        

                          
                           

                       
                               

                                                      




                                                                      
                                    
                                              
                               
         
        
                       
                                
         
                    

 

                                                                          
                                               
                                                       
 
                                         
                            
                                             
 
                                                     

                                                                  
                                            
                                                         
                                             
                                            
                        
                                                         
                 
                       
         
 
                                  
                                                                             
                                               
                                 
                                                                
            
                                                                                 

                           
                       
 
                                
 
                                                         
                                                      
 





                                                                         

         
                                                 



                                                                
                                         
               


                                     


                           




                                                              

                                                                               
 

                                                       
                                                             
                                                        
                                                                                
                
 


                                                                  
                               
 




                                                                          
                                                                        
                    
                                                 

                   
 

                                                                             
 



                                         

                                                 
                                                            
                                                 
                                            

                                 
                                                     
                                       
                 
                                                                       
 



                                                             
 
                                    
                     
                                     



                                
                                                      

                                                                                
 
                                                    
                                                
                                                               


                                             

                                                                    
                    
                                      

                   

 
                                                                   

                                                                           
 
                                                                       

                         



                                                                             
                                                                      
 
                                          

 

                                                                     
 
                                                       
                                                         

                                                                              
                  
                  
                
                               
 





                                                                         
                               
 
                                                                          
                                                                           
                                   
                                         
                                                     
                                

                                                                               
                                      
                   


                                           
 
                                      
                                                  
                                                                        
                    
                                                 
 

                   
 






                                                                      

                                                                            

                
                 
 

                                          
                                                                 
                                         
                                                           
                    
                                                             
 
                                     
                        
                              
                                   

                                                                         
                                  


                                                       
                    

 
                                                

                                                  
                                                        
 
                                                

 
                                                                               
 
                          

                                                                              





                                                                               

 

















                                                                              





                                                             

                                                 

                                                                     
                         

              
                                                    
                                                      
 
                                        
                                                                 
                                                       
 





                                                                             

 

                                                                
 
                                                     
                                           
                         
                  
              
 
                                                        

                          
                                                
                                                                           
                                                                             
                                        

                 
                                                
                                                                         
                                                                              
                                        
                 

         
                                                                                                 
                                                       
                                                              
         
 
                                                 



                                                                

                                                          
         
                                               
 
                             

 
                                                                   
 
                         

                                 
                  
 


                                                                 
 
                                                       
 



                                                                  
 



                                                







                                                             



                                                 

 









                                                                   






                                                                       
                                                                   


                                      
                                             






                                                  



                                                               

                                                     
                              
                                  
 

                                 
                                                          
 
                             

                                                                
 



                                        
                                        




                                                                                 
         
 


                        


                                                   
                   
 


                                                                            
                                                                       
                                                                      




                                                                   






                                                                             
                                                                   
                                                                          




                 



                                                                            

                                            
 






                                                           
                                                     
                                     









                                                  
                                                               
 
                                             
                                                                         
                                                     
                   
                                       
                      
 
                                                                






                                                                        
                                                                               

                                        
 
                                                   

                                                                             
                      
 
                                  
                                     
 
                                                                              


                                                                

                                         



                                                                              



                                                                    
         
                                                                             

 
                                                                          

                                                   

                                              
                
                                                                  
                                        
                                        
 
                                                                       
                                      
 


                                                    


                                          
                             
                                
 
                                     
                            


                                            

                                                                           
                                        
                                       
                                    
         
 
                                                             

                                
                                 
         
 








                                                                               
           
                                                 
                                               

                                                                     
                                                                         
                                                      
                                                                  
                                                                
                                                             

                         
         
 






                                                                                 
 
                            

 


                                                                     
                                          

                                                                           
                                                         
 
                                              
                                  

                                       
                                                                    
 


                                    
                        
                                                                 
                              
 

                                     
 
                                
                                                                        
                             
 
                                              
                                                                    
 
                                               
                                                         



                                                                             

                                                   

                            
                                                 

 



                                                                   
                
                              
                              
 


                                                               
                                                     

                               

                                                     
                
                         
 


                                                                   
                                                                               

                                                                               

                                      
                 

                                                                   






                                                                                


                                           
                
 


                    

 

                                                        

                                                   
                           
              
 
                                                   
                                                               

                                                      

                    



                                                                             

                    
                                                                            


                                                     
                                                                             




                                            
                                          


                                                
                                                        

                                                            

 



                                                               
                                                                      



                                                                          
                                                                        

                              




                                                                           

                                                                              

                                                         
                                                                          


                                                                             
                
                                                   


                                                                            
         



                 









                                                                       


                                                   
              
 

                                                           
 
                                                   
                                                       
                                              
                                                         
         
 


                 


                                                       
                                 
                                             
                                  
                     
                     

                  
 
                                                                            


                                                                

                                                                
 
                                                                       

                                                         
                                                                  
                                                                       
                                                           
 
                                                                         

                                                         
                                                                   
                                                                        
                                                            
 
                                        
                                        
                                                        


                                                                   
                 
                       
 

                             
                                                                         
                                                                
              
                                        
                                                                    

                                    
                                                                      

                                                   
                                                        
 
                                                                
              

                                                                    

                                    
                                                                      

                                                          
                                                                     
                                                                         

                   

 

                                                          



                                                   
                                                       
                                                                

                                                           
                                                                 
                                                                        
                 

 

                                                           



                                                   
                                                       
                                                                

                                                           
                                                                 
                                                                         
                 

 
                                                           


              

                                                           

                                                                   

                 

                                              
 
 

                                                         







                                                     
                                           
                       

         





                                                               
 











                                                                             

         
                                     
                               

 
                                                                        
 




                                                                             
 






                                                                             
 




                                                                             
                                   
























                                                                            

 




                                                               

                                                                      

                                                

 












                                                                               













                                                                              
                                                                  

                               






                                                                             
                          
                                                   



                                                               
                                         
         
                          



                   


















































                                                                                
                                              
























                                                                              












                                                                   
                                                                         

                                                                      
 

                                                   
 


                                                                           













                                                                       

 

                                                                               



                                                   
                                

                                                 
                                                         


                                               
                                      



                 

                                                                         


                                                   


                                      



                 














































                                                                         







                                                         

















                                                                           
                                                       
                                           
                                        
                                               


                                                       

                                             
                                              

                                                         

                                             

  







                                                               
                                  
                                     
                                    


                                              
                                                           
                                                      
                                                                 
                 




















                                                                       
                                                           
                                                                           


                                                                

         
                                  
                                     
                                    


                   


                                                                            
                                                                 
 
                                                                          


                                                                             
                                                                        












                                                                








                                                                  



                                                        

                                                                         



                                                                              
                                
                   
 



                                                                           


                                                                                                                        



                                                          
                                                                                                             



                                
                                                                          



                                                                             





                                                             
                                                                              




                                                                     



                                                        

                                                            
 
                                                                   

                                                           
                                                      


                                                                 
 







                                                                     
 


                                                        
                                                            
                                     
 



                                                                     
                                                                 
                 


                                                   

                                               
                                         
                                                                           


                                                                


                 
 
    





                                                                         
                                 
                                                           
                                                                           


                                                                
         


                                                            

 
                                                    

                                                   
                                        


                                                   


                                                                
         
                 

 
                                                                      


                               
                                                                    
                            
                                                      





                               















                                                                        
























                                                               




                                                     
                                                       
                                                   
                                             

                                                         
                                              
                                                   
                                                          
                                                             
                                                       

  
                                                                 
 

                                                                     

              

                                                                           
                       

                                        
                                             

                                              




                                                 
                       



                                                
                                            
                                          
                                                  

                                           
                                                  






                                                              
                                        

 

                                                        

              

                                                   
                                                
                                                
         
 




                                                                       

                      
                        

 
                                                       
 
                                  




                                                                            




                                                                
         





                                                      
                      

 







                                                            






                                                     
                                                                         
                                               

                                                   
                                                                  
                 





                                                                         
                                                    
                                                                 
                                                     
                                                            
                                
                                                                 
                         
                 


         



                                                    
                                   
 
                                    

                                

 











                                                                                        

                                                       

 
                                                    
 




                                  
                  








                                                                              
                                                           

                            
                                                                             

                                  
                                                                     

                               
                                                        
                                                                   




                                     





                                                      
 







                                                                        

                                              
         
 
                                                                             
                                                           

                              
 

                                             
                                                                         
                                                                         
         


                                                   
                                                                                


                                              
                                 
 

         

                   












                                                        


                                                          
                                                                           

                            
                                                                           
                    






                                                                      

                                                                            

                                                                      
                                                              
                                                                      


                                                       






                      

                        















                                            
                          
                                 

                          





                                

 

                                                                           
                          


                                                                    

                                                                              
                                 


                                                   
                                   

                                                                             















                                                                     

































                                                                              


                           
                                                       
 





                                                                           


                                             












                                                                           
                             




                                
                                                                


                                                                          





                                                                       

                                                                        
                                                                              



                                              
                                                                  
                                          
                                        
 
                                                


                                                 
                                                          
                                                                
                                                                 
                         
                                                                      

                                                                 
                                                       

                                                                 
                                                                    





                                                                     
 

                                                    
                        
                                                                            
                                                                            
         

                                                              




                                                                       
 

                                           



                                   
                                                                           




                                                                           
                                        


                                                    

                        
                        
 
                                                                 
 
                                                                             

                                                                

                                                               

                                       


                                                             

                                                               



                                                                      

                                                            

                                         


                                                           



                                                                       
                                         


                                                                             


                                                                                       
                                  
                 
 


                                   


                                                                

         

                                                   
 




                                                         


                                                                       
                           
                
                          
 



                                                                     

                                                                
 

                                   

                                                             

                                                    
                                      
                 

         


                                                                    
                                   
         
 

                                  
                                        

                                                                          
                                            

         
                                                     
 

                                                             
                               
                                                                
                                                

                                                  
                                            

                                      
 


                                                                        
                                                        
 


                                                                          

                 
                       

                                      
                               

                                           
         
                                              
                                    
                            




                         
                                                     
 
                                          

                                                                
                             
 
                              
 

                                    
                            

 
                                                      


                                             
                                     
 

                                                                
 

                                   

                                           
                             
 
                             

 
                                                                    


                                             
                                     
                                  




                             
                                                                     

                                             
                
 
                                       

                           

                                                     
                                        


                           

                 
 




                                             









                                                                                  
                                                             
                                                       
 
                                  





                                         
                            

  
                                                 

                                                   

                                                                 


                                       
                                         
                                      
                                       
                                                 
                      


                                        

  



                                              
                                                                               




                                                             
                                                                             


















                                                              
                                                     

                                                      

                                    



                                            
// SPDX-License-Identifier: GPL-2.0-or-later
/* A network driver using virtio.
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/scatterlist.h>
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/average.h>
#include <linux/filter.h>
#include <linux/kernel.h>
#include <net/route.h>
#include <net/xdp.h>
#include <net/net_failover.h>

static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);

static bool csum = true, gso = true, napi_tx = true;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);
module_param(napi_tx, bool, 0644);

/* FIXME: MTU in config. */
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
#define GOOD_COPY_LEN	128

#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)

/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
#define VIRTIO_XDP_HEADROOM 256

/* Separating two types of XDP xmit */
#define VIRTIO_XDP_TX		BIT(0)
#define VIRTIO_XDP_REDIR	BIT(1)

#define VIRTIO_XDP_FLAG	BIT(0)

/* RX packet size EWMA. The average packet size is used to determine the packet
 * buffer size when refilling RX rings. As the entire RX ring may be refilled
 * at once, the weight is chosen so that the EWMA will be insensitive to short-
 * term, transient changes in packet size.
 */
DECLARE_EWMA(pkt_len, 0, 64)

#define VIRTNET_DRIVER_VERSION "1.0.0"

static const unsigned long guest_offloads[] = {
	VIRTIO_NET_F_GUEST_TSO4,
	VIRTIO_NET_F_GUEST_TSO6,
	VIRTIO_NET_F_GUEST_ECN,
	VIRTIO_NET_F_GUEST_UFO,
	VIRTIO_NET_F_GUEST_CSUM
};

struct virtnet_stat_desc {
	char desc[ETH_GSTRING_LEN];
	size_t offset;
};

struct virtnet_sq_stats {
	struct u64_stats_sync syncp;
	u64 packets;
	u64 bytes;
	u64 xdp_tx;
	u64 xdp_tx_drops;
	u64 kicks;
};

struct virtnet_rq_stats {
	struct u64_stats_sync syncp;
	u64 packets;
	u64 bytes;
	u64 drops;
	u64 xdp_packets;
	u64 xdp_tx;
	u64 xdp_redirects;
	u64 xdp_drops;
	u64 kicks;
};

#define VIRTNET_SQ_STAT(m)	offsetof(struct virtnet_sq_stats, m)
#define VIRTNET_RQ_STAT(m)	offsetof(struct virtnet_rq_stats, m)

static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
	{ "packets",		VIRTNET_SQ_STAT(packets) },
	{ "bytes",		VIRTNET_SQ_STAT(bytes) },
	{ "xdp_tx",		VIRTNET_SQ_STAT(xdp_tx) },
	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
};

static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
	{ "packets",		VIRTNET_RQ_STAT(packets) },
	{ "bytes",		VIRTNET_RQ_STAT(bytes) },
	{ "drops",		VIRTNET_RQ_STAT(drops) },
	{ "xdp_packets",	VIRTNET_RQ_STAT(xdp_packets) },
	{ "xdp_tx",		VIRTNET_RQ_STAT(xdp_tx) },
	{ "xdp_redirects",	VIRTNET_RQ_STAT(xdp_redirects) },
	{ "xdp_drops",		VIRTNET_RQ_STAT(xdp_drops) },
	{ "kicks",		VIRTNET_RQ_STAT(kicks) },
};

#define VIRTNET_SQ_STATS_LEN	ARRAY_SIZE(virtnet_sq_stats_desc)
#define VIRTNET_RQ_STATS_LEN	ARRAY_SIZE(virtnet_rq_stats_desc)

/* Internal representation of a send virtqueue */
struct send_queue {
	/* Virtqueue associated with this send _queue */
	struct virtqueue *vq;

	/* TX: fragments + linear part + virtio header */
	struct scatterlist sg[MAX_SKB_FRAGS + 2];

	/* Name of the send queue: output.$index */
	char name[40];

	struct virtnet_sq_stats stats;

	struct napi_struct napi;
};

/* Internal representation of a receive virtqueue */
struct receive_queue {
	/* Virtqueue associated with this receive_queue */
	struct virtqueue *vq;

	struct napi_struct napi;

	struct bpf_prog __rcu *xdp_prog;

	struct virtnet_rq_stats stats;

	/* Chain pages by the private ptr. */
	struct page *pages;

	/* Average packet length for mergeable receive buffers. */
	struct ewma_pkt_len mrg_avg_pkt_len;

	/* Page frag for packet buffer allocation. */
	struct page_frag alloc_frag;

	/* RX: fragments + linear part + virtio header */
	struct scatterlist sg[MAX_SKB_FRAGS + 2];

	/* Min single buffer size for mergeable buffers case. */
	unsigned int min_buf_len;

	/* Name of this receive queue: input.$index */
	char name[40];

	struct xdp_rxq_info xdp_rxq;
};

/* Control VQ buffers: protected by the rtnl lock */
struct control_buf {
	struct virtio_net_ctrl_hdr hdr;
	virtio_net_ctrl_ack status;
	struct virtio_net_ctrl_mq mq;
	u8 promisc;
	u8 allmulti;
	__virtio16 vid;
	__virtio64 offloads;
};

struct virtnet_info {
	struct virtio_device *vdev;
	struct virtqueue *cvq;
	struct net_device *dev;
	struct send_queue *sq;
	struct receive_queue *rq;
	unsigned int status;

	/* Max # of queue pairs supported by the device */
	u16 max_queue_pairs;

	/* # of queue pairs currently used by the driver */
	u16 curr_queue_pairs;

	/* # of XDP queue pairs currently used by the driver */
	u16 xdp_queue_pairs;

	/* I like... big packets and I cannot lie! */
	bool big_packets;

	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

	/* Has control virtqueue */
	bool has_cvq;

	/* Host can handle any s/g split between our header and packet data */
	bool any_header_sg;

	/* Packet virtio header size */
	u8 hdr_len;

	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

	/* Work struct for config space updates */
	struct work_struct config_work;

	/* Does the affinity hint is set for virtqueues? */
	bool affinity_hint_set;

	/* CPU hotplug instances for online & dead */
	struct hlist_node node;
	struct hlist_node node_dead;

	struct control_buf *ctrl;

	/* Ethtool settings */
	u8 duplex;
	u32 speed;

	unsigned long guest_offloads;
	unsigned long guest_offloads_capable;

	/* failover when STANDBY feature enabled */
	struct failover *failover;
};

struct padded_vnet_hdr {
	struct virtio_net_hdr_mrg_rxbuf hdr;
	/*
	 * hdr is in a separate sg buffer, and data sg buffer shares same page
	 * with this header sg. This padding makes next sg 16 byte aligned
	 * after the header.
	 */
	char padding[4];
};

static bool is_xdp_frame(void *ptr)
{
	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
}

static void *xdp_to_ptr(struct xdp_frame *ptr)
{
	return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
}

static struct xdp_frame *ptr_to_xdp(void *ptr)
{
	return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}

/* Converting between virtqueue no. and kernel tx/rx queue no.
 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 */
static int vq2txq(struct virtqueue *vq)
{
	return (vq->index - 1) / 2;
}

static int txq2vq(int txq)
{
	return txq * 2 + 1;
}

static int vq2rxq(struct virtqueue *vq)
{
	return vq->index / 2;
}

static int rxq2vq(int rxq)
{
	return rxq * 2;
}

static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
{
	return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
}

/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct receive_queue *rq, struct page *page)
{
	struct page *end;

	/* Find end of list, sew whole thing into vi->rq.pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)rq->pages;
	rq->pages = page;
}

static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
{
	struct page *p = rq->pages;

	if (p) {
		rq->pages = (struct page *)p->private;
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
		p = alloc_page(gfp_mask);
	return p;
}

static void virtqueue_napi_schedule(struct napi_struct *napi,
				    struct virtqueue *vq)
{
	if (napi_schedule_prep(napi)) {
		virtqueue_disable_cb(vq);
		__napi_schedule(napi);
	}
}

static void virtqueue_napi_complete(struct napi_struct *napi,
				    struct virtqueue *vq, int processed)
{
	int opaque;

	opaque = virtqueue_enable_cb_prepare(vq);
	if (napi_complete_done(napi, processed)) {
		if (unlikely(virtqueue_poll(vq, opaque)))
			virtqueue_napi_schedule(napi, vq);
	} else {
		virtqueue_disable_cb(vq);
	}
}

static void skb_xmit_done(struct virtqueue *vq)
{
	struct virtnet_info *vi = vq->vdev->priv;
	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;

	/* Suppress further interrupts. */
	virtqueue_disable_cb(vq);

	if (napi->weight)
		virtqueue_napi_schedule(napi, vq);
	else
		/* We were probably waiting for more output buffers. */
		netif_wake_subqueue(vi->dev, vq2txq(vq));
}

#define MRG_CTX_HEADER_SHIFT 22
static void *mergeable_len_to_ctx(unsigned int truesize,
				  unsigned int headroom)
{
	return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
}

static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
{
	return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
}

static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
{
	return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
}

/* Called from bottom half context */
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct receive_queue *rq,
				   struct page *page, unsigned int offset,
				   unsigned int len, unsigned int truesize,
				   bool hdr_valid)
{
	struct sk_buff *skb;
	struct virtio_net_hdr_mrg_rxbuf *hdr;
	unsigned int copy, hdr_len, hdr_padded_len;
	char *p;

	p = page_address(page) + offset;

	/* copy small packet so we can reuse these pages for small data */
	skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;

	hdr = skb_vnet_hdr(skb);

	hdr_len = vi->hdr_len;
	if (vi->mergeable_rx_bufs)
		hdr_padded_len = sizeof(*hdr);
	else
		hdr_padded_len = sizeof(struct padded_vnet_hdr);

	if (hdr_valid)
		memcpy(hdr, p, hdr_len);

	len -= hdr_len;
	offset += hdr_padded_len;
	p += hdr_padded_len;

	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	skb_put_data(skb, p, copy);

	len -= copy;
	offset += copy;

	if (vi->mergeable_rx_bufs) {
		if (len)
			skb_add_rx_frag(skb, 0, page, offset, len, truesize);
		else
			put_page(page);
		return skb;
	}

	/*
	 * Verify that we can indeed put this data into a skb.
	 * This is here to handle cases when the device erroneously
	 * tries to receive more than is possible. This is usually
	 * the case of a broken device.
	 */
	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
		net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
		dev_kfree_skb(skb);
		return NULL;
	}
	BUG_ON(offset >= PAGE_SIZE);
	while (len) {
		unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
				frag_size, truesize);
		len -= frag_size;
		page = (struct page *)page->private;
		offset = 0;
	}

	if (page)
		give_pages(rq, page);

	return skb;
}

static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
				   struct send_queue *sq,
				   struct xdp_frame *xdpf)
{
	struct virtio_net_hdr_mrg_rxbuf *hdr;
	int err;

	/* virtqueue want to use data area in-front of packet */
	if (unlikely(xdpf->metasize > 0))
		return -EOPNOTSUPP;

	if (unlikely(xdpf->headroom < vi->hdr_len))
		return -EOVERFLOW;

	/* Make room for virtqueue hdr (also change xdpf->headroom?) */
	xdpf->data -= vi->hdr_len;
	/* Zero header and leave csum up to XDP layers */
	hdr = xdpf->data;
	memset(hdr, 0, vi->hdr_len);
	xdpf->len   += vi->hdr_len;

	sg_init_one(sq->sg, xdpf->data, xdpf->len);

	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf),
				   GFP_ATOMIC);
	if (unlikely(err))
		return -ENOSPC; /* Caller handle free/refcnt */

	return 0;
}

static struct send_queue *virtnet_xdp_sq(struct virtnet_info *vi)
{
	unsigned int qp;

	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
	return &vi->sq[qp];
}

static int virtnet_xdp_xmit(struct net_device *dev,
			    int n, struct xdp_frame **frames, u32 flags)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct receive_queue *rq = vi->rq;
	struct bpf_prog *xdp_prog;
	struct send_queue *sq;
	unsigned int len;
	int packets = 0;
	int bytes = 0;
	int drops = 0;
	int kicks = 0;
	int ret, err;
	void *ptr;
	int i;

	/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
	 * indicate XDP resources have been successfully allocated.
	 */
	xdp_prog = rcu_dereference(rq->xdp_prog);
	if (!xdp_prog)
		return -ENXIO;

	sq = virtnet_xdp_sq(vi);

	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
		ret = -EINVAL;
		drops = n;
		goto out;
	}

	/* Free up any pending old buffers before queueing new ones. */
	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
		if (likely(is_xdp_frame(ptr))) {
			struct xdp_frame *frame = ptr_to_xdp(ptr);

			bytes += frame->len;
			xdp_return_frame(frame);
		} else {
			struct sk_buff *skb = ptr;

			bytes += skb->len;
			napi_consume_skb(skb, false);
		}
		packets++;
	}

	for (i = 0; i < n; i++) {
		struct xdp_frame *xdpf = frames[i];

		err = __virtnet_xdp_xmit_one(vi, sq, xdpf);
		if (err) {
			xdp_return_frame_rx_napi(xdpf);
			drops++;
		}
	}
	ret = n - drops;

	if (flags & XDP_XMIT_FLUSH) {
		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
			kicks = 1;
	}
out:
	u64_stats_update_begin(&sq->stats.syncp);
	sq->stats.bytes += bytes;
	sq->stats.packets += packets;
	sq->stats.xdp_tx += n;
	sq->stats.xdp_tx_drops += drops;
	sq->stats.kicks += kicks;
	u64_stats_update_end(&sq->stats.syncp);

	return ret;
}

static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
{
	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
}

/* We copy the packet for XDP in the following cases:
 *
 * 1) Packet is scattered across multiple rx buffers.
 * 2) Headroom space is insufficient.
 *
 * This is inefficient but it's a temporary condition that
 * we hit right after XDP is enabled and until queue is refilled
 * with large buffers with sufficient headroom - so it should affect
 * at most queue size packets.
 * Afterwards, the conditions to enable
 * XDP should preclude the underlying device from sending packets
 * across multiple buffers (num_buf > 1), and we make sure buffers
 * have enough headroom.
 */
static struct page *xdp_linearize_page(struct receive_queue *rq,
				       u16 *num_buf,
				       struct page *p,
				       int offset,
				       int page_off,
				       unsigned int *len)
{
	struct page *page = alloc_page(GFP_ATOMIC);

	if (!page)
		return NULL;

	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
	page_off += *len;

	while (--*num_buf) {
		int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
		unsigned int buflen;
		void *buf;
		int off;

		buf = virtqueue_get_buf(rq->vq, &buflen);
		if (unlikely(!buf))
			goto err_buf;

		p = virt_to_head_page(buf);
		off = buf - page_address(p);

		/* guard against a misconfigured or uncooperative backend that
		 * is sending packet larger than the MTU.
		 */
		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
			put_page(p);
			goto err_buf;
		}

		memcpy(page_address(page) + page_off,
		       page_address(p) + off, buflen);
		page_off += buflen;
		put_page(p);
	}

	/* Headroom does not contribute to packet length */
	*len = page_off - VIRTIO_XDP_HEADROOM;
	return page;
err_buf:
	__free_pages(page, 0);
	return NULL;
}

static struct sk_buff *receive_small(struct net_device *dev,
				     struct virtnet_info *vi,
				     struct receive_queue *rq,
				     void *buf, void *ctx,
				     unsigned int len,
				     unsigned int *xdp_xmit,
				     struct virtnet_rq_stats *stats)
{
	struct sk_buff *skb;
	struct bpf_prog *xdp_prog;
	unsigned int xdp_headroom = (unsigned long)ctx;
	unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
	unsigned int headroom = vi->hdr_len + header_offset;
	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	struct page *page = virt_to_head_page(buf);
	unsigned int delta = 0;
	struct page *xdp_page;
	int err;

	len -= vi->hdr_len;
	stats->bytes += len;

	rcu_read_lock();
	xdp_prog = rcu_dereference(rq->xdp_prog);
	if (xdp_prog) {
		struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
		struct xdp_frame *xdpf;
		struct xdp_buff xdp;
		void *orig_data;
		u32 act;

		if (unlikely(hdr->hdr.gso_type))
			goto err_xdp;

		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
			int offset = buf - page_address(page) + header_offset;
			unsigned int tlen = len + vi->hdr_len;
			u16 num_buf = 1;

			xdp_headroom = virtnet_get_headroom(vi);
			header_offset = VIRTNET_RX_PAD + xdp_headroom;
			headroom = vi->hdr_len + header_offset;
			buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
			xdp_page = xdp_linearize_page(rq, &num_buf, page,
						      offset, header_offset,
						      &tlen);
			if (!xdp_page)
				goto err_xdp;

			buf = page_address(xdp_page);
			put_page(page);
			page = xdp_page;
		}

		xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
		xdp.data = xdp.data_hard_start + xdp_headroom;
		xdp_set_data_meta_invalid(&xdp);
		xdp.data_end = xdp.data + len;
		xdp.rxq = &rq->xdp_rxq;
		orig_data = xdp.data;
		act = bpf_prog_run_xdp(xdp_prog, &xdp);
		stats->xdp_packets++;

		switch (act) {
		case XDP_PASS:
			/* Recalculate length in case bpf program changed it */
			delta = orig_data - xdp.data;
			len = xdp.data_end - xdp.data;
			break;
		case XDP_TX:
			stats->xdp_tx++;
			xdpf = convert_to_xdp_frame(&xdp);
			if (unlikely(!xdpf))
				goto err_xdp;
			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
			if (unlikely(err < 0)) {
				trace_xdp_exception(vi->dev, xdp_prog, act);
				goto err_xdp;
			}
			*xdp_xmit |= VIRTIO_XDP_TX;
			rcu_read_unlock();
			goto xdp_xmit;
		case XDP_REDIRECT:
			stats->xdp_redirects++;
			err = xdp_do_redirect(dev, &xdp, xdp_prog);
			if (err)
				goto err_xdp;
			*xdp_xmit |= VIRTIO_XDP_REDIR;
			rcu_read_unlock();
			goto xdp_xmit;
		default:
			bpf_warn_invalid_xdp_action(act);
			/* fall through */
		case XDP_ABORTED:
			trace_xdp_exception(vi->dev, xdp_prog, act);
		case XDP_DROP:
			goto err_xdp;
		}
	}
	rcu_read_unlock();

	skb = build_skb(buf, buflen);
	if (!skb) {
		put_page(page);
		goto err;
	}
	skb_reserve(skb, headroom - delta);
	skb_put(skb, len);
	if (!delta) {
		buf += header_offset;
		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
	} /* keep zeroed vnet hdr since packet was changed by bpf */

err:
	return skb;

err_xdp:
	rcu_read_unlock();
	stats->xdp_drops++;
	stats->drops++;
	put_page(page);
xdp_xmit:
	return NULL;
}

static struct sk_buff *receive_big(struct net_device *dev,
				   struct virtnet_info *vi,
				   struct receive_queue *rq,
				   void *buf,
				   unsigned int len,
				   struct virtnet_rq_stats *stats)
{
	struct page *page = buf;
	struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len,
					  PAGE_SIZE, true);

	stats->bytes += len - vi->hdr_len;
	if (unlikely(!skb))
		goto err;

	return skb;

err:
	stats->drops++;
	give_pages(rq, page);
	return NULL;
}

static struct sk_buff *receive_mergeable(struct net_device *dev,
					 struct virtnet_info *vi,
					 struct receive_queue *rq,
					 void *buf,
					 void *ctx,
					 unsigned int len,
					 unsigned int *xdp_xmit,
					 struct virtnet_rq_stats *stats)
{
	struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
	struct page *page = virt_to_head_page(buf);
	int offset = buf - page_address(page);
	struct sk_buff *head_skb, *curr_skb;
	struct bpf_prog *xdp_prog;
	unsigned int truesize;
	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
	int err;

	head_skb = NULL;
	stats->bytes += len - vi->hdr_len;

	rcu_read_lock();
	xdp_prog = rcu_dereference(rq->xdp_prog);
	if (xdp_prog) {
		struct xdp_frame *xdpf;
		struct page *xdp_page;
		struct xdp_buff xdp;
		void *data;
		u32 act;

		/* Transient failure which in theory could occur if
		 * in-flight packets from before XDP was enabled reach
		 * the receive path after XDP is loaded.
		 */
		if (unlikely(hdr->hdr.gso_type))
			goto err_xdp;

		/* This happens when rx buffer size is underestimated
		 * or headroom is not enough because of the buffer
		 * was refilled before XDP is set. This should only
		 * happen for the first several packets, so we don't
		 * care much about its performance.
		 */
		if (unlikely(num_buf > 1 ||
			     headroom < virtnet_get_headroom(vi))) {
			/* linearize data for XDP */
			xdp_page = xdp_linearize_page(rq, &num_buf,
						      page, offset,
						      VIRTIO_XDP_HEADROOM,
						      &len);
			if (!xdp_page)
				goto err_xdp;
			offset = VIRTIO_XDP_HEADROOM;
		} else {
			xdp_page = page;
		}

		/* Allow consuming headroom but reserve enough space to push
		 * the descriptor on if we get an XDP_TX return code.
		 */
		data = page_address(xdp_page) + offset;
		xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
		xdp.data = data + vi->hdr_len;
		xdp_set_data_meta_invalid(&xdp);
		xdp.data_end = xdp.data + (len - vi->hdr_len);
		xdp.rxq = &rq->xdp_rxq;

		act = bpf_prog_run_xdp(xdp_prog, &xdp);
		stats->xdp_packets++;

		switch (act) {
		case XDP_PASS:
			/* recalculate offset to account for any header
			 * adjustments. Note other cases do not build an
			 * skb and avoid using offset
			 */
			offset = xdp.data -
					page_address(xdp_page) - vi->hdr_len;

			/* recalculate len if xdp.data or xdp.data_end were
			 * adjusted
			 */
			len = xdp.data_end - xdp.data + vi->hdr_len;
			/* We can only create skb based on xdp_page. */
			if (unlikely(xdp_page != page)) {
				rcu_read_unlock();
				put_page(page);
				head_skb = page_to_skb(vi, rq, xdp_page,
						       offset, len,
						       PAGE_SIZE, false);
				return head_skb;
			}
			break;
		case XDP_TX:
			stats->xdp_tx++;
			xdpf = convert_to_xdp_frame(&xdp);
			if (unlikely(!xdpf))
				goto err_xdp;
			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
			if (unlikely(err < 0)) {
				trace_xdp_exception(vi->dev, xdp_prog, act);
				if (unlikely(xdp_page != page))
					put_page(xdp_page);
				goto err_xdp;
			}
			*xdp_xmit |= VIRTIO_XDP_TX;
			if (unlikely(xdp_page != page))
				put_page(page);
			rcu_read_unlock();
			goto xdp_xmit;
		case XDP_REDIRECT:
			stats->xdp_redirects++;
			err = xdp_do_redirect(dev, &xdp, xdp_prog);
			if (err) {
				if (unlikely(xdp_page != page))
					put_page(xdp_page);
				goto err_xdp;
			}
			*xdp_xmit |= VIRTIO_XDP_REDIR;
			if (unlikely(xdp_page != page))
				put_page(page);
			rcu_read_unlock();
			goto xdp_xmit;
		default:
			bpf_warn_invalid_xdp_action(act);
			/* fall through */
		case XDP_ABORTED:
			trace_xdp_exception(vi->dev, xdp_prog, act);
			/* fall through */
		case XDP_DROP:
			if (unlikely(xdp_page != page))
				__free_pages(xdp_page, 0);
			goto err_xdp;
		}
	}
	rcu_read_unlock();

	truesize = mergeable_ctx_to_truesize(ctx);
	if (unlikely(len > truesize)) {
		pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
			 dev->name, len, (unsigned long)ctx);
		dev->stats.rx_length_errors++;
		goto err_skb;
	}

	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog);
	curr_skb = head_skb;

	if (unlikely(!curr_skb))
		goto err_skb;
	while (--num_buf) {
		int num_skb_frags;

		buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
		if (unlikely(!buf)) {
			pr_debug("%s: rx error: %d buffers out of %d missing\n",
				 dev->name, num_buf,
				 virtio16_to_cpu(vi->vdev,
						 hdr->num_buffers));
			dev->stats.rx_length_errors++;
			goto err_buf;
		}

		stats->bytes += len;
		page = virt_to_head_page(buf);

		truesize = mergeable_ctx_to_truesize(ctx);
		if (unlikely(len > truesize)) {
			pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
				 dev->name, len, (unsigned long)ctx);
			dev->stats.rx_length_errors++;
			goto err_skb;
		}

		num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
		if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
			struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);

			if (unlikely(!nskb))
				goto err_skb;
			if (curr_skb == head_skb)
				skb_shinfo(curr_skb)->frag_list = nskb;
			else
				curr_skb->next = nskb;
			curr_skb = nskb;
			head_skb->truesize += nskb->truesize;
			num_skb_frags = 0;
		}
		if (curr_skb != head_skb) {
			head_skb->data_len += len;
			head_skb->len += len;
			head_skb->truesize += truesize;
		}
		offset = buf - page_address(page);
		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
			put_page(page);
			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
					     len, truesize);
		} else {
			skb_add_rx_frag(curr_skb, num_skb_frags, page,
					offset, len, truesize);
		}
	}

	ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
	return head_skb;

err_xdp:
	rcu_read_unlock();
	stats->xdp_drops++;
err_skb:
	put_page(page);
	while (num_buf-- > 1) {
		buf = virtqueue_get_buf(rq->vq, &len);
		if (unlikely(!buf)) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 dev->name, num_buf);
			dev->stats.rx_length_errors++;
			break;
		}
		stats->bytes += len;
		page = virt_to_head_page(buf);
		put_page(page);
	}
err_buf:
	stats->drops++;
	dev_kfree_skb(head_skb);
xdp_xmit:
	return NULL;
}

static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
			void *buf, unsigned int len, void **ctx,
			unsigned int *xdp_xmit,
			struct virtnet_rq_stats *stats)
{
	struct net_device *dev = vi->dev;
	struct sk_buff *skb;
	struct virtio_net_hdr_mrg_rxbuf *hdr;

	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs) {
			put_page(virt_to_head_page(buf));
		} else if (vi->big_packets) {
			give_pages(rq, buf);
		} else {
			put_page(virt_to_head_page(buf));
		}
		return;
	}

	if (vi->mergeable_rx_bufs)
		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
					stats);
	else if (vi->big_packets)
		skb = receive_big(dev, vi, rq, buf, len, stats);
	else
		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);

	if (unlikely(!skb))
		return;

	hdr = skb_vnet_hdr(skb);

	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
		skb->ip_summed = CHECKSUM_UNNECESSARY;

	if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
				  virtio_is_little_endian(vi->vdev))) {
		net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
				     dev->name, hdr->hdr.gso_type,
				     hdr->hdr.gso_size);
		goto frame_err;
	}

	skb_record_rx_queue(skb, vq2rxq(rq->vq));
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

	napi_gro_receive(&rq->napi, skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

/* Unlike mergeable buffers, all buffers are allocated to the
 * same size, except for the headroom. For this reason we do
 * not need to use  mergeable_len_to_ctx here - it is enough
 * to store the headroom as the context ignoring the truesize.
 */
static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
			     gfp_t gfp)
{
	struct page_frag *alloc_frag = &rq->alloc_frag;
	char *buf;
	unsigned int xdp_headroom = virtnet_get_headroom(vi);
	void *ctx = (void *)(unsigned long)xdp_headroom;
	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
	int err;

	len = SKB_DATA_ALIGN(len) +
	      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
		return -ENOMEM;

	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
	get_page(alloc_frag->page);
	alloc_frag->offset += len;
	sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
		    vi->hdr_len + GOOD_PACKET_LEN);
	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
	if (err < 0)
		put_page(virt_to_head_page(buf));
	return err;
}

static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
			   gfp_t gfp)
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

	sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);

	/* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(rq, gfp);
		if (!first) {
			if (list)
				give_pages(rq, list);
			return -ENOMEM;
		}
		sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);

		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}

	first = get_a_page(rq, gfp);
	if (!first) {
		give_pages(rq, list);
		return -ENOMEM;
	}
	p = page_address(first);

	/* rq->sg[0], rq->sg[1] share the same page */
	/* a separated rq->sg[0] for header - required in case !any_header_sg */
	sg_set_buf(&rq->sg[0], p, vi->hdr_len);

	/* rq->sg[1] for data packet, from offset */
	offset = sizeof(struct padded_vnet_hdr);
	sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);

	/* chain first in list head */
	first->private = (unsigned long)list;
	err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
				  first, gfp);
	if (err < 0)
		give_pages(rq, first);

	return err;
}

static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
					  struct ewma_pkt_len *avg_pkt_len,
					  unsigned int room)
{
	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
	unsigned int len;

	if (room)
		return PAGE_SIZE - room;

	len = hdr_len +	clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
				rq->min_buf_len, PAGE_SIZE - hdr_len);

	return ALIGN(len, L1_CACHE_BYTES);
}

static int add_recvbuf_mergeable(struct virtnet_info *vi,
				 struct receive_queue *rq, gfp_t gfp)
{
	struct page_frag *alloc_frag = &rq->alloc_frag;
	unsigned int headroom = virtnet_get_headroom(vi);
	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
	char *buf;
	void *ctx;
	int err;
	unsigned int len, hole;

	/* Extra tailroom is needed to satisfy XDP's assumption. This
	 * means rx frags coalescing won't work, but consider we've
	 * disabled GSO for XDP, it won't be a big issue.
	 */
	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
		return -ENOMEM;

	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
	buf += headroom; /* advance address leaving hole at front of pkt */
	get_page(alloc_frag->page);
	alloc_frag->offset += len + room;
	hole = alloc_frag->size - alloc_frag->offset;
	if (hole < len + room) {
		/* To avoid internal fragmentation, if there is very likely not
		 * enough space for another buffer, add the remaining space to
		 * the current buffer.
		 */
		len += hole;
		alloc_frag->offset += hole;
	}

	sg_init_one(rq->sg, buf, len);
	ctx = mergeable_len_to_ctx(len, headroom);
	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
	if (err < 0)
		put_page(virt_to_head_page(buf));

	return err;
}

/*
 * Returns false if we couldn't fill entirely (OOM).
 *
 * Normally run in the receive path, but can also be run from ndo_open
 * before we're receiving packets, or from refill_work which is
 * careful to disable receiving (using napi_disable).
 */
static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
			  gfp_t gfp)
{
	int err;
	bool oom;

	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, rq, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, rq, gfp);
		else
			err = add_recvbuf_small(vi, rq, gfp);

		oom = err == -ENOMEM;
		if (err)
			break;
	} while (rq->vq->num_free);
	if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
		u64_stats_update_begin(&rq->stats.syncp);
		rq->stats.kicks++;
		u64_stats_update_end(&rq->stats.syncp);
	}

	return !oom;
}

static void skb_recv_done(struct virtqueue *rvq)
{
	struct virtnet_info *vi = rvq->vdev->priv;
	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];

	virtqueue_napi_schedule(&rq->napi, rvq);
}

static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
{
	napi_enable(napi);

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets now.
	 * Call local_bh_enable after to trigger softIRQ processing.
	 */
	local_bh_disable();
	virtqueue_napi_schedule(napi, vq);
	local_bh_enable();
}

static void virtnet_napi_tx_enable(struct virtnet_info *vi,
				   struct virtqueue *vq,
				   struct napi_struct *napi)
{
	if (!napi->weight)
		return;

	/* Tx napi touches cachelines on the cpu handling tx interrupts. Only
	 * enable the feature if this is likely affine with the transmit path.
	 */
	if (!vi->affinity_hint_set) {
		napi->weight = 0;
		return;
	}

	return virtnet_napi_enable(vq, napi);
}

static void virtnet_napi_tx_disable(struct napi_struct *napi)
{
	if (napi->weight)
		napi_disable(napi);
}

static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi =
		container_of(work, struct virtnet_info, refill