aboutsummaryrefslogblamecommitdiffstats
path: root/drivers/net/tun.c
blob: 683d371e6e82063bec7ade102a3b40aa38b1b6af (plain) (tree)
1
2
3
4
5
6
7
8
9
10
11
                                            



                                                                 





                                                    


                                                        
                                          
                                                










                                                                      

                                           




                                                                           


                         
                               










                              
                         



                           
                          
                        
                          
                             
                           
                              
                              
                          
                     
                    
                           
                      
                            

                            
                        
 
                          
                          
 


                                                                           


                                   

                 
 









                                                                 
     









                                                                 

      
                                               
 





                                                    

                                          
                                  

                                                                  

                                                                  

                        






                                                                            


                                                                 
                           
 

                                
                       



                               





                                    
                                                                         
                                                                                 

                                                                                
                                                                                
         

             
                                                                               
                                                                   
   
                 

                             
                                     


                                     



                                     
                                
                          
                                
                                                                               

                                    
                                
                                    

  




                          





                                    
                       
                        
                                                           


                                 
                                                        
 
                 



                              
                                                                               
                                                                           

                                          
                   

                                                        
                                      

                                      
 
                                     
                                             
                                                                         
                                       
 
                                      
                                            




                                                

                  
      
                        


                                                      

                                  
                       
                       
                       
                                                   
                                        
                                             
                                           
                                                     
  
 


                            
  
 
                                


                                                 
                                












                                                            








































                                                                            
                                                        
 
                                      
                                                          
                      

                                                                        



                                          
                                                    
 
                                


                                           
                                                
 
                                


                                             
                                                                
 
                                         

 















































                                                                      

                                                               
                                          
                                                 

 

                                                                      
                                                                 



                                                                      
                                                                 

 

                                        
                                              




                                                                                
 
                                                      









                                                                          

                                                                   




                                                                            
                                  


                                                        
                                  



                 




                                                                             
                          
                          








                                                    
                                     
 
                                                                          











                                                                             
                                     
 
                                                                            






                                                          
                                                  
 
                                                                   






                                                        
                              

                                                    
                                     
 
                                                                            
                                                 
 
                                                        
                                                                  
                                                        



                                                                





                                                                             
                                

 
                                                               
                                                   



                                               
                                             
 
                                               


                        


                                                                        

                                                                

                                             
                                                         

                                         

                                                    







                                                                        


                          





                                                                               
                                            
                                     

 
                                                                
                                                                    

                                                                       
                          
   
                                                                               
 
                                 


                          
                                              
 
                                            






                                                                        

         


                   

                                                                             
                              
                      

                    



                                              



                                                             
                               


                                                                        
                                                      













                                                        


                                                          
                                            


                                                                             
                                                         

 





                                                               






                                                                             
                                                                  








                                                 
                            


                       

                                                             
 
                                       



                                             
                                
 

                                                   
                  
 

                                                                 
 
                                                   


                                                   



                                                            
 

                                           
                           

                                        

         
                                      

                                                


                                                                    
                                                              
                                            

                                                                   

                                 
                            
                                                           
                                             
                      
                                                      

                                  
                                                                  
                                     
                                       
                                             
                                              
                                              

                                     

                    


                                                                          
                                                          
                                                                     
                                                               
                 

                                                            
                                                                
                                     




                                                          


                               
                    

                                           
                                   

                                         





                                                  
                                     


                                  
                                                         
                               
                                        
                                                             
                                                                  
                                                   

                                 
                                                          
                                                             
                                                                  
                                                   
         



                                    
                                                         
                                    
                                     
                                       
                                                    

                                     

                                                                    
                                       
                                                    


                                      
 
                                     
                                        

 
                                                                

                                                                   
 
                                                    
                                          
                
 



                                                                       
                      
                                                             


                         
                                                                   


                         

                                                                


                         
 
                                                    
                                                             


                                                                      


                                 

                               

                                                               



                              
                                            
                                                       












                                                                             





                                                                             


                        
                              
                                        
                
                                      
                                                            
         
 


                                                    



                                                                       



                                                                      

                                                    

                                                               
                                     

                   

 
                                                         
 
                               
 




                                          

                   

 

                                           
                          

 
                   



























                                                                     


                                                   












                                                                    

                                                     
                                                      


                                                          
                                       
                 
                                                       
         











                                                               
          













                                                                           
                                                                  




















                                                                             

                                       
                                                
 


                                                  
                            

 


                                               
                                       
 





                                                
                                      



                           
                                                                        
 
                 
                                                                         


                                                                                
                                         

                             
                                                       


                                                                           
         
      

 











                                                                  





                                                                            
                           


                                                  

                                                      
                   



                                                 
 

                                                                 

                       





                                                             

                                             

                          
                                             
                                            

                          
                                                           

                          
                              
 
                                                              

                               

                        
                         
 
                                                   
                          

                                               

                                                            
                                                          

                          
                            

     
                                                  
                          
                       
                          
                             

 
                                                  
 




                                                                     

 

                                                                     




                                                                                










                                                                
           













                                                                            



                                                                         














                                                                  

 



                                                                     
                               
                                  
              





                                                   













                                                              














                                                   
                                                                  





                                                                





                               














                                                                           
                                                     
                                                 

                                                
                                               
                                                       
                                                   
                                                   
                                                      
                                                         

  







                                                            

                                                             

                                                  

                               


                      
 
                                                   

                               

                        
         

                                              

                                                                       



                                                                

                              













                                                                             
         
                                                   
 


                                             
                          
                           

 






                                                                   
                                                            

 
                                                     
                                                 

                                                
                                               
                                                       
                                                 

                                                    
                                                   
                                                          
                                                   
                                                      
                                          
                                               
                                                         

  
                                                 


              



                                                  


                                                                





                                                   

 


                     



                                                  
 
                                             
                     

                                                  





                                               
                                        
                                                                         

                      
                     
                                                  
                                         
                                 
                                                       
                                                        
 
                                        
 

                      


                                                      

 






                                                                              


                           
                                                                 
 
                                                    
                                                
                        
                          

                 
                                
 
                              
 
                                                    
 
                                            
 
                                             
                                              
 







                                                                              
                                               
 
                                                     
                                
 
                     


                    




























                                                                      

                                  




                                                        

                                                 


                                      


                                                                      








                                                                      

                                                                       
                                                            

                                                                
 
                                           
                            
                

                                                         
                                                
                             
 
                                                                              
                                            
                 
                                    


                                 

                                     



                   









                                                                          
                                                             


















                                                                  

                                                                      
                                                

                                                             




                                       





















                                                                             

                                                                               
                                                                    





                                                     
                              
                          
                                               






                                     







                                                                         























                                                             

                                                            
                                                           
                                                                
                                                           
 
                                                           
                                  
                                                                    

                      

                             



                                                  
                                           

                                            
 
                                                                             




                                                                            
                                                              



                                                



                                                                     
                                         
                             

                                                                           


                     
 
                           

                                                  
                       
                                    


                                          
                                     
                                                
                                              
                                          
 

                                                           

                                                     
                 


                                                            

                                           




                                                     
         

                          
 
                                                                         
 
        
                                   
    
                          
                          
                    

 
                                       
                                                                           
                                                                     
                                                   
 
                                                        
                            
                                                
                                                           
                                          
                                     
                        


                              
                       
                        
                                                   
 
                                        
                                     
                                       
                                  
 
                                                                


                                       
                                        


                                                              
                                       
                                   
 
                                                                  

                                       
                                                                

                                                                                                                                    
 
                                                         
                                       
                                                                  

         
                                                      
                                      
                                              
                                                                                         

                                       
 

                                          
                          

                                          

                                                                       

                                                                   
                                                                                      

                                              
                                 

                                                                  


                                        
                                                                              




                                                                           
                                  
                                                                  

                                            

                                         







                                                                         
 












                                                                                


                                                                          

                                                                 

                                            
 





                                                                             

                                      

                                                                  




                                                                 
                                   
                 
         
 
                                                                          

                                                               




                                                         


                               
                                             

                                             



                                                                           

                                                           
                               


                                                             
                                                                          




                                               
                                          
                                         
                                    
                      
                     

                                                                      
                      
         
 



                                                                   
                                                               


                                                     

         
                                      
                                        
 
                      


                                          
                                   





                                                            
                                                  



                                                 
                                  

         





                                                                            
                                                       
 


                                                    
                                  


                          

                                                        

                                                                  
 
                                                           

                                                                  
                                          








                                                         
                                         
















                                                                       
                          
 

                                              

                                             

                                            
 


                                                    
                         
 
 
                                                                            
 
                                          
                                                    
                                                
                       



                               

                                                                 


                      

 

                                                       
                                                            


                                                      
                                     














                                                                     
                                                                      


                                              

                                             





                                            
                                         
                                                   
                                                   
                                                
                                                  

                                                
                                     
                      
                            
                          
                            
 
                                      
                                      
 
                                      
                                                          
 

                                                   
                                        
                                                      

                                       

                                                   


                                                    
 
                                                                      
                                       
         
 
                          
                                          
 
                                                       

                                       
                                                      

                                                                            
                                                                        










                                                                                           
 
                                                                         
                                       

                                                                  

         
                        
                        
                                 

                                                    
                                                               

                                                                         
 

                                                                        

                                  

                                                                 

                                  
 
                                                                               
 
     


                                              

                                                              

                                            



                     
                                                                         

                                         
                         
                      
 

                                                

                         
                                


                         
                                                      

                   
                                                      

                                                        

                                              
                                             


                                                                   
                                        





                              
                                          
                                                         

    
                     
                   

 
                                                                          
                                               
                                                  
 
                    
                
 
                                                   
 
                                  
                                  
                         
         
 
                   
                                           

                                                          

                                   
 

                                                             
 
                                                             
                                       








                                                        
 


                   
                                                                         


                                                    
                                                
                                              


                               
                                                                            
                                       

                                   
                     


                   
                                               
 
                                                                        




                                     

                                                         
                                                
 
                                          







                                                        
                                 
                                                
                                                                     
                                         
                                   

                
                                                   



                 



                                                   
                                              
 
                                     




                                                                       
                             
                                                      
                                                       
                                                     

 



                                                  

                                 
                                                              
 
                                            

                                               

                                            

 


                                                          

                                                                   
 


                                                                           

 





















































                                                                              




                                                          

                                        

  

                                                 
                               
                                  



                                
                                                                             

                       

                                               

                                                                           
 

                                                      

 





                                                                   

                                              

                                                        
 
                                                          








                                                       
                          

























                                                               







                                                            






















                                                                         
                                        






                                                    

                                                                         

                                                       
                                                     

                               



                                                                
                                              

                                                  
                                            







                                                    
                                                                               
 
                   
                                                                             
                                                
                                                 
                             


                               
 
                                                
                                      


                                 

                                                 




                                                                
                                                                     







                                           

                                     


                                

                                                                           

                                                       
    

                     

 
                                                                               

                                 
                                                                             
                                                
                                   
                
 

                              
                              
         
 
                                                             
                              
                                 
         




                                                                       
                                                                               
                                       


                                                          
    
                     
                   


                     

                          
                   

 


                                      

                                                                     
 
                                         






                                                     





                                                                             
                             


                         
                                                                    




                     

                                                 
                                 



                               


                                      
                                                  
  
 

                                            
                                                                             












                                                                                



                                                                         





                                                                                



                                                                         





                                                          










                                                      
                                                                             

                               
                                                    


                               


                               








                                                                

                                                    

                                                  






                                                                                          
                                                           
                                                     

                                       
                                         
                                      
                                                           


                                   
                                                                          
                                                           
                                                                        

                                   
 
                                                   



                                                                               
                                                 

                                 


                                                              


                                         

                                        

                                                               
 
                                                             
                                      


                                                
 


                                               
                                         


                                                      
                                         
                                       
                      
                                       
 


                                             
                                                                       

                                                                           
 


                                       
                                      
                                                   
                                              
                                                       
 


                                       
                                     
                                                                 
 
                                         

                                                          
                                    
                                                           
 





                                                                                 

                                           

                                                                      
                                           
 
                                  
                                   
 
                                                                  

                                                                                
                                                                


                                                                
 


                                                              
                                               
                                                                             
                                                                         
                            
                                           
 

                                                   
                                        



                                                                         

         

                                   
                                                   
 



                                                           
                                                   
 


                                              

                            





                                                                          
 


                                                      

                                     
             
                         


                   
                                                                  
 
                                                   


                                              
                                        
 

 

                                                                    
                                                                 
 
                                       

                               
                                            












                                                            

                                  






                                                                          
                                     

                                                        
                                         



                 





                                                            
                                                         


                                                   










                                                    
                                                         


                                                                      















                                                  
                                                         



                                                          



                                                              




                                                
                                      
                           
                                      




                                                                   
                                                                         
                                                                    
                                                       
                                                   
                                                                               



                                                   

                              


                                              
       



                      

                                                                         














                                                                          
                                                 

 

                                                                
 
                                                    
                                               
                               
                                              
                                      
                         

                     
                   
                        
               
                
                               
 

                                                                    
                                                          
                                       
                
                                             
         


                                                                               


                                                                 
                                                            
                                        
                                                 




                                                             
 
                

                    
                             




                                    

                                                
                                                   
 

                                    
 
                                                        

                                      
         












                                                                    
 
                      
                 
                            
 
                                                                 
 
                                
                
                      
                       
                                       
 

                                                          

                                                      
 
                                                        
                                      

                      

                                             
 

                                                                       
                                                        


                           


                                                                              

                                                         
                                                  
                                         
                 

                                                         
                                                
                                         
                 
 

                                                         



                                             





                                                          
                                 
                                                              
                                                                

                      

                                             





                                                          
                                 
                                                              
                                                                

                      


                                                                            

                                                                                   
                                     

                                                   

                                                                         
                                
                 
                      
 




                                 
                           
                                            
                      
 

                                              
                              
                                                            
                              
                                                                     
                      

                           
                                    

                                                                             
                                                        

                                      

                           
                                    

                                                                   
 
                                                                           
                      

                          
                                                     








                                                                    



                                      
 

                                     

                      


















                                                                              
















                                                       







                                                 


                                              
                                                            

                              
                                                                          

                              
                                             




                                              
                                                            
                              

                                                       

                      

                              
                                                            






                                                                        
                                
                                                                   

                      



                                                                 







                                                                      






                                                             
                

                              
         
 


                                              



                             
                   

 


































                                                                            

                                                            
                                                    

                
                                                                    
                         
 
                 
                                                                     
                                           
              
                                            

                
                   



                                                                
                                                   
                               
 
                                                
 
                                                                       
                                                           

                               




                                                            
                                       
                                           
                         
                           
 
                                                    




                                                   



                                                        
                                   
                                     
 

                                                 




                                                                
                                                    
 
                                



                 
                     
                                                                      
 
                                                    





                                     
                             
                
                                       




                             
                                                  


      
                                                
                              
                            
                                        
                                         
                               



                                             

                                 



                                           




                                        
                              
                          



                       

                                                                          







                                                                
















                                                                           






                                                                                 

                                                                   

                                             
                     
                                                                       
                      
                     
                                                                       





















                                                               






















                                                              
                                                   


                                           
                                              
                                                 

                                           
                                                     
                                                     

  



                                                   
                                


                                                  

                                                             



                                                         
                                           

                                                        
                                             
 

                                                                     
                                                     
 
                     







                                                                  
              
 


                                                




                                          







                                                                           









                                                                 
 



                                
                                                          
 
                                                
                  
                                                    
                                 

         
                                          
                  
                                                                     

                              
 





                                                               
                  


                                      
         

                                            




                             
                                      
                                            
                                                           

 





                                                                               
                               

                                        

                                   
                                        
                              


                                  
                                                   







                                        
                               
 
                                   
 





                                    
                                
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  TUN - Universal TUN/TAP device driver.
 *  Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
 *
 *  $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
 */

/*
 *  Changes:
 *
 *  Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14
 *    Add TUNSETLINK ioctl to set the link encapsulation
 *
 *  Mark Smith <markzzzsmith@yahoo.com.au>
 *    Use eth_random_addr() for tap MAC address.
 *
 *  Harald Roelle <harald.roelle@ifi.lmu.de>  2004/04/20
 *    Fixes in packet dropping, queue length setting and queue wakeup.
 *    Increased default tx queue length.
 *    Added ethtool API.
 *    Minor cleanups
 *
 *  Daniel Podlejski <underley@underley.eu.org>
 *    Modifications for 2.3.99-pre5 kernel.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#define DRV_NAME	"tun"
#define DRV_VERSION	"1.6"
#define DRV_DESCRIPTION	"Universal TUN/TAP device driver"
#define DRV_COPYRIGHT	"(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"

#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/major.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/fcntl.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/miscdevice.h>
#include <linux/ethtool.h>
#include <linux/rtnetlink.h>
#include <linux/compat.h>
#include <linux/if.h>
#include <linux/if_arp.h>
#include <linux/if_ether.h>
#include <linux/if_tun.h>
#include <linux/if_vlan.h>
#include <linux/crc32.h>
#include <linux/nsproxy.h>
#include <linux/virtio_net.h>
#include <linux/rcupdate.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
#include <net/sock.h>
#include <net/xdp.h>
#include <linux/seq_file.h>
#include <linux/uio.h>
#include <linux/skb_array.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/mutex.h>

#include <linux/uaccess.h>
#include <linux/proc_fs.h>

static void tun_default_link_ksettings(struct net_device *dev,
				       struct ethtool_link_ksettings *cmd);

/* Uncomment to enable debugging */
/* #define TUN_DEBUG 1 */

#ifdef TUN_DEBUG
static int debug;

#define tun_debug(level, tun, fmt, args...)			\
do {								\
	if (tun->debug)						\
		netdev_printk(level, tun->dev, fmt, ##args);	\
} while (0)
#define DBG1(level, fmt, args...)				\
do {								\
	if (debug == 2)						\
		printk(level fmt, ##args);			\
} while (0)
#else
#define tun_debug(level, tun, fmt, args...)			\
do {								\
	if (0)							\
		netdev_printk(level, tun->dev, fmt, ##args);	\
} while (0)
#define DBG1(level, fmt, args...)				\
do {								\
	if (0)							\
		printk(level fmt, ##args);			\
} while (0)
#endif

#define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)

/* TUN device flags */

/* IFF_ATTACH_QUEUE is never stored in device flags,
 * overload it to mean fasync when stored there.
 */
#define TUN_FASYNC	IFF_ATTACH_QUEUE
/* High bits in flags field are unused. */
#define TUN_VNET_LE     0x80000000
#define TUN_VNET_BE     0x40000000

#define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
		      IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS)

#define GOODCOPY_LEN 128

#define FLT_EXACT_COUNT 8
struct tap_filter {
	unsigned int    count;    /* Number of addrs. Zero means disabled */
	u32             mask[2];  /* Mask of the hashed addrs */
	unsigned char	addr[FLT_EXACT_COUNT][ETH_ALEN];
};

/* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
 * to max number of VCPUs in guest. */
#define MAX_TAP_QUEUES 256
#define MAX_TAP_FLOWS  4096

#define TUN_FLOW_EXPIRE (3 * HZ)

struct tun_pcpu_stats {
	u64_stats_t rx_packets;
	u64_stats_t rx_bytes;
	u64_stats_t tx_packets;
	u64_stats_t tx_bytes;
	struct u64_stats_sync syncp;
	u32 rx_dropped;
	u32 tx_dropped;
	u32 rx_frame_errors;
};

/* A tun_file connects an open character device to a tuntap netdevice. It
 * also contains all socket related structures (except sock_fprog and tap_filter)
 * to serve as one transmit queue for tuntap device. The sock_fprog and
 * tap_filter were kept in tun_struct since they were used for filtering for the
 * netdevice not for a specific queue (at least I didn't see the requirement for
 * this).
 *
 * RCU usage:
 * The tun_file and tun_struct are loosely coupled, the pointer from one to the
 * other can only be read while rcu_read_lock or rtnl_lock is held.
 */
struct tun_file {
	struct sock sk;
	struct socket socket;
	struct tun_struct __rcu *tun;
	struct fasync_struct *fasync;
	/* only used for fasnyc */
	unsigned int flags;
	union {
		u16 queue_index;
		unsigned int ifindex;
	};
	struct napi_struct napi;
	bool napi_enabled;
	bool napi_frags_enabled;
	struct mutex napi_mutex;	/* Protects access to the above napi */
	struct list_head next;
	struct tun_struct *detached;
	struct ptr_ring tx_ring;
	struct xdp_rxq_info xdp_rxq;
};

struct tun_page {
	struct page *page;
	int count;
};

struct tun_flow_entry {
	struct hlist_node hash_link;
	struct rcu_head rcu;
	struct tun_struct *tun;

	u32 rxhash;
	u32 rps_rxhash;
	int queue_index;
	unsigned long updated ____cacheline_aligned_in_smp;
};

#define TUN_NUM_FLOW_ENTRIES 1024
#define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1)

struct tun_prog {
	struct rcu_head rcu;
	struct bpf_prog *prog;
};

/* Since the socket were moved to tun_file, to preserve the behavior of persist
 * device, socket filter, sndbuf and vnet header size were restore when the
 * file were attached to a persist device.
 */
struct tun_struct {
	struct tun_file __rcu	*tfiles[MAX_TAP_QUEUES];
	unsigned int            numqueues;
	unsigned int 		flags;
	kuid_t			owner;
	kgid_t			group;

	struct net_device	*dev;
	netdev_features_t	set_features;
#define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
			  NETIF_F_TSO6)

	int			align;
	int			vnet_hdr_sz;
	int			sndbuf;
	struct tap_filter	txflt;
	struct sock_fprog	fprog;
	/* protected by rtnl lock */
	bool			filter_attached;
#ifdef TUN_DEBUG
	int debug;
#endif
	spinlock_t lock;
	struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
	struct timer_list flow_gc_timer;
	unsigned long ageing_time;
	unsigned int numdisabled;
	struct list_head disabled;
	void *security;
	u32 flow_count;
	u32 rx_batched;
	struct tun_pcpu_stats __percpu *pcpu_stats;
	struct bpf_prog __rcu *xdp_prog;
	struct tun_prog __rcu *steering_prog;
	struct tun_prog __rcu *filter_prog;
	struct ethtool_link_ksettings link_ksettings;
};

struct veth {
	__be16 h_vlan_proto;
	__be16 h_vlan_TCI;
};

bool tun_is_xdp_frame(void *ptr)
{
	return (unsigned long)ptr & TUN_XDP_FLAG;
}
EXPORT_SYMBOL(tun_is_xdp_frame);

void *tun_xdp_to_ptr(void *ptr)
{
	return (void *)((unsigned long)ptr | TUN_XDP_FLAG);
}
EXPORT_SYMBOL(tun_xdp_to_ptr);

void *tun_ptr_to_xdp(void *ptr)
{
	return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
}
EXPORT_SYMBOL(tun_ptr_to_xdp);

static int tun_napi_receive(struct napi_struct *napi, int budget)
{
	struct tun_file *tfile = container_of(napi, struct tun_file, napi);
	struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
	struct sk_buff_head process_queue;
	struct sk_buff *skb;
	int received = 0;

	__skb_queue_head_init(&process_queue);

	spin_lock(&queue->lock);
	skb_queue_splice_tail_init(queue, &process_queue);
	spin_unlock(&queue->lock);

	while (received < budget && (skb = __skb_dequeue(&process_queue))) {
		napi_gro_receive(napi, skb);
		++received;
	}

	if (!skb_queue_empty(&process_queue)) {
		spin_lock(&queue->lock);
		skb_queue_splice(&process_queue, queue);
		spin_unlock(&queue->lock);
	}

	return received;
}

static int tun_napi_poll(struct napi_struct *napi, int budget)
{
	unsigned int received;

	received = tun_napi_receive(napi, budget);

	if (received < budget)
		napi_complete_done(napi, received);

	return received;
}

static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
			  bool napi_en, bool napi_frags)
{
	tfile->napi_enabled = napi_en;
	tfile->napi_frags_enabled = napi_en && napi_frags;
	if (napi_en) {
		netif_tx_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
				  NAPI_POLL_WEIGHT);
		napi_enable(&tfile->napi);
	}
}

static void tun_napi_disable(struct tun_file *tfile)
{
	if (tfile->napi_enabled)
		napi_disable(&tfile->napi);
}

static void tun_napi_del(struct tun_file *tfile)
{
	if (tfile->napi_enabled)
		netif_napi_del(&tfile->napi);
}

static bool tun_napi_frags_enabled(const struct tun_file *tfile)
{
	return tfile->napi_frags_enabled;
}

#ifdef CONFIG_TUN_VNET_CROSS_LE
static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
{
	return tun->flags & TUN_VNET_BE ? false :
		virtio_legacy_is_little_endian();
}

static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
{
	int be = !!(tun->flags & TUN_VNET_BE);

	if (put_user(be, argp))
		return -EFAULT;

	return 0;
}

static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
{
	int be;

	if (get_user(be, argp))
		return -EFAULT;

	if (be)
		tun->flags |= TUN_VNET_BE;
	else
		tun->flags &= ~TUN_VNET_BE;

	return 0;
}
#else
static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
{
	return virtio_legacy_is_little_endian();
}

static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
{
	return -EINVAL;
}

static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
{
	return -EINVAL;
}
#endif /* CONFIG_TUN_VNET_CROSS_LE */

static inline bool tun_is_little_endian(struct tun_struct *tun)
{
	return tun->flags & TUN_VNET_LE ||
		tun_legacy_is_little_endian(tun);
}

static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)
{
	return __virtio16_to_cpu(tun_is_little_endian(tun), val);
}

static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)
{
	return __cpu_to_virtio16(tun_is_little_endian(tun), val);
}

static inline u32 tun_hashfn(u32 rxhash)
{
	return rxhash & TUN_MASK_FLOW_ENTRIES;
}

static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
{
	struct tun_flow_entry *e;

	hlist_for_each_entry_rcu(e, head, hash_link) {
		if (e->rxhash == rxhash)
			return e;
	}
	return NULL;
}

static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
					      struct hlist_head *head,
					      u32 rxhash, u16 queue_index)
{
	struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC);

	if (e) {
		tun_debug(KERN_INFO, tun, "create flow: hash %u index %u\n",
			  rxhash, queue_index);
		e->updated = jiffies;
		e->rxhash = rxhash;
		e->rps_rxhash = 0;
		e->queue_index = queue_index;
		e->tun = tun;
		hlist_add_head_rcu(&e->hash_link, head);
		++tun->flow_count;
	}
	return e;
}

static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
{
	tun_debug(KERN_INFO, tun, "delete flow: hash %u index %u\n",
		  e->rxhash, e->queue_index);
	hlist_del_rcu(&e->hash_link);
	kfree_rcu(e, rcu);
	--tun->flow_count;
}

static void tun_flow_flush(struct tun_struct *tun)
{
	int i;

	spin_lock_bh(&tun->lock);
	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
		struct tun_flow_entry *e;
		struct hlist_node *n;

		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
			tun_flow_delete(tun, e);
	}
	spin_unlock_bh(&tun->lock);
}

static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
{
	int i;

	spin_lock_bh(&tun->lock);
	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
		struct tun_flow_entry *e;
		struct hlist_node *n;

		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
			if (e->queue_index == queue_index)
				tun_flow_delete(tun, e);
		}
	}
	spin_unlock_bh(&tun->lock);
}

static void tun_flow_cleanup(struct timer_list *t)
{
	struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);
	unsigned long delay = tun->ageing_time;
	unsigned long next_timer = jiffies + delay;
	unsigned long count = 0;
	int i;

	tun_debug(KERN_INFO, tun, "tun_flow_cleanup\n");

	spin_lock(&tun->lock);
	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
		struct tun_flow_entry *e;
		struct hlist_node *n;

		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
			unsigned long this_timer;

			this_timer = e->updated + delay;
			if (time_before_eq(this_timer, jiffies)) {
				tun_flow_delete(tun, e);
				continue;
			}
			count++;
			if (time_before(this_timer, next_timer))
				next_timer = this_timer;
		}
	}

	if (count)
		mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
	spin_unlock(&tun->lock);
}

static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
			    struct tun_file *tfile)
{
	struct hlist_head *head;
	struct tun_flow_entry *e;
	unsigned long delay = tun->ageing_time;
	u16 queue_index = tfile->queue_index;

	head = &tun->flows[tun_hashfn(rxhash)];

	rcu_read_lock();

	e = tun_flow_find(head, rxhash);
	if (likely(e)) {
		/* TODO: keep queueing to old queue until it's empty? */
		if (READ_ONCE(e->queue_index) != queue_index)
			WRITE_ONCE(e->queue_index, queue_index);
		if (e->updated != jiffies)
			e->updated = jiffies;
		sock_rps_record_flow_hash(e->rps_rxhash);
	} else {
		spin_lock_bh(&tun->lock);
		if (!tun_flow_find(head, rxhash) &&
		    tun->flow_count < MAX_TAP_FLOWS)
			tun_flow_create(tun, head, rxhash, queue_index);

		if (!timer_pending(&tun->flow_gc_timer))
			mod_timer(&tun->flow_gc_timer,
				  round_jiffies_up(jiffies + delay));
		spin_unlock_bh(&tun->lock);
	}

	rcu_read_unlock();
}

/**
 * Save the hash received in the stack receive path and update the
 * flow_hash table accordingly.
 */
static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
{
	if (unlikely(e->rps_rxhash != hash))
		e->rps_rxhash = hash;
}

/* We try to identify a flow through its rxhash. The reason that
 * we do not check rxq no. is because some cards(e.g 82599), chooses
 * the rxq based on the txq where the last packet of the flow comes. As
 * the userspace application move between processors, we may get a
 * different rxq no. here.
 */
static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
{
	struct tun_flow_entry *e;
	u32 txq = 0;
	u32 numqueues = 0;

	numqueues = READ_ONCE(tun->numqueues);

	txq = __skb_get_hash_symmetric(skb);
	e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
	if (e) {
		tun_flow_save_rps_rxhash(e, txq);
		txq = e->queue_index;
	} else {
		/* use multiply and shift instead of expensive divide */
		txq = ((u64)txq * numqueues) >> 32;
	}

	return txq;
}

static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
{
	struct tun_prog *prog;
	u32 numqueues;
	u16 ret = 0;

	numqueues = READ_ONCE(tun->numqueues);
	if (!numqueues)
		return 0;

	prog = rcu_dereference(tun->steering_prog);
	if (prog)
		ret = bpf_prog_run_clear_cb(prog->prog, skb);

	return ret % numqueues;
}

static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
			    struct net_device *sb_dev)
{
	struct tun_struct *tun = netdev_priv(dev);
	u16 ret;

	rcu_read_lock();
	if (rcu_dereference(tun->steering_prog))
		ret = tun_ebpf_select_queue(tun, skb);
	else
		ret = tun_automq_select_queue(tun, skb);
	rcu_read_unlock();

	return ret;
}

static inline bool tun_not_capable(struct tun_struct *tun)
{
	const struct cred *cred = current_cred();
	struct net *net = dev_net(tun->dev);

	return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||
		  (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&
		!ns_capable(net->user_ns, CAP_NET_ADMIN);
}

static void tun_set_real_num_queues(struct tun_struct *tun)
{
	netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
	netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
}

static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
{
	tfile->detached = tun;
	list_add_tail(&tfile->next, &tun->disabled);
	++tun->numdisabled;
}

static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
{
	struct tun_struct *tun = tfile->detached;

	tfile->detached = NULL;
	list_del_init(&tfile->next);
	--tun->numdisabled;
	return tun;
}

void tun_ptr_free(void *ptr)
{
	if (!ptr)
		return;
	if (tun_is_xdp_frame(ptr)) {
		struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);

		xdp_return_frame(xdpf);
	} else {
		__skb_array_destroy_skb(ptr);
	}
}
EXPORT_SYMBOL_GPL(tun_ptr_free);

static void tun_queue_purge(struct tun_file *tfile)
{
	void *ptr;

	while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
		tun_ptr_free(ptr);

	skb_queue_purge(&tfile->sk.sk_write_queue);
	skb_queue_purge(&tfile->sk.sk_error_queue);
}

static void __tun_detach(struct tun_file *tfile, bool clean)
{
	struct tun_file *ntfile;
	struct tun_struct *tun;

	tun = rtnl_dereference(tfile->tun);

	if (tun && clean) {
		tun_napi_disable(tfile);
		tun_napi_del(tfile);
	}

	if (tun && !tfile->detached) {
		u16 index = tfile->queue_index;
		BUG_ON(index >= tun->numqueues);

		rcu_assign_pointer(tun->tfiles[index],
				   tun->tfiles[tun->numqueues - 1]);
		ntfile = rtnl_dereference(tun->tfiles[index]);
		ntfile->queue_index = index;
		rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
				   NULL);

		--tun->numqueues;
		if (clean) {
			RCU_INIT_POINTER(tfile->tun, NULL);
			sock_put(&tfile->sk);
		} else
			tun_disable_queue(tun, tfile);

		synchronize_net();
		tun_flow_delete_by_queue(tun, tun->numqueues + 1);
		/* Drop read queue */
		tun_queue_purge(tfile);
		tun_set_real_num_queues(tun);
	} else if (tfile->detached && clean) {
		tun = tun_enable_queue(tfile);
		sock_put(&tfile->sk);
	}

	if (clean) {
		if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
			netif_carrier_off(tun->dev);

			if (!(tun->flags & IFF_PERSIST) &&
			    tun->dev->reg_state == NETREG_REGISTERED)
				unregister_netdevice(tun->dev);
		}
		if (tun)
			xdp_rxq_info_unreg(&tfile->xdp_rxq);
		ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
		sock_put(&tfile->sk);
	}
}

static void tun_detach(struct tun_file *tfile, bool clean)
{
	struct tun_struct *tun;
	struct net_device *dev;

	rtnl_lock();
	tun = rtnl_dereference(tfile->tun);
	dev = tun ? tun->dev : NULL;
	__tun_detach(tfile, clean);
	if (dev)
		netdev_state_change(dev);
	rtnl_unlock();
}

static void tun_detach_all(struct net_device *dev)
{
	struct tun_struct *tun = netdev_priv(dev);
	struct tun_file *tfile, *tmp;
	int i, n = tun->numqueues;

	for (i = 0; i < n; i++) {
		tfile = rtnl_dereference(tun->tfiles[i]);
		BUG_ON(!tfile);
		tun_napi_disable(tfile);
		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
		RCU_INIT_POINTER(tfile->tun, NULL);
		--tun->numqueues;
	}
	list_for_each_entry(tfile, &tun->disabled, next) {
		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
		RCU_INIT_POINTER(tfile->tun, NULL);
	}
	BUG_ON(tun->numqueues != 0);

	synchronize_net();
	for (i = 0; i < n; i++) {
		tfile = rtnl_dereference(tun->tfiles[i]);
		tun_napi_del(tfile);
		/* Drop read queue */
		tun_queue_purge(tfile);
		xdp_rxq_info_unreg(&tfile->xdp_rxq);
		sock_put(&tfile->sk);
	}
	list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
		tun_enable_queue(tfile);
		tun_queue_purge(tfile);
		xdp_rxq_info_unreg(&tfile->xdp_rxq);
		sock_put(&tfile->sk);
	}
	BUG_ON(tun->numdisabled != 0);

	if (tun->flags & IFF_PERSIST)
		module_put(THIS_MODULE);
}

static int tun_attach(struct tun_struct *tun, struct file *file,
		      bool skip_filter, bool napi, bool napi_frags,
		      bool publish_tun)
{
	struct tun_file *tfile = file->private_data;
	struct net_device *dev = tun->dev;
	int err;

	err = security_tun_dev_attach(tfile->socket.sk, tun->security);
	if (err < 0)
		goto out;

	err = -EINVAL;
	if (rtnl_dereference(tfile->tun) && !tfile->detached)
		goto out;

	err = -EBUSY;
	if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
		goto out;

	err = -E2BIG;
	if (!tfile->detached &&
	    tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
		goto out;

	err = 0;

	/* Re-attach the filter to persist device */
	if (!skip_filter && (tun->filter_attached == true)) {
		lock_sock(tfile->socket.sk);
		err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
		release_sock(tfile->socket.sk);
		if (!err)
			goto out;
	}

	if (!tfile->detached &&
	    ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
			    GFP_KERNEL, tun_ptr_free)) {
		err = -ENOMEM;
		goto out;
	}

	tfile->queue_index = tun->numqueues;
	tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;

	if (tfile->detached) {
		/* Re-attach detached tfile, updating XDP queue_index */
		WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));

		if (tfile->xdp_rxq.queue_index    != tfile->queue_index)
			tfile->xdp_rxq.queue_index = tfile->queue_index;
	} else {
		/* Setup XDP RX-queue info, for new tfile getting attached */
		err = xdp_rxq_info_reg(&tfile->xdp_rxq,
				       tun->dev, tfile->queue_index);
		if (err < 0)
			goto out;
		err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
						 MEM_TYPE_PAGE_SHARED, NULL);
		if (err < 0) {
			xdp_rxq_info_unreg(&tfile->xdp_rxq);
			goto out;
		}
		err = 0;
	}

	if (tfile->detached) {
		tun_enable_queue(tfile);
	} else {
		sock_hold(&tfile->sk);
		tun_napi_init(tun, tfile, napi, napi_frags);
	}

	if (rtnl_dereference(tun->xdp_prog))
		sock_set_flag(&tfile->sk, SOCK_XDP);

	/* device is allowed to go away first, so no need to hold extra
	 * refcnt.
	 */

	/* Publish tfile->tun and tun->tfiles only after we've fully
	 * initialized tfile; otherwise we risk using half-initialized
	 * object.
	 */
	if (publish_tun)
		rcu_assign_pointer(tfile->tun, tun);
	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
	tun->numqueues++;
	tun_set_real_num_queues(tun);
out:
	return err;
}

static struct tun_struct *tun_get(struct tun_file *tfile)
{
	struct tun_struct *tun;

	rcu_read_lock();
	tun = rcu_dereference(tfile->tun);
	if (tun)
		dev_hold(tun->dev);
	rcu_read_unlock();

	return tun;
}

static void tun_put(struct tun_struct *tun)
{
	dev_put(tun->dev);
}

/* TAP filtering */
static void addr_hash_set(u32 *mask, const u8 *addr)
{
	int n = ether_crc(ETH_ALEN, addr) >> 26;
	mask[n >> 5] |= (1 << (n & 31));
}

static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
{
	int n = ether_crc(ETH_ALEN, addr) >> 26;
	return mask[n >> 5] & (1 << (n & 31));
}

static int update_filter(struct tap_filter *filter, void __user *arg)
{
	struct { u8 u[ETH_ALEN]; } *addr;
	struct tun_filter uf;
	int err, alen, n, nexact;

	if (copy_from_user(&uf, arg, sizeof(uf)))
		return -EFAULT;

	if (!uf.count) {
		/* Disabled */
		filter->count = 0;
		return 0;
	}

	alen = ETH_ALEN * uf.count;
	addr = memdup_user(arg + sizeof(uf), alen);
	if (IS_ERR(addr))
		return PTR_ERR(addr);

	/* The filter is updated without holding any locks. Which is
	 * perfectly safe. We disable it first and in the worst
	 * case we'll accept a few undesired packets. */
	filter->count = 0;
	wmb();

	/* Use first set of addresses as an exact filter */
	for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
		memcpy(filter->addr[n], addr[n].u, ETH_ALEN);

	nexact = n;

	/* Remaining multicast addresses are hashed,
	 * unicast will leave the filter disabled. */
	memset(filter->mask, 0, sizeof(filter->mask));
	for (; n < uf.count; n++) {
		if (!is_multicast_ether_addr(addr[n].u)) {
			err = 0; /* no filter */
			goto free_addr;
		}
		addr_hash_set(filter->mask, addr[n].u);
	}

	/* For ALLMULTI just set the mask to all ones.
	 * This overrides the mask populated above. */
	if ((uf.flags & TUN_FLT_ALLMULTI))
		memset(filter->mask, ~0, sizeof(filter->mask));

	/* Now enable the filter */
	wmb();
	filter->count = nexact;

	/* Return the number of exact filters */
	err = nexact;
free_addr:
	kfree(addr);
	return err;
}

/* Returns: 0 - drop, !=0 - accept */
static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
{
	/* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect
	 * at this point. */
	struct ethhdr *eh = (struct ethhdr *) skb->data;
	int i;

	/* Exact match */
	for (i = 0; i < filter->count; i++)
		if (ether_addr_equal(eh->h_dest, filter->addr[i]))
			return 1;

	/* Inexact match (multicast only) */
	if (is_multicast_ether_addr(eh->h_dest))
		return addr_hash_test(filter->mask, eh->h_dest);

	return 0;
}

/*
 * Checks whether the packet is accepted or not.
 * Returns: 0 - drop, !=0 - accept
 */
static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
{
	if (!filter->count)
		return 1;

	return run_filter(filter, skb);
}

/* Network device part of the driver */

static const struct ethtool_ops tun_ethtool_ops;

/* Net device detach from fd. */
static void tun_net_uninit(struct net_device *dev)
{
	tun_detach_all(dev);
}

/* Net device open. */
static int tun_net_open(struct net_device *dev)
{
	netif_tx_start_all_queues(dev);

	return 0;
}

/* Net device close. */
static int tun_net_close(struct net_device *dev)
{
	netif_tx_stop_all_queues(dev);
	return 0;
}

/* Net device start xmit */
static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
{
#ifdef CONFIG_RPS
	if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) {
		/* Select queue was not called for the skbuff, so we extract the
		 * RPS hash and save it into the flow_table here.
		 */
		struct tun_flow_entry *e;
		__u32 rxhash;

		rxhash = __skb_get_hash_symmetric(skb);
		e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);
		if (e)
			tun_flow_save_rps_rxhash(e, rxhash);
	}
#endif
}

static unsigned int run_ebpf_filter(struct tun_struct *tun,
				    struct sk_buff *skb,
				    int len)
{
	struct tun_prog *prog = rcu_dereference(tun->filter_prog);

	if (prog)
		len = bpf_prog_run_clear_cb(prog->prog, skb);

	return len;
}

/* Net device start xmit */
static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct tun_struct *tun = netdev_priv(dev);
	int txq = skb->queue_mapping;
	struct tun_file *tfile;
	int len = skb->len;

	rcu_read_lock();
	tfile = rcu_dereference(tun->tfiles[txq]);

	/* Drop packet if interface is not attached */
	if (!tfile)
		goto drop;

	if (!rcu_dereference(tun->steering_prog))
		tun_automq_xmit(tun, skb);

	tun_debug(KERN_INFO, tun, "tun_net_xmit %d\n", skb->len);

	BUG_ON(!tfile);

	/* Drop if the filter does not like it.
	 * This is a noop if the filter is disabled.
	 * Filter can be enabled only for the TAP devices. */
	if (!check_filter(&tun->txflt, skb))
		goto drop;

	if (tfile->socket.sk->sk_filter &&
	    sk_filter(tfile->socket.sk, skb))
		goto drop;

	len = run_ebpf_filter(tun, skb, len);
	if (len == 0 || pskb_trim(skb, len))
		goto drop;

	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
		goto drop;

	skb_tx_timestamp(skb);

	/* Orphan the skb - required as we might hang on to it
	 * for indefinite time.
	 */
	skb_orphan(skb);

	nf_reset_ct(skb);

	if (ptr_ring_produce(&tfile->tx_ring, skb))
		goto drop;

	/* Notify and wake up reader process */
	if (tfile->flags & TUN_FASYNC)
		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
	tfile->socket.sk->sk_data_ready(tfile->socket.sk);

	rcu_read_unlock();
	return NETDEV_TX_OK;

drop:
	this_cpu_inc(tun->pcpu_stats->tx_dropped);
	skb_tx_error(skb);
	kfree_skb(skb);
	rcu_read_unlock();
	return NET_XMIT_DROP;
}

static void tun_net_mclist(struct net_device *dev)
{
	/*
	 * This callback is supposed to deal with mc filter in
	 * _rx_ path and has nothing to do with the _tx_ path.
	 * In rx path we always accept everything userspace gives us.
	 */
}

static netdev_features_t tun_net_fix_features(struct net_device *dev,
	netdev_features_t features)
{
	struct tun_struct *tun = netdev_priv(dev);

	return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
}

static void tun_set_headroom(struct net_device *dev, int new_hr)
{
	struct tun_struct *tun = netdev_priv(dev);

	if (new_hr < NET_SKB_PAD)
		new_hr = NET_SKB_PAD;

	tun->align = new_hr;
}

static void
tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
{
	u32 rx_dropped = 0, tx_dropped = 0, rx_frame_errors = 0;
	struct tun_struct *tun = netdev_priv(dev);
	struct tun_pcpu_stats *p;
	int i;

	for_each_possible_cpu(i) {
		u64 rxpackets, rxbytes, txpackets, txbytes;
		unsigned int start;

		p = per_cpu_ptr(