aboutsummaryrefslogblamecommitdiffstats
path: root/net/packet/af_packet.c
blob: d37075b0d6d5cc045a65d76bf37870856de0d197 (plain) (tree)
1
2
3
4
5
6
7
8






                                                                            
                         


                                                                
         





















                                                                              
                                                                      
                                                                    


                                                                               
                                                                             
                                                        
                                                      



                                                                                






                                                                             
 
                        
                     
                             






                            
                         
                       
                       
                          
                              





                         


                        
                           





                           
                        
                          
                             
                           
                             
                         



                            

                     
  













                                                                             

                          

                                

                          

                                

                                                                          
                                                   
                     

                                

                                                    








                                                                          

                          

                                                                             

                     




                                                       

                                       


                                                
                        



                                                 
  
 






                                
                                                                       

                                          

                           
                                                                                



                                                         
                          
 







                                                                         

                                                                   

                                                                            
 



                                                                   


                                                                
                                              
                                                                
                                                           


                                                        
                                                           








                                                                               

                                                 








                                                                       
 
                                                                              
                                 
                                                                 
                                                 
                                                                                



                                                          


                                                                     


                                                  

                                   
                                 

                                           

                                             


                                                 

                                 
 
                                         
 


                                                   
                                                   
                                                              
                                 
 


                                    
                               
 
                   
     
                                          

                             

 























                                                                       




                                                                
                                                                              
 
                                                                      

 















                                                                             






                                                                  
 
                           
                               
                                              
                    
                                                     
 
















                                                                     

                       
                                        
            
                                                  
 
















                                                            
                                                          





                                             
                                                                                
 
                             
 
                      

                                 
                                         
                                                                 

                        
                                         
                                                                 
                      
                        
                
                                                            
                      
         

                  

 
                                                                   
 
                             
 

                  


                                 
                                                                 
                                       
                        
                                                                 
                                       
                        
                
                                                            

                         
         
 
 

                                                                            


                                                                      



                                                              

                                                    
                                             
 
                 

 

                                                                        


                             
                        
 

                                                                          



















                                                                                

                         

 





                                                        
                             



                                                       

                                                






                                                     
                                                         





                                                             
                                                                   







                                                                 
                                      
 

                                                         
 
                                      
                                  
                                        




                                                      
                                              









                                                                           
                                      



                      

                                                         







                                                                     

                                
                  
 



                                                                 
                                              

                                                 
                                         

                      



                                                                
                                                                   
                                                      


                                           














                                                        
                                                         









                                                                
                                                             
                                       




                                     
                                                            
                                           




                                                  
                                             







                                                                           
                                                                               







                                                
                                                                           































                                                                       
                                                                        
                            
                                       






























































                                                                                
                                                           
                                                              








































                                                                  

                                                           




                                                          
                                                   
                                  
 
                                      





















                                                            

                              


                                                           
                                                         










                                           

                                                          

                           
                                                   


                  


                                                              
 


                                                               
 



                                              
 













                                                                                    
























                                                                               
                                                           


                                                         
                                           









                                                                        
                                                                   

                                       
                                       




















                                                                        
                                                                   

                                                            
                                                                             




















                                                                            

 
                                                             
                                                                     



                                                  
                                                          



                                              
                                                                    
 
                                                               


                                           
                                                          

                                                 
                                                    

 
                                                           




                                                 
                                                             

                                                 

                                                                  

                                                                                  
                
                                         
                                          
                                                     


         
                                                             

                                                 
                                







                                                          
                                           

                                                               




















                                                                   

                                       

                         
                                              
























                                                                             
                                           
























                                                                        
                                                            














                                                                            
                            


         
                                                     
                                                                   
                                                      

                                                
                                                               
                                                                   





                                        
                                                              









                                                      
                                                         






                                                                       
                                                             








                                                                           
                                                            













                                                                          
                                                          






                                                                        
                                                                  



                                                                      








































                                                                            






















                                                                                 

                                                 

                                             

                                                 

                                        
                                                                           


                       
                                

 









                                                                    


                                                              
 
                                                        

 


                                                            






                                                                     







                                                             

 



                                                             
                                    

 



                                                                              
 
                          
 













                                                                               






                                                            


                                                              

 

                                                                                 



                                                     
                         






                                                        


                                
                                                                    
                                                                        


                                         
                                                     

                              
                                                   

                               

                                                    


                                                    


                                                   

                                                                               
                      

         





                                                                   



                                                                      

                                




























                                                                    
                                                                          
 
                                                                                




                            
                                                              


                                            
                                    
                                   


                       


                                                             

                                
                               
                               
                              



















                                                         
                      
                                           
                         
                     
                              
                                                            




                                                      
                                     







                                                           
                                                               

                                                     
         










                                                                      

                 
    












                                            
                                  

                          






                                               
 
                                         
 
                                              
 

                                                                               









                                                                               
 






                                                               
                                                                  





                                               
                                                

                         

                                               


                                   
                          
 


                                      
                                           
 
                                                       













                                                                          
                                             












                                                                              
 



                                                                       
                                                                      
                                   
                               
                         
                
                          
 
          
                                           

           
                    
                                                               




                                                                                        

          
                                                      

           
                                                               
      

                                                                    


                                
 



                                   
          

                                                                                  
           
 







                                                            
                        
                                                                          

                                

                                                         
                                                


                                                                                
                                                                             















                                                                                
                                                                   


                                      

         
                                                                  











                                                                 
 


                                        
                                

                                                          
 


                                     
                                           
 
                            
                          
                   
 
           
                          

                       

                   
 
                                                         
                                                            
                                                       

                                 
 

                                                
                           
                                                 
                          
 
                   


  








                                                                    

   

                                                                          



                                
                                 
                               
                                  






                                             
                                                

                          

                       
                              
                                                                  




                                                                   

                                              
                                                                       

                                                                                   
                                                               




                           

                                           
                                    

                              
 
                                                             










                                                                  
                                 


                           



                                                                     



                                          
                                  


                                                     
 
                                                              
 
                                               
 




                                    
                          
 


                                      
                                              
                                      
                                                    

                                                     
                              


                 
                                              
                                    

                                                






                                                       
                         


                 

                                                                           



                                
                             
                                 
                               
                                  
                                              
                                              
                                        
                           
                        
 






                                                                                 





                                             
                                                

                          
                              
                                              
                                                                       

                                                                                   
                                                               


                 


                                                 

                           

                                           
                                    

                              

                                        

                                                                     
                
                                                              
                                                      

                                                                     

                                         


                                                                
                                                                              







                                                                              
                         


                                                                  
                 











                                                                                              
         
                                              

                                                                            
                   
                                  







                                                                    
                                              

                                                   
                                      



                                                                  

                                                
                                                       

                                                                          
                                    
 

                            





                                           

                                                           






                                           

                                           

                                                                  

                                                                                   

                                              
                                               
                 
                                                                      

                                       








                                                                  

                                           
                                                                      

                                       


                      
 
                                            
                                                              



                                          
                                  


                                                     
 
                 
 
                                          
                                           

                                




                                                                    
         
                  
      
 
                                           
                                                       

                                      
                                                        
         






                                                       
                       


                 
                                    

                                                
                              
                            


                            


                                                     
 
                                         
                         

                         
                                                     
                                                 


                                                                      




                        



                                                                      
                                                                                 






                                                                               

                                                                        
                                                            
 
                              
                                                             









                                               
                                   
                                                               









                                                 



                                                                                
 
                               
                                      
 


                                                   



























                                                                           






                                                                   
                                          
                                                     
                                       










                                                      












                                                          

                                                                              


                                       

                                         

                                        
                                                                     










                                                                  


                               
                             
                 
                                                                     
                                                          


                             
                                         
                       
 

                                     
                                    
                                                    









                                                                      

                                              
                                                                              

         


                                  



                                             
                                                   
                                         
                                                               
 


                                              

                                                           
                                                                  
                                           

                                                        



                                                

                                              
                                                  
                                                                         





                                                                            






                                                                         
 




                                                                













                                                                     

                                               

                                                               
                                                 

                                                
                                    













                                                                    

                                                    







                                                                        



                      



                                            
                     



                                       
 



                                                                       







                                                                              
                                           










                                     
                                                                          

                                   
                                                                     

                               
                     
                            
                             




                                               
                       
                          
                  

          
                                           
           
 
                                    
                                                    





                                                                  

                                                                                                   

                                              
                                                                         

         
                     
                                  
                                
                        
                                             

                                

                                               








                                                
                              
                                                                            
                                      


                                                                     





                                                                                    

                              
                                                                     

























                                                                              







                                                            
                        
                                                                            

                                
                       

                                      

                                                                          
                                                                    
                        

                                
                                             
 
                      







                                                                                  

                                      
                                                                            

                              

                                                          
 
                                                                  










                                                                 

         

                              
                                        
                                

                                       
 

                                                                   


                                                                               




                                              

                                                                    








                                                                    

                                                         


                                     
                            


                                                        
                     
 
                   



                       
                




                             


                                                                  

                                            
 


                                            


                                                  








                                                                           
                        
                                  



                         
                           

                        
                                             
                                 


                                               
                                                  
                         
 
                                  
                                        

                                    



                                           
                                    
 
                                
 

                                                 
                                                  
         
 

                                                 
                                                  
         
 

                           
                          


                                                                  





                                               
                                
                                    








                             
                                                                                

                                            


                                          
 



                                     
                               
         

                      
                                  
 






                                                             
 

                                           
 







                                                     
 
                                       

                                
                                            
                                       



                                                











                                        

                                                                        
 
                                   


                               
 


                              
 
                                                
                               
                                                    
 
                                                  
                
                                                               

                   


                                                                                 

                                                              






                                      
 






                                                  
                                                                       



                                                                             











                                               
                                            

   

                                                                            


                               
                                                                            

                
                                                   
                              

                                                                 




                                        
                                                                 



                                

                                             
 



                                  
                        
                                  
 



                                       
                                    

                                               
                                





                                       
                                     
                                        
 

                                                     
 

                                          

                                           
                                       

         
                                             
                                                 


                                               
                                                   
                         
 
                 

                    














                                                                     
                             

                      
                                                                                    







                                                         
                                   

                                                                          


                         
          







                                                                           
                                                                       

          
                                                                           



                                                                          
                        

                         




                                                       
                                       

                                      

                                    



                                                                             



                                                                           
















                                                                             



                                                                           

                                                                     

                                               
                                                                          



                                      


                                                                      
           
                          


                                            

         
                                                         


                              
                                             
 




                                                                    
                                                                           





                                                                             

                                                              
         
 
                                  







                                                                
                                                     

                                                                

                                                                                          

                                            
                                             
                 
                                                                             

         



                                                                    
                                                                     






                                   









                                                                           
                                                          


                                                                      
                                                                           
                          



                                    






                                                                      
                                                           






                                       
                             

                                                              



                                                                    



                                                                           
                          
                                                                             



                 

                                                                         


                                 

                                             
                             
                                                        
                    
                                                        

                               
                                                      
                                
                                                   
                               

                                             
                             
                                                        
                    
                                                        
                      

                      
         
                 



                                                                                        
                                 




                                                    
                                                                       








                                            
                                                                 



                          
                                          


                          
                                            






















                                                                        




                                       





                      
                                                                        












                                                                               

                                                                                    
                                                                   






















                                                

                                                                    
                                                   



                          

          
                                                                                                        

                                   
                                            




                                    
                          
                                   

                                    



                                                     
                                       

                                           
                                                       
                                       

                                                                                    





                                                        
 
                            
                            
         

                                          
 










                                                 
                                       

                                             
                                                            
                                       

                                                     




                                
                                          
                                       
                                                              




                                              





                                          
                                                             





                                                              
                                





                                             





                                          
                                                             





                                                              












                                                              











                                                              











                                                              















                                                              











                                                              










                                                               












                                                              











                                                                     








                                                                         
                                  

                                            
                          
                                 



                                    

                                  


                               
 
                          
                               
                                                         



                                                           
                                                   
                                                             
                                                                   
                                          
                        
                                                          
                                                                   
                                          
                 
 

                            
                                  
                      
                            
                                  
                      
                             
                                       
                      
                            
                                     












                                                          


                                                          


                                       
                      
                            
                                     
                      
                         
                                  
                      
                              
                                    
                      
                           

                                             

                                                         
                          
                      


                                        


                                                 



                                    

                         

                                  

                                            
                 


 

                                                        

                        
                                                                  
                                       
 
                        
                                                  



                                                    

                                                                       

                                         



                                                          
                                                                          




                                                                        
                                                                    
                                                         

                                                                           





                                                                 

                                                          

                                                               
                                                            
                         


                              
                          








                                                              
                      


                                                   
 

















                                                                               
 
                  













                            
                                                            

      

                                    



                 
                                                                       






                                                            
                                 

                                                               


                                                    





                                                                                










                                                           
                                                 
                                   
 






                                                       
                                                 
                                   
 



                                                
                                                            

                                        

  

                                                               


              
                                   
                                               
                                                              





                                                                           



                      
                                                       
 
                     



                                                                    


                              
                                                           
                                                   


                              
                                                     
                                    
                                                             


                              
                                        
                    

 
                                                                   

                                                 
                           

              
                                                                   



                                        
                                                                
                                                










                                             
 
                                                                       
                                         
 
                                  
                                            
                                   

                                      
                   








                                                                   
 

                                                                         
 



                                             
                                            

                                 
 




                                                        
 






                                                        


                                                        

                 
                              
                                                           
                                 
                                                                   
                                 



                                                                          
                                                                 

                                                        
                                                                           
                                 
 





                                                                             

                              


                                                      
                                 






                                                                              
                              


                              



                              
                                               
                                 








                                        
                            
                                                  

                                    
 


                          
                                     

                                                       
                                              
                                         




                                                       

                                                       




                                                                         
                                             

                                                                
         
                                       

                                  
                          
                              
                                       

                                    




                                                                             

                         





                                                             

                                                              


                                            

                                          






                               
                                     










                                                                  
                         


                                           

                         
                              




                                                           

                                                           

                                   

                                                                               



                                                                       
                                                   
                         
                 
         
 
                                



                                       
                                       

                   
 
                                                 


















                                            
 
                                            






                                           
                                       











                                          
                                                          





                                                       
                                                


                     

                                                                
                       
 
                                            


                                                                   



                                                                        
                                            
                                                               


                                                          
                       
 
                          

 
                                                         



                                                                                           
                                             


                                                         
                                                                             






                                                          
                                                                             
                                          




                 
                                                     







                                                                  

                                                            

 
                                                       



                                          
                                          



      
                                                      
 
                                             
                                             
 
                                                                       




                               
                                                       
 
                                                   







                                                  

                                    
                                                               
                                                  











                                                  
                                                
                                                             







                                 
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		PACKET - implements raw packet sockets.
 *
 * Authors:	Ross Biro
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *
 * Fixes:
 *		Alan Cox	:	verify_area() now used correctly
 *		Alan Cox	:	new skbuff lists, look ma no backlogs!
 *		Alan Cox	:	tidied skbuff lists.
 *		Alan Cox	:	Now uses generic datagram routines I
 *					added. Also fixed the peek/read crash
 *					from all old Linux datagram code.
 *		Alan Cox	:	Uses the improved datagram code.
 *		Alan Cox	:	Added NULL's for socket options.
 *		Alan Cox	:	Re-commented the code.
 *		Alan Cox	:	Use new kernel side addressing
 *		Rob Janssen	:	Correct MTU usage.
 *		Dave Platt	:	Counter leaks caused by incorrect
 *					interrupt locking and some slightly
 *					dubious gcc output. Can you read
 *					compiler: it said _VOLATILE_
 *	Richard Kooijman	:	Timestamp fixes.
 *		Alan Cox	:	New buffers. Use sk->mac.raw.
 *		Alan Cox	:	sendmsg/recvmsg support.
 *		Alan Cox	:	Protocol setting support
 *	Alexey Kuznetsov	:	Untied from IPv4 stack.
 *	Cyrus Durgin		:	Fixed kerneld for kmod.
 *	Michal Ostrowski        :       Module initialization cleanup.
 *         Ulises Alonso        :       Frame number limit removal and
 *                                      packet_set_ring memory leak.
 *		Eric Biederman	:	Allow for > 8 byte hardware addresses.
 *					The convention is that longer addresses
 *					will simply extend the hardware address
 *					byte arrays at the end of sockaddr_ll
 *					and packet_mreq.
 *		Johann Baudy	:	Added TX RING.
 *		Chetan Loke	:	Implemented TPACKET_V3 block abstraction
 *					layer.
 *					Copyright (C) 2011, <lokec@ccs.neu.edu>
 *
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 */

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/capability.h>
#include <linux/fcntl.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/if_packet.h>
#include <linux/wireless.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <net/net_namespace.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/errno.h>
#include <linux/timer.h>
#include <asm/uaccess.h>
#include <asm/ioctls.h>
#include <asm/page.h>
#include <asm/cacheflush.h>
#include <asm/io.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/poll.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/if_vlan.h>
#include <linux/virtio_net.h>
#include <linux/errqueue.h>
#include <linux/net_tstamp.h>
#include <linux/percpu.h>
#ifdef CONFIG_INET
#include <net/inet_common.h>
#endif

#include "internal.h"

/*
   Assumptions:
   - if device has no dev->hard_header routine, it adds and removes ll header
     inside itself. In this case ll header is invisible outside of device,
     but higher levels still should reserve dev->hard_header_len.
     Some devices are enough clever to reallocate skb, when header
     will not fit to reserved space (tunnel), another ones are silly
     (PPP).
   - packet socket receives packets with pulled ll header,
     so that SOCK_RAW should push it back.

On receive:
-----------

Incoming, dev->hard_header!=NULL
   mac_header -> ll header
   data       -> data

Outgoing, dev->hard_header!=NULL
   mac_header -> ll header
   data       -> ll header

Incoming, dev->hard_header==NULL
   mac_header -> UNKNOWN position. It is very likely, that it points to ll
		 header.  PPP makes it, that is wrong, because introduce
		 assymetry between rx and tx paths.
   data       -> data

Outgoing, dev->hard_header==NULL
   mac_header -> data. ll header is still not built!
   data       -> data

Resume
  If dev->hard_header==NULL we are unlikely to restore sensible ll header.


On transmit:
------------

dev->hard_header != NULL
   mac_header -> ll header
   data       -> ll header

dev->hard_header == NULL (ll header is added by device, we cannot control it)
   mac_header -> data
   data       -> data

   We should set nh.raw on output to correct posistion,
   packet classifier depends on it.
 */

/* Private packet socket structures. */

/* identical to struct packet_mreq except it has
 * a longer address field.
 */
struct packet_mreq_max {
	int		mr_ifindex;
	unsigned short	mr_type;
	unsigned short	mr_alen;
	unsigned char	mr_address[MAX_ADDR_LEN];
};

union tpacket_uhdr {
	struct tpacket_hdr  *h1;
	struct tpacket2_hdr *h2;
	struct tpacket3_hdr *h3;
	void *raw;
};

static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
		int closing, int tx_ring);

#define V3_ALIGNMENT	(8)

#define BLK_HDR_LEN	(ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))

#define BLK_PLUS_PRIV(sz_of_priv) \
	(BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))

#define PGV_FROM_VMALLOC 1

#define BLOCK_STATUS(x)	((x)->hdr.bh1.block_status)
#define BLOCK_NUM_PKTS(x)	((x)->hdr.bh1.num_pkts)
#define BLOCK_O2FP(x)		((x)->hdr.bh1.offset_to_first_pkt)
#define BLOCK_LEN(x)		((x)->hdr.bh1.blk_len)
#define BLOCK_SNUM(x)		((x)->hdr.bh1.seq_num)
#define BLOCK_O2PRIV(x)	((x)->offset_to_priv)
#define BLOCK_PRIV(x)		((void *)((char *)(x) + BLOCK_O2PRIV(x)))

struct packet_sock;
static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
		       struct packet_type *pt, struct net_device *orig_dev);

static void *packet_previous_frame(struct packet_sock *po,
		struct packet_ring_buffer *rb,
		int status);
static void packet_increment_head(struct packet_ring_buffer *buff);
static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
			struct tpacket_block_desc *);
static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
			struct packet_sock *);
static void prb_retire_current_block(struct tpacket_kbdq_core *,
		struct packet_sock *, unsigned int status);
static int prb_queue_frozen(struct tpacket_kbdq_core *);
static void prb_open_block(struct tpacket_kbdq_core *,
		struct tpacket_block_desc *);
static void prb_retire_rx_blk_timer_expired(unsigned long);
static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
static void prb_init_blk_timer(struct packet_sock *,
		struct tpacket_kbdq_core *,
		void (*func) (unsigned long));
static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
static void prb_clear_rxhash(struct tpacket_kbdq_core *,
		struct tpacket3_hdr *);
static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
		struct tpacket3_hdr *);
static void packet_flush_mclist(struct sock *sk);

struct packet_skb_cb {
	unsigned int origlen;
	union {
		struct sockaddr_pkt pkt;
		struct sockaddr_ll ll;
	} sa;
};

#define PACKET_SKB_CB(__skb)	((struct packet_skb_cb *)((__skb)->cb))

#define GET_PBDQC_FROM_RB(x)	((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
#define GET_PBLOCK_DESC(x, bid)	\
	((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
#define GET_CURR_PBLOCK_DESC_FROM_CORE(x)	\
	((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
#define GET_NEXT_PRB_BLK_NUM(x) \
	(((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
	((x)->kactive_blk_num+1) : 0)

static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
static void __fanout_link(struct sock *sk, struct packet_sock *po);

static int packet_direct_xmit(struct sk_buff *skb)
{
	struct net_device *dev = skb->dev;
	netdev_features_t features;
	struct netdev_queue *txq;
	int ret = NETDEV_TX_BUSY;

	if (unlikely(!netif_running(dev) ||
		     !netif_carrier_ok(dev)))
		goto drop;

	features = netif_skb_features(skb);
	if (skb_needs_linearize(skb, features) &&
	    __skb_linearize(skb))
		goto drop;

	txq = skb_get_tx_queue(dev, skb);

	local_bh_disable();

	HARD_TX_LOCK(dev, txq, smp_processor_id());
	if (!netif_xmit_frozen_or_drv_stopped(txq))
		ret = netdev_start_xmit(skb, dev, txq, false);
	HARD_TX_UNLOCK(dev, txq);

	local_bh_enable();

	if (!dev_xmit_complete(ret))
		kfree_skb(skb);

	return ret;
drop:
	atomic_long_inc(&dev->tx_dropped);
	kfree_skb(skb);
	return NET_XMIT_DROP;
}

static struct net_device *packet_cached_dev_get(struct packet_sock *po)
{
	struct net_device *dev;

	rcu_read_lock();
	dev = rcu_dereference(po->cached_dev);
	if (likely(dev))
		dev_hold(dev);
	rcu_read_unlock();

	return dev;
}

static void packet_cached_dev_assign(struct packet_sock *po,
				     struct net_device *dev)
{
	rcu_assign_pointer(po->cached_dev, dev);
}

static void packet_cached_dev_reset(struct packet_sock *po)
{
	RCU_INIT_POINTER(po->cached_dev, NULL);
}

static bool packet_use_direct_xmit(const struct packet_sock *po)
{
	return po->xmit == packet_direct_xmit;
}

static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
{
	return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
}

static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
{
	const struct net_device_ops *ops = dev->netdev_ops;
	u16 queue_index;

	if (ops->ndo_select_queue) {
		queue_index = ops->ndo_select_queue(dev, skb, NULL,
						    __packet_pick_tx_queue);
		queue_index = netdev_cap_txqueue(dev, queue_index);
	} else {
		queue_index = __packet_pick_tx_queue(dev, skb);
	}

	skb_set_queue_mapping(skb, queue_index);
}

/* register_prot_hook must be invoked with the po->bind_lock held,
 * or from a context in which asynchronous accesses to the packet
 * socket is not possible (packet_create()).
 */
static void register_prot_hook(struct sock *sk)
{
	struct packet_sock *po = pkt_sk(sk);

	if (!po->running) {
		if (po->fanout)
			__fanout_link(sk, po);
		else
			dev_add_pack(&po->prot_hook);

		sock_hold(sk);
		po->running = 1;
	}
}

/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
 * held.   If the sync parameter is true, we will temporarily drop
 * the po->bind_lock and do a synchronize_net to make sure no
 * asynchronous packet processing paths still refer to the elements
 * of po->prot_hook.  If the sync parameter is false, it is the
 * callers responsibility to take care of this.
 */
static void __unregister_prot_hook(struct sock *sk, bool sync)
{
	struct packet_sock *po = pkt_sk(sk);

	po->running = 0;

	if (po->fanout)
		__fanout_unlink(sk, po);
	else
		__dev_remove_pack(&po->prot_hook);

	__sock_put(sk);

	if (sync) {
		spin_unlock(&po->bind_lock);
		synchronize_net();
		spin_lock(&po->bind_lock);
	}
}

static void unregister_prot_hook(struct sock *sk, bool sync)
{
	struct packet_sock *po = pkt_sk(sk);

	if (po->running)
		__unregister_prot_hook(sk, sync);
}

static inline struct page * __pure pgv_to_page(void *addr)
{
	if (is_vmalloc_addr(addr))
		return vmalloc_to_page(addr);
	return virt_to_page(addr);
}

static void __packet_set_status(struct packet_sock *po, void *frame, int status)
{
	union tpacket_uhdr h;

	h.raw = frame;
	switch (po->tp_version) {
	case TPACKET_V1:
		h.h1->tp_status = status;
		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
		break;
	case TPACKET_V2:
		h.h2->tp_status = status;
		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
		break;
	case TPACKET_V3:
	default:
		WARN(1, "TPACKET version not supported.\n");
		BUG();
	}

	smp_wmb();
}

static int __packet_get_status(struct packet_sock *po, void *frame)
{
	union tpacket_uhdr h;

	smp_rmb();

	h.raw = frame;
	switch (po->tp_version) {
	case TPACKET_V1:
		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
		return h.h1->tp_status;
	case TPACKET_V2:
		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
		return h.h2->tp_status;
	case TPACKET_V3:
	default:
		WARN(1, "TPACKET version not supported.\n");
		BUG();
		return 0;
	}
}

static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
				   unsigned int flags)
{
	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);

	if (shhwtstamps &&
	    (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
	    ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
		return TP_STATUS_TS_RAW_HARDWARE;

	if (ktime_to_timespec_cond(skb->tstamp, ts))
		return TP_STATUS_TS_SOFTWARE;

	return 0;
}

static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
				    struct sk_buff *skb)
{
	union tpacket_uhdr h;
	struct timespec ts;
	__u32 ts_status;

	if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
		return 0;

	h.raw = frame;
	switch (po->tp_version) {
	case TPACKET_V1:
		h.h1->tp_sec = ts.tv_sec;
		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
		break;
	case TPACKET_V2:
		h.h2->tp_sec = ts.tv_sec;
		h.h2->tp_nsec = ts.tv_nsec;
		break;
	case TPACKET_V3:
	default:
		WARN(1, "TPACKET version not supported.\n");
		BUG();
	}

	/* one flush is safe, as both fields always lie on the same cacheline */
	flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
	smp_wmb();

	return ts_status;
}

static void *packet_lookup_frame(struct packet_sock *po,
		struct packet_ring_buffer *rb,
		unsigned int position,
		int status)
{
	unsigned int pg_vec_pos, frame_offset;
	union tpacket_uhdr h;

	pg_vec_pos = position / rb->frames_per_block;
	frame_offset = position % rb->frames_per_block;

	h.raw = rb->pg_vec[pg_vec_pos].buffer +
		(frame_offset * rb->frame_size);

	if (status != __packet_get_status(po, h.raw))
		return NULL;

	return h.raw;
}

static void *packet_current_frame(struct packet_sock *po,
		struct packet_ring_buffer *rb,
		int status)
{
	return packet_lookup_frame(po, rb, rb->head, status);
}

static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
{
	del_timer_sync(&pkc->retire_blk_timer);
}

static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
		int tx_ring,
		struct sk_buff_head *rb_queue)
{
	struct tpacket_kbdq_core *pkc;

	pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
			GET_PBDQC_FROM_RB(&po->rx_ring);

	spin_lock_bh(&rb_queue->lock);
	pkc->delete_blk_timer = 1;
	spin_unlock_bh(&rb_queue->lock);

	prb_del_retire_blk_timer(pkc);
}

static void prb_init_blk_timer(struct packet_sock *po,
		struct tpacket_kbdq_core *pkc,
		void (*func) (unsigned long))
{
	init_timer(&pkc->retire_blk_timer);
	pkc->retire_blk_timer.data = (long)po;
	pkc->retire_blk_timer.function = func;
	pkc->retire_blk_timer.expires = jiffies;
}

static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
{
	struct tpacket_kbdq_core *pkc;

	if (tx_ring)
		BUG();

	pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
			GET_PBDQC_FROM_RB(&po->rx_ring);
	prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
}

static int prb_calc_retire_blk_tmo(struct packet_sock *po,
				int blk_size_in_bytes)
{
	struct net_device *dev;
	unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
	struct ethtool_cmd ecmd;
	int err;
	u32 speed;

	rtnl_lock();
	dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
	if (unlikely(!dev)) {
		rtnl_unlock();
		return DEFAULT_PRB_RETIRE_TOV;
	}
	err = __ethtool_get_settings(dev, &ecmd);
	speed = ethtool_cmd_speed(&ecmd);
	rtnl_unlock();
	if (!err) {
		/*
		 * If the link speed is so slow you don't really
		 * need to worry about perf anyways
		 */
		if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
			return DEFAULT_PRB_RETIRE_TOV;
		} else {
			msec = 1;
			div = speed / 1000;
		}
	}

	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);

	if (div)
		mbits /= div;

	tmo = mbits * msec;

	if (div)
		return tmo+1;
	return tmo;
}

static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
			union tpacket_req_u *req_u)
{
	p1->feature_req_word = req_u->req3.tp_feature_req_word;
}

static void init_prb_bdqc(struct packet_sock *po,
			struct packet_ring_buffer *rb,
			struct pgv *pg_vec,
			union tpacket_req_u *req_u, int tx_ring)
{
	struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
	struct tpacket_block_desc *pbd;

	memset(p1, 0x0, sizeof(*p1));

	p1->knxt_seq_num = 1;
	p1->pkbdq = pg_vec;
	pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
	p1->pkblk_start	= pg_vec[0].buffer;
	p1->kblk_size = req_u->req3.tp_block_size;
	p1->knum_blocks	= req_u->req3.tp_block_nr;
	p1->hdrlen = po->tp_hdrlen;
	p1->version = po->tp_version;
	p1->last_kactive_blk_num = 0;
	po->stats.stats3.tp_freeze_q_cnt = 0;
	if (req_u->req3.tp_retire_blk_tov)
		p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
	else
		p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
						req_u->req3.tp_block_size);
	p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
	p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;

	p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
	prb_init_ft_ops(p1, req_u);
	prb_setup_retire_blk_timer(po, tx_ring);
	prb_open_block(p1, pbd);
}

/*  Do NOT update the last_blk_num first.
 *  Assumes sk_buff_head lock is held.
 */
static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
{
	mod_timer(&pkc->retire_blk_timer,
			jiffies + pkc->tov_in_jiffies);
	pkc->last_kactive_blk_num = pkc->kactive_blk_num;
}

/*
 * Timer logic:
 * 1) We refresh the timer only when we open a block.
 *    By doing this we don't waste cycles refreshing the timer
 *	  on packet-by-packet basis.
 *
 * With a 1MB block-size, on a 1Gbps line, it will take
 * i) ~8 ms to fill a block + ii) memcpy etc.
 * In this cut we are not accounting for the memcpy time.
 *
 * So, if the user sets the 'tmo' to 10ms then the timer
 * will never fire while the block is still getting filled
 * (which is what we want). However, the user could choose
 * to close a block early and that's fine.
 *
 * But when the timer does fire, we check whether or not to refresh it.
 * Since the tmo granularity is in msecs, it is not too expensive
 * to refresh the timer, lets say every '8' msecs.
 * Either the user can set the 'tmo' or we can derive it based on
 * a) line-speed and b) block-size.
 * prb_calc_retire_blk_tmo() calculates the tmo.
 *
 */
static void prb_retire_rx_blk_timer_expired(unsigned long data)
{
	struct packet_sock *po = (struct packet_sock *)data;
	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
	unsigned int frozen;
	struct tpacket_block_desc *pbd;

	spin_lock(&po->sk.sk_receive_queue.lock);

	frozen = prb_queue_frozen(pkc);
	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

	if (unlikely(pkc->delete_blk_timer))
		goto out;

	/* We only need to plug the race when the block is partially filled.
	 * tpacket_rcv:
	 *		lock(); increment BLOCK_NUM_PKTS; unlock()
	 *		copy_bits() is in progress ...
	 *		timer fires on other cpu:
	 *		we can't retire the current block because copy_bits
	 *		is in progress.
	 *
	 */
	if (BLOCK_NUM_PKTS(pbd)) {
		while (atomic_read(&pkc->blk_fill_in_prog)) {
			/* Waiting for skb_copy_bits to finish... */
			cpu_relax();
		}
	}

	if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
		if (!frozen) {
			prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
			if (!prb_dispatch_next_block(pkc, po))
				goto refresh_timer;
			else
				goto out;
		} else {
			/* Case 1. Queue was frozen because user-space was
			 *	   lagging behind.
			 */
			if (prb_curr_blk_in_use(pkc, pbd)) {
				/*
				 * Ok, user-space is still behind.
				 * So just refresh the timer.
				 */
				goto refresh_timer;
			} else {
			       /* Case 2. queue was frozen,user-space caught up,
				* now the link went idle && the timer fired.
				* We don't have a block to close.So we open this
				* block and restart the timer.
				* opening a block thaws the queue,restarts timer
				* Thawing/timer-refresh is a side effect.
				*/
				prb_open_block(pkc, pbd);
				goto out;
			}
		}
	}

refresh_timer:
	_prb_refresh_rx_retire_blk_timer(pkc);

out:
	spin_unlock(&po->sk.sk_receive_queue.lock);
}

static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
		struct tpacket_block_desc *pbd1, __u32 status)
{
	/* Flush everything minus the block header */

#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
	u8 *start, *end;

	start = (u8 *)pbd1;

	/* Skip the block header(we know header WILL fit in 4K) */
	start += PAGE_SIZE;

	end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
	for (; start < end; start += PAGE_SIZE)
		flush_dcache_page(pgv_to_page(start));

	smp_wmb();
#endif

	/* Now update the block status. */

	BLOCK_STATUS(pbd1) = status;

	/* Flush the block header */

#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
	start = (u8 *)pbd1;
	flush_dcache_page(pgv_to_page(start));

	smp_wmb();
#endif
}

/*
 * Side effect:
 *
 * 1) flush the block
 * 2) Increment active_blk_num
 *
 * Note:We DONT refresh the timer on purpose.
 *	Because almost always the next block will be opened.
 */
static void prb_close_block(struct tpacket_kbdq_core *pkc1,
		struct tpacket_block_desc *pbd1,
		struct packet_sock *po, unsigned int stat)
{
	__u32 status = TP_STATUS_USER | stat;

	struct tpacket3_hdr *last_pkt;
	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
	struct sock *sk = &po->sk;

	if (po->stats.stats3.tp_drops)
		status |= TP_STATUS_LOSING;

	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
	last_pkt->tp_next_offset = 0;

	/* Get the ts of the last pkt */
	if (BLOCK_NUM_PKTS(pbd1)) {
		h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
		h1->ts_last_pkt.ts_nsec	= last_pkt->tp_nsec;
	} else {
		/* Ok, we tmo'd - so get the current time */
		struct timespec ts;
		getnstimeofday(&ts);
		h1->ts_last_pkt.ts_sec = ts.tv_sec;
		h1->ts_last_pkt.ts_nsec	= ts.tv_nsec;
	}

	smp_wmb();

	/* Flush the block */
	prb_flush_block(pkc1, pbd1, status);

	sk->sk_data_ready(sk);

	pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
}

static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
{
	pkc->reset_pending_on_curr_blk = 0;
}

/*
 * Side effect of opening a block:
 *
 * 1) prb_queue is thawed.
 * 2) retire_blk_timer is refreshed.
 *
 */
static void prb_open_block(struct tpacket_kbdq_core *pkc1,
	struct tpacket_block_desc *pbd1)
{
	struct timespec ts;
	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;

	smp_rmb();

	/* We could have just memset this but we will lose the
	 * flexibility of making the priv area sticky
	 */

	BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
	BLOCK_NUM_PKTS(pbd1) = 0;
	BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);

	getnstimeofday(&ts);

	h1->ts_first_pkt.ts_sec = ts.tv_sec;
	h1->ts_first_pkt.ts_nsec = ts.tv_nsec;

	pkc1->pkblk_start = (char *)pbd1;
	pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);

	BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
	BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;

	pbd1->version = pkc1->version;
	pkc1->prev = pkc1->nxt_offset;
	pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;

	prb_thaw_queue(pkc1);
	_prb_refresh_rx_retire_blk_timer(pkc1);

	smp_wmb();
}

/*
 * Queue freeze logic:
 * 1) Assume tp_block_nr = 8 blocks.
 * 2) At time 't0', user opens Rx ring.
 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
 * 4) user-space is either sleeping or processing block '0'.
 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
 *    it will close block-7,loop around and try to fill block '0'.
 *    call-flow:
 *    __packet_lookup_frame_in_block
 *      prb_retire_current_block()
 *      prb_dispatch_next_block()
 *        |->(BLOCK_STATUS == USER) evaluates to true
 *    5.1) Since block-0 is currently in-use, we just freeze the queue.
 * 6) Now there are two cases:
 *    6.1) Link goes idle right after the queue is frozen.
 *         But remember, the last open_block() refreshed the timer.
 *         When this timer expires,it will refresh itself so that we can
 *         re-open block-0 in near future.
 *    6.2) Link is busy and keeps on receiving packets. This is a simple
 *         case and __packet_lookup_frame_in_block will check if block-0
 *         is free and can now be re-used.
 */
static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
				  struct packet_sock *po)
{
	pkc->reset_pending_on_curr_blk = 1;
	po->stats.stats3.tp_freeze_q_cnt++;
}

#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))

/*
 * If the next block is free then we will dispatch it
 * and return a good offset.
 * Else, we will freeze the queue.
 * So, caller must check the return value.
 */
static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
		struct packet_sock *po)
{
	struct tpacket_block_desc *pbd;

	smp_rmb();

	/* 1. Get current block num */
	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

	/* 2. If this block is currently in_use then freeze the queue */
	if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
		prb_freeze_queue(pkc, po);
		return NULL;
	}

	/*
	 * 3.
	 * open this block and return the offset where the first packet
	 * needs to get stored.
	 */
	prb_open_block(pkc, pbd);
	return (void *)pkc->nxt_offset;
}

static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
		struct packet_sock *po, unsigned int status)
{
	struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

	/* retire/close the current block */
	if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
		/*
		 * Plug the case where copy_bits() is in progress on
		 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
		 * have space to copy the pkt in the current block and
		 * called prb_retire_current_block()
		 *
		 * We don't need to worry about the TMO case because
		 * the timer-handler already handled this case.
		 */
		if (!(status & TP_STATUS_BLK_TMO)) {
			while (atomic_read(&pkc->blk_fill_in_prog)) {
				/* Waiting for skb_copy_bits to finish... */
				cpu_relax();
			}
		}
		prb_close_block(pkc, pbd, po, status);
		return;
	}
}

static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
				      struct tpacket_block_desc *pbd)
{
	return TP_STATUS_USER & BLOCK_STATUS(pbd);
}

static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
{
	return pkc->reset_pending_on_curr_blk;
}

static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
{
	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
	atomic_dec(&pkc->blk_fill_in_prog);
}

static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
			struct tpacket3_hdr *ppd)
{
	ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
}

static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
			struct tpacket3_hdr *ppd)
{
	ppd->hv1.tp_rxhash = 0;
}

static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
			struct tpacket3_hdr *ppd)
{
	if (skb_vlan_tag_present(pkc->skb)) {
		ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
		ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
		ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
	} else {
		ppd->hv1.tp_vlan_tci = 0;
		ppd->hv1.tp_vlan_tpid = 0;
		ppd->tp_status = TP_STATUS_AVAILABLE;
	}
}

static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
			struct tpacket3_hdr *ppd)
{
	ppd->hv1.tp_padding = 0;
	prb_fill_vlan_info(pkc, ppd);

	if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
		prb_fill_rxhash(pkc, ppd);
	else
		prb_clear_rxhash(pkc, ppd);
}

static void prb_fill_curr_block(char *curr,
				struct tpacket_kbdq_core *pkc,
				struct tpacket_block_desc *pbd,
				unsigned int len)
{
	struct tpacket3_hdr *ppd;

	ppd  = (struct tpacket3_hdr *)curr;
	ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
	pkc->prev = curr;
	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
	BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
	BLOCK_NUM_PKTS(pbd) += 1;
	atomic_inc(&pkc->blk_fill_in_prog);
	prb_run_all_ft_ops(pkc, ppd);
}

/* Assumes caller has the sk->rx_queue.lock */
static void *__packet_lookup_frame_in_block(struct packet_sock *po,
					    struct sk_buff *skb,
						int status,
					    unsigned int len
					    )
{
	struct tpacket_kbdq_core *pkc;
	struct tpacket_block_desc *pbd;
	char *curr, *end;

	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

	/* Queue is frozen when user space is lagging behind */
	if (prb_queue_frozen(pkc)) {
		/*
		 * Check if that last block which caused the queue to freeze,
		 * is still in_use by user-space.
		 */
		if (prb_curr_blk_in_use(pkc, pbd)) {
			/* Can't record this packet */
			return NULL;
		} else {
			/*
			 * Ok, the block was released by user-space.
			 * Now let's open that block.
			 * opening a block also thaws the queue.
			 * Thawing is a side effect.
			 */
			prb_open_block(pkc, pbd);
		}
	}

	smp_mb();
	curr = pkc->nxt_offset;
	pkc->skb = skb;
	end = (char *)pbd + pkc->kblk_size;

	/* first try the current block */
	if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
		prb_fill_curr_block(curr, pkc, pbd, len);
		return (void *)curr;
	}

	/* Ok, close the current block */
	prb_retire_current_block(pkc, po, 0);

	/* Now, try to dispatch the next block */
	curr = (char *)prb_dispatch_next_block(pkc, po);
	if (curr) {
		pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
		prb_fill_curr_block(curr, pkc, pbd, len);
		return (void *)curr;
	}

	/*
	 * No free blocks are available.user_space hasn't caught up yet.
	 * Queue was just frozen and now this packet will get dropped.
	 */
	return NULL;
}

static void *packet_current_rx_frame(struct packet_sock *po,
					    struct sk_buff *skb,
					    int status, unsigned int len)
{
	char *curr = NULL;
	switch (po->tp_version) {
	case TPACKET_V1:
	case TPACKET_V2:
		curr = packet_lookup_frame(po, &po->rx_ring,
					po->rx_ring.head, status);
		return curr;
	case TPACKET_V3:
		return __packet_lookup_frame_in_block(po, skb, status, len);
	default:
		WARN(1, "TPACKET version not supported\n");
		BUG();
		return NULL;
	}
}

static void *prb_lookup_block(struct packet_sock *po,
				     struct packet_ring_buffer *rb,
				     unsigned int idx,
				     int status)
{
	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);

	if (status != BLOCK_STATUS(pbd))
		return NULL;
	return pbd;
}

static int prb_previous_blk_num(struct packet_ring_buffer *rb)
{
	unsigned int prev;
	if (rb->prb_bdqc.kactive_blk_num)
		prev = rb->prb_bdqc.kactive_blk_num-1;
	else
		prev = rb->prb_bdqc.knum_blocks-1;
	return prev;
}

/* Assumes caller has held the rx_queue.lock */
static void *__prb_previous_block(struct packet_sock *po,
					 struct packet_ring_buffer *rb,
					 int status)
{
	unsigned int previous = prb_previous_blk_num(rb);
	return prb_lookup_block(po, rb, previous, status);
}

static void *packet_previous_rx_frame(struct packet_sock *po,
					     struct packet_ring_buffer *rb,
					     int status)
{
	if (po->tp_version <= TPACKET_V2)
		return packet_previous_frame(po, rb, status);

	return __prb_previous_block(po, rb, status);
}

static void packet_increment_rx_head(struct packet_sock *po,
					    struct packet_ring_buffer *rb)
{
	switch (po->tp_version) {
	case TPACKET_V1:
	case TPACKET_V2:
		return packet_increment_head(rb);
	case TPACKET_V3:
	default:
		WARN(1, "TPACKET version not supported.\n");
		BUG();
		return;
	}
}

static void *packet_previous_frame(struct packet_sock *po,
		struct packet_ring_buffer *rb,
		int status)
{
	unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
	return packet_lookup_frame(po, rb, previous, status);
}

static void packet_increment_head(struct packet_ring_buffer *buff)
{
	buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
}

static void packet_inc_pending(struct packet_ring_buffer *rb)
{
	this_cpu_inc(*rb->pending_refcnt);
}

static void packet_dec_pending(struct packet_ring_buffer *rb)
{
	this_cpu_dec(*rb->pending_refcnt);
}

static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
{
	unsigned int refcnt = 0;
	int cpu;

	/* We don't use pending refcount in rx_ring. */
	if (rb->pending_refcnt == NULL)
		return 0;

	for_each_possible_cpu(cpu)
		refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);

	return refcnt;
}

static int packet_alloc_pending(struct packet_sock *po)
{
	po->rx_ring.pending_refcnt = NULL;

	po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
	if (unlikely(po->tx_ring.pending_refcnt == NULL))
		return -ENOBUFS;

	return 0;
}

static void packet_free_pending(struct packet_sock *po)
{
	free_percpu(po->tx_ring.pending_refcnt);
}

static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
{
	struct sock *sk = &po->sk;
	bool has_room;

	if (po->prot_hook.func != tpacket_rcv)
		return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
			<= sk->sk_rcvbuf;

	spin_lock(&sk->sk_receive_queue.lock);
	if (po->tp_version == TPACKET_V3)
		has_room = prb_lookup_block(po, &po->rx_ring,
					    po->rx_ring.prb_bdqc.kactive_blk_num,
					    TP_STATUS_KERNEL);
	else
		has_room = packet_lookup_frame(po, &po->rx_ring,
					       po->rx_ring.head,
					       TP_STATUS_KERNEL);
	spin_unlock(&sk->sk_receive_queue.lock);

	return has_room;
}

static void packet_sock_destruct(struct sock *sk)
{
	skb_queue_purge(&sk->sk_error_queue);

	WARN_ON(atomic_read(&sk->sk_rmem_alloc));
	WARN_ON(atomic_read(&sk->sk_wmem_alloc));

	if (!sock_flag(sk, SOCK_DEAD)) {
		pr_err("Attempt to release alive packet socket: %p\n", sk);
		return;
	}

	sk_refcnt_debug_dec(sk);
}

static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
{
	int x = atomic_read(&f->rr_cur) + 1;

	if (x >= num)
		x = 0;

	return x;
}

static unsigned int fanout_demux_hash(struct packet_fanout *f,
				      struct sk_buff *skb,
				      unsigned int num)
{
	return reciprocal_scale(skb_get_hash(skb), num);
}

static unsigned int fanout_demux_lb(struct packet_fanout *f,
				    struct sk_buff *skb,
				    unsigned int num)
{
	int cur, old;

	cur = atomic_read(&f->rr_cur);
	while ((old = atomic_cmpxchg(&f->rr_cur, cur,
				     fanout_rr_next(f, num))) != cur)
		cur = old;
	return cur;
}

static unsigned int fanout_demux_cpu(struct packet_fanout *f,
				     struct sk_buff *skb,
				     unsigned int num)
{
	return smp_processor_id() % num;
}

static unsigned int fanout_demux_rnd(struct packet_fanout *f,
				     struct sk_buff *skb,
				     unsigned int num)
{
	return prandom_u32_max(num);
}

static unsigned int fanout_demux_rollover(struct packet_fanout *f,
					  struct sk_buff *skb,
					  unsigned int idx, unsigned int skip,
					  unsigned int num)
{
	unsigned int i, j;

	i = j = min_t(int, f->next[idx], num - 1);
	do {
		if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
			if (i != j)
				f->next[idx] = i;
			return i;
		}
		if (++i == num)
			i = 0;
	} while (i != j);

	return idx;
}

static unsigned int fanout_demux_qm(struct packet_fanout *f,
				    struct sk_buff *skb,
				    unsigned int num)
{
	return skb_get_queue_mapping(skb) % num;
}

static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
{
	return f->flags & (flag >> 8);
}

static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
			     struct packet_type *pt, struct net_device *orig_dev)
{
	struct packet_fanout *f = pt->af_packet_priv;
	unsigned int num = f->num_members;
	struct packet_sock *po;
	unsigned int idx;

	if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
	    !num) {
		kfree_skb(skb);
		return 0;
	}

	switch (f->type) {
	case PACKET_FANOUT_HASH:
	default:
		if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
			skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
			if (!skb)
				return 0;
		}
		idx = fanout_demux_hash(f, skb, num);
		break;
	case PACKET_FANOUT_LB:
		idx = fanout_demux_lb(f, skb, num);
		break;
	case PACKET_FANOUT_CPU:
		idx = fanout_demux_cpu(f, skb, num);
		break;
	case PACKET_FANOUT_RND:
		idx = fanout_demux_rnd(f, skb, num);
		break;
	case PACKET_FANOUT_QM:
		idx = fanout_demux_qm(f, skb, num);
		break;
	case PACKET_FANOUT_ROLLOVER:
		idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
		break;
	}

	po = pkt_sk(f->arr[idx]);
	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
	    unlikely(!packet_rcv_has_room(po, skb))) {
		idx = fanout_demux_rollover(f, skb, idx, idx, num);
		po = pkt_sk(f->arr[idx]);
	}

	return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
}

DEFINE_MUTEX(fanout_mutex);
EXPORT_SYMBOL_GPL(fanout_mutex);
static LIST_HEAD(fanout_list);

static void __fanout_link(struct sock *sk, struct packet_sock *po)
{
	struct packet_fanout *f = po->fanout;

	spin_lock(&f->lock);
	f->arr[f->num_members] = sk;
	smp_wmb();
	f->num_members++;
	spin_unlock(&f->lock);
}

static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
{
	struct packet_fanout *f = po->fanout;
	int i;

	spin_lock(&f->lock);
	for (i = 0; i < f->num_members; i++) {
		if (f->arr[i] == sk)
			break;
	}
	BUG_ON(i >= f->num_members);
	f->arr[i] = f->arr[f->num_members - 1];
	f->num_members--;
	spin_unlock(&f->lock);
}

static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
{
	if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
		return true;

	return false;
}

static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
{
	struct packet_sock *po = pkt_sk(sk);
	struct packet_fanout *f, *match;
	u8 type = type_flags & 0xff;
	u8 flags = type_flags >> 8;
	int err;

	switch (type) {
	case PACKET_FANOUT_ROLLOVER:
		if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
			return -EINVAL;
	case PACKET_FANOUT_HASH:
	case PACKET_FANOUT_LB:
	case PACKET_FANOUT_CPU:
	case PACKET_FANOUT_RND:
	case PACKET_FANOUT_QM:
		break;
	default:
		return -EINVAL;
	}

	if (!po->running)
		return -EINVAL;

	if (po->fanout)
		return -EALREADY;

	mutex_lock(&fanout_mutex);
	match = NULL;
	list_for_each_entry(f, &fanout_list, list) {
		if (f->id == id &&
		    read_pnet(&f->net) == sock_net(sk)) {
			match = f;
			break;
		}
	}
	err = -EINVAL;
	if (match && match->flags != flags)
		goto out;
	if (!match) {
		err = -ENOMEM;
		match = kzalloc(sizeof(*match), GFP_KERNEL);
		if (!match)
			goto out;
		write_pnet(&match->net, sock_net(sk));
		match->id = id;
		match->type = type;
		match->flags = flags;
		atomic_set(&match->rr_cur, 0);
		INIT_LIST_HEAD(&match->list);
		spin_lock_init(&match->lock);
		atomic_set(&match->sk_ref, 0);
		match->prot_hook.type = po->prot_hook.t