aboutsummaryrefslogblamecommitdiffstats
path: root/net/ipv4/tcp_output.c
blob: 4522579aaca2d24b7be61abac658a3c8cbd2b960 (plain) (tree)
1
2
3
4
5
6
7
8






                                                                            
                         



























                                                                           

                               


                           
                      
                         
                             
 
                             
 






                                            

                                          





                                          

                                                                              
 
                                                             
                                                                         
 
                                                         
                                         
                                                     
 
                                               
 


                                                   
                                               
                                                                         
                                  
 

                                                              

 

                                                                              




                                                                           
                                                             
 
                                               
 


                                                                               

                                   
                                       


















                                                                           
                                                       

                             






                                                             





                                                                        


                                                       
 
                                         
                                                               

                                
                                                
 
                                                    

                                               
                                                                            

                                               
                                           


                              
                                                               
                                                    
                                                
 
                                                         
                                      
 


                                                    



                                                                   
           




                                                                      

 
                                 

                                                                         
 

                                         
                                                                    
                                                                       

                                                                           


                                                                          


                                                                       

                                                      








                                                                  
                                                                             
                                                                   

                                                               




                                                                         
                                                              



                                                                       
                                              

                                                             





                                                                    
           
                                                                    

                                                        



                                                             
 
                        
                        
                                                               
                                                                                 
                                                           
                                                         

                                                             
         
                                                                  
                                                                                
 
                                         





                                                                      
                                             

                                         
                                  



                                              
                                






                                                            


                                                                      
                                                                     






                                                          

                                                                    






                                                                          
                                                             
                           
                                   





                                                                            



                       
                                    
                                                                     
 

                                               
                                                  
                                          
                                                          

                                          
                                  

 
                                  
                                                                  

                                         
                                                      
                                                                
                                                      






                                                               

                          

                      
                                                                      
                                           
                                                          
                                          


         








                                                                          
           
                                                                      
 
                                  


                            


                                                                         
                                                              
                                                               







                                                                             
                                                                
                                                                    
                                            

                                                                             
                                                   



                                                                        
                                    


         




                                                                         
                                          
 
                                           

                                    
                                   

                                   
                                              



                                       
                                                          



                                         


                                        
                                        
                                        















                                                               

                        

                                                           

                                                                      
                                                            
                                                                   

                                                                              

  


                                                                         

                                                                         
                                                                        






                                                                          
                                                               

                                                           
                                                           
 
                                             

                                                                        

                                                   
                         
         
 





                                                    

                                                                



                                                                  
                                                          









                                                                
                                                        





                                                        
                                                








                                                                




                                                    
                                                                            
                                                                              
 

                                                                      


                                                                
 
                                     
         


                                                                        












                                                                       
 



                                                     
                 
                                      
         


































                                                                           

 


                                                             
                                                                         
                                                             

                                                            
                                         
                                                      
                                                                 
 
                    
                        
                                                      
                                                  




                                                            

         










                                                                               
                                         
 
                                                                        
                                           
                                                                    
                                                   
                                                    
         
                                                                   
                                                 
                                               
                                                    
         
                                                         
                                                       
                                                           
                                                              

         
                                                    



                                                                          





                                                                  
                                                                            

                 
 

                                             
                                                

 
                                      

                                                                



                                                                             
 
                                                       
                                                      
 
                        
                  
                                            






                                                                         
                                                  

         
 
                                           
                        
                                         


                                            
                                               
                                                    
         
                                      
                                           
                                                                            
                                             
                                                    


                                                       
                                               
                                                              
         




                                                              






                                                                 
 

                                                                
                                                

 


                                                               
                                                                                 
                                                                     

                                                                    
                                         
                              
                               
 

                          
                    
                        
                                                      
                                                  




                                                           
         



                                           
                                                                              



                                                   

                                                            
                                                                           
                                       
                                                      






                                                                       
 
 








                                                                                 
                                                                      










                                                                    
                                          


                                                               



                                                     

                                                               
                                                      
                 

                                                                    
                                              
         
 









                                                                        
  
                                               
                                                          
                                                   



















                                                              
                                        

                                                         
                                    



                            



                                                         








                                                           
                                    
 

                                                                          
                                         


                                                   
                                                                     
 



                                        










                                                                            
                                                
                                            

                               
                                                 
                                             

                               
                                                
                                                           

                               


















                                                                   
                                                                

                   
                                   


                                         
                                        



                                                                        
                                                                              







                                                                                
                                                                                                    
                         
 
                                                                
                                        
                           
 


                                                                     
                                                              
                                                              


                                 

                                                        
                                                 
                                               
                                                    

                                                        
                                         
                       
         

                    

 

                                                                    




                                                                                 
 

                            
 


                                 

                                                                           
 

                                         
                                                     
                                                        




                                                                            
                                                                      





                                                                                  

                 


                                                                          










                                                             

                                                                        
 



                                                               
                                    
                                                       
                                    
                                   
                          
                         


                                             
                        


                                                                        
                       

                                                                        
                           







                                                                


                                        
 
                           
                              
                                       
 
                                                  




                                                                          
 
                                                                          
                                                                 
                                           


                                                                            
           
                                                                
 






                                                                

                                        


                        
                                                                              
                                      
                                                        
 

                                                                     
                                               
                                        

                                                   
                                                  
                                                 
                                                                        
                                                        
 

                                    
 
                                                                              




                                                                         
                                                    

                                    
         
 
                                                         
                                                    








                                                                  


                                                                
                                                    
                                                                  
                                                             


         
                                               
 
                                                
                                                                     
 
                                          
                                            
                                                         
                                                             
         
 
                                                                         

                                                            
 
                                            
                                                                   
                                                        
                                                     
 
                                                                               




                                                               
                                                                     
 



                                         
                           
                                                                  

                                            
                   

 






                                                                               
                                                   









                                                                            
                                  
                                          

                                            

 
                                           
                                                                           
 
                                  


                                                        
                                           
                                                  
                
                                                                         
                                                        


         


                                                                             
                                                                                   















                                                                   

                                                                               
                                                           




                                          





                                                               



                                                                          
                                               






                                                                   

                                                                             


         





                                                                           











                                                                  

                                                                       
                                                                

                                                           

                                                           
                                                 


                                         
                              
                 
                 
 

                                    
 



                                       
                                  


                                             
                                                         
                  
                                                                 
 

                                             


                                      






                                                                  


                                                                        
                                                           
                                        
 
                                  
 
                                           
 
                                   
                                       
 

                                         
                                                               

                                            
 


                                                       


                                                            
                                                              

                                                             
 

                                                         


                                            
                                   
                                                               

                                                                              



                 

                                                                       
   
                                                         
 
                                       

                      




                                                
                                 
         

                  


                                                            

                                  
                                               
                                    
                        
                                                            
                                  

                                                                          




                                        
                             
 

                                 
                   

 
                                                            

                                                                

                           
                                         

                               
                                                    

                                    
                                          
 





                                                     
 
                                                                          
                                    
                                                            



                 

                                                             
 

                                                               






                                                                                   







                                                                          









                                                                               


                       







                                                                    
                      
                                            
 

                                                               






                                                






                                                                      

                   
                              
 
                                 



                                                         
                                       
 
                                                                       
                                                                                    
                                                                 
                                                                                       
                                       
                                    
                                                                
 
                             
 





                                                                           
                                                       

                                        
                                                                      








                                                            

                                                              
   


                                                    
                                                         
                    
 

                                                   
 
                                           
                                                     

                                      
                                      

                                                                                       
                                


                       
                            


                                                                  
   
                                             
 

                                                       
                    
                                

                                    


                                

                                       
                                                          


                                                        









                                                                              
 


                       


















                                                                              
                                           

 
                                                                    
 
                                                                            
                                         
 








                                                                  
 
                                      

                                            
                                                   




                                                            
                                                                          
                                                                                          
                                          
                                                         




                                                                            
                                                                    

                                                                        
                                                                 


                                                                               


         













                                                                
   














                                                                           
                                        




                                                                          



                                                  

                                                                        


                        


                                                                





                                                        
                                                         
 
                    

 





                                                                            
                              
 




                                                                   
                                                         

 





                                                                  
 
                                               
                                             
 
                                                        
                                     
 

                                                                         
 

                                       

                               
 




                                                                           
                                                       


                                        




                                                                         

                                                                   
 
                                      

                                                                             

                                                        



                                              

                              
 




                                                         

 
                                 
                                                               

                     
                                                                       


                                           
                                                                         
                                                   




                                               
 
                                                         

            

                                                                                       







                                                                              
                            
 

                                                                              
                            
 
                                                              
                            
 
                     


                                                                      


                                                       





                                                         
                                                

 






                                                                      
                                                                               
                                                        
 
                                  
                             
                 

                                                                 
                                      

                                                                
 
                                                     
                            

                               

                                             
                               







                                                                  


                                                                        



                                                                  

                                        
                                           
                                  
                                       

                                                               

                                            

                                            
                                   
                                                                             








                                                                     
                                                                      


                                                       
 
                                                               
                                                 
                                         
                             
                        
                  
 
                                                   

                              
                                                  


                                                                      
           

                                                                        
                              
 

                                              

                                          
 
                                                          





                                                                        
                                                         
                                              
                              
 



                                                                             
                                                                               
                          




                                                                           
                                     
                                   
                                      





                                                                     
                                                                         
                                      

         



                                             
                                                   
                                                                              
                                                                       

                              
















                                                      
 
                                                              

                                                        
                              
 
                    

         
                     

 








                                                         
                                                                










                                                                     
                                                                


         

















                                                                      
                                          



                                                                

                                                            

                       

                                         
                                                         
                                         
                                          
                                       
                       
                        
                      
                    
                     



                                           






                                                                

                          



                                                                 
                                      

                                                                      
                                                                        
                                                                            



                                                                        
                                                                           




                                                                  



                                                          
                                                      

                          

                                      
                                                              
                         
 


                                                                                




                                  


                                                              
                                                    
                                                                      
                  
                          

                                             
 
                                


                                                                      
                                                 

                                     
                                           
 
                                                     
                                                
 
                
                                                      
                                                              
                                                                 



                                                                  
                                                                                  



                                                                         
                                                        
                                                  
                        
                                                                                   
                                                                            

                                                         

                                                            
                                                                   




                                                     


                                      
         
                                           

                                                              

                                                                    

                                                             
                                              
                               
                                                  

                                                                           

                                                                        






                         
                                             
 














                                                             

 













                                                                             
                            
 


                                                                 


                                                                                

                         
                                                        
                                                         



                                                                       
                                            

                                     
                                                          




                                                                
                                                              




                                    

                                                                          
                                      
                                              
 

                                                                   








                                                                   

                                                                             









                                                                          







                                                                       
                                               


                                                      

 



                                                               



                                                                 


                                                                 

                                                                          
   

                                                                              

                                         
                            

                                         
                   
                                                              
                     
 
                      
 
                               



                                           
                                     


                                        

         
                                             
                                           

                                   



                                                                                               
                                                        


                                                                    


                                         
                                                           
                                  
 
                                                    






                                                                 
 

                                                                    
                              
                 
 
                                    




                                                                                  
                                        
                                                                           
                                                                             

                                      
 
                                
                                                      






                                                                       
                                                                         

                              

                                                      
 
                                                                
                              
 
       


                                                                
                                                 
 
                                                      
                                                 


                              
         
 




                                                              
                                

                                                 


                                                                
                                                           
                                                                               
                                                       
                             
         
                                                              

 
                                                                 


                                                         
                                  
                          
 


                                                                        
                             

                             
                                                                    
                                                                      
                                                                                
           
                                                         
                                                   

                                                  

                             
                                                             

                                                                         
           








                                                             
 
                                                                            


                                                                              

                                                                              
 

                                                               


                    


                                                                  



                                                              
                                                 

                                                                    




                            
                                                                           



                                         
                                         


                                      
 
                                





                                                                      
         
                                              






                                                                            
 



                                                         


                                             




                                                              

                                                                         
                                                       
                                         
                                       




                                                  
                                             
                                 
 
                                                
                                       
 
           
                                                             

                                                                  
            
                          

 



                                                          

                                                                     
 






                                                                      
                                                   
                                                        
                                          

 




                                                                     
                                                


                                           
                                                                          

 

                                                         
  

















                                                                       
  

                                                                 
  
                                              
  


























                                                                      
                                                         
                                         
                                                                     




                                                                          
                                         
                                       

                                                                     

                   
                                         
                                 


                                 
                                             
                                         
 
                                                  

                                                                
 












                                                                                








                                                                       






                                                                           
                                                                     
                
                                     








                                                                           
                                                            
                                             
                                                                 
                                            




                      

                                                            
 


                                                           

                                                                 
                                                                             
                                                   

                                                          


         
                                                         
                                                                      

                                         
                                                    
                          
 
                                      
 
                                                                          
 






                                                                               
                                                    
 

                                                                 
 
                                                                          
                                                                      




                                                                                     
                                                         

                                                            


                                                
 

                                                                  

                                               
                                                    
                    

 
                                        
                                                                              

                                    
                             
                            
                             
                                                                          
                                                         
                             
 
                    

 


                                                                 




                                                                         
                          
 
                                                            
                       
                                                    

                       
                                             


                                               


                                                 


                                  
                                      




                                 



                                                                     

                                                  


         



                                                                   
                                                                        
 
                                                         
                                         
                             

                           
 

                                       
                                               
 
                                                                        
                                                                    
           
                                               

                                                                      

                               


                                             
                                                        



                                                                              



                                                                               


                                                                       
                                      
 




                                                                 

                                                             

                               

                             

                                                                     
                                                                    
                

                                                 
 






                                                                   

         



                                                                            





                                                                       
                                      
 





                                                                        

                                     





                                                                                
                           
                                                                              

                                                   
                
                                                               
         
 




                                                                          



                                                                     
                           
                                                  
                                   
                                                                            
         
                   

 
                                                                      

                                         
                                                      

                       
                         
                                                                     
                                                                    



                                                         
         
 



                                                             


                                                






                                                                 


                                               
                                                               
                                                     
                                         
                     
                    
 


                             

                                                  
                                                         
                                   
                            
                         
 


                                         
                                                                 
                          
                                                      
 

                                                                
                               
                                                 



                                                                           
 

                                                      
                                                    
                                                                                           

                                           
 





                                                                        
 
                                                                       

                                 


                                                      
                                                      
                               
 
                                                                          
 
                                              

                                                           
                                      
                                                                



                                                                    


         

                                                               

                                                                

                                                                 
   
                                                      
 
                




                                                     

                                         

                                                           

 

                                                                          


                                  
                                                              
                                         
 



                                                                              
           



                                                       


                                                          
                                
                                                








                                                                                
                




                                                                          
                 
                                                         
                                                 
                                                          
                                                                                      
                                                        
                                                              

                                       
                                                                          




                                                                      
                                     
   
                                                           
 

                            

                                                     


                                                                         
                                                                      




                                                                 
                                                         
                                                      
                                       
                          
                                                   
                                                                      




                                                                       

 

                                                                      





                                                                      
                            
 
                                     
                                                                 
                                                            

                               
                                                         
                                      




                                                                 
                                  
                                               
                                                                  
                                                               
                                                   
                                                                    

                                                             


                                   
                                                         
                                             
         
                                                        

 




                                        



                                                     
                                                                             
                                                         
                                                                
                                                                 
 
                                                       
                                               
                                          

                                    
                            
                          
                
 
                                                    

                                 
                            
         


                                         

                               
                                                     






                                                                    




                                                                             
                      
         
                              
 
                                                        
 
                                       

                                     
                                                                

            
                                                    


                        
                                                                            
      
                                                                  

                                                                           
 

                                        
 
                                        


                                             
                                     
                                         
                                     
                                  

                                               

                                                                        

                                                                            
                                                          
                                                           
                                          
                                                       


                                                                   
                
                                                                            
                                                                         
                          

      
                                                            
                        

                   
                               
 


















                                                                         
                                                                   
                                             
 
                                                       

                                         
                    



                                                                       


                                                             
 
                        
                                                


                                                             



                                                             
                          

                                       

                                 

                                                                
                                                               
 
                                   
 




                                                                                



                                                         
                                                         


                                                                                                                             
                                                                               
                                              
                                           






                                           
                           
                                  

                                    
                                   
                                    


                                
            
                                               

                                     
 
                                                      
                                           


                              





                                                                       
                                  
















                                                                         
                           
                                 
 
                                                                      
                                                                               





                                                                          

                                                                       
                                                                      

                                     




                                                                   
                                                                            
                      
                              

                                                       



                                                                            
                                                                 






                                                   
                                                        
         



                                                             
 
                                            

                                                      
 
                                                                   
 
                                                     
 







                                                                     
                                                
                                                                
                                                                       

                          
 

                                                        

                                                    






                                                                     




                                                                            
                                  



                                         
                
 
                                                               



                                                                       

                             




                                             
                                                                   
                            

                                
                                                                

                                               
                                        
                                   
                                                    
 


                                                               

                                 





                                                                  




                                                        
                                                         

                                                          

                                                                       

                 
                           






                                                                         

                                                         


                                   
                                                       
                                     
 
                                                    
                                                               




                                                                              
                                                                                          

                            


                                                                                











                                                                       
                                                      


                                                                     

                                                                                   



                                         

                                                                  
         


                                                                  


                                                            
                                                 
 
                             
 


                                                           
 



                                                            


                                                                     




                                                                       
         


                                                                 
                                                                       
 


                                                                             


                                   
                                                           

                                                                   
                                  



                                                












                                                                     
                                                                   




                                                                   

                                                                    
                 



                                                             



                                                                
                                                                     
                                         
                                                              

 
                                              



                                                              
                                               
                                                                 


         
                                                    
                                              
 

                                         
 


                                      

                                                                   
                        
                                                       











                                                                                 
                                                                 

                                                                         

                                                
                                                       
 
                                                         




                                                               
                                                                               

                                                       
         






                                                                     
                                                         
                                         
                                       
                              

                
                                                          
 
                                                           
                                                                
                                          
                                       


                       
                                
                       
                                                                       
                                             
                                                           

                                                                  
                                                                        
                   


                                                                               
 
 
                                                                   




                                                                              
                                                
                                                                               
                   
                                                                   
                                                                       

                                                       
                                                     



                              
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		Implementation of the Transmission Control Protocol(TCP).
 *
 * Authors:	Ross Biro
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Mark Evans, <evansmp@uhura.aston.ac.uk>
 *		Corey Minyard <wf-rch!minyard@relay.EU.net>
 *		Florian La Roche, <flla@stud.uni-sb.de>
 *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
 *		Linus Torvalds, <torvalds@cs.helsinki.fi>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Matthew Dillon, <dillon@apollo.west.oic.com>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Jorge Cwik, <jorge@laser.satlink.net>
 */

/*
 * Changes:	Pedro Roque	:	Retransmit queue handled by TCP.
 *				:	Fragmentation on mtu decrease
 *				:	Segment collapse on retransmit
 *				:	AF independence
 *
 *		Linus Torvalds	:	send_delayed_ack
 *		David S. Miller	:	Charge memory using the right skb
 *					during syn/ack processing.
 *		David S. Miller :	Output engine completely rewritten.
 *		Andrea Arcangeli:	SYNACK carry ts_recent in tsecr.
 *		Cacophonix Gaul :	draft-minshall-nagle-01
 *		J Hadi Salim	:	ECN support
 *
 */

#define pr_fmt(fmt) "TCP: " fmt

#include <net/tcp.h>

#include <linux/compiler.h>
#include <linux/gfp.h>
#include <linux/module.h>
#include <linux/static_key.h>

#include <trace/events/tcp.h>

/* Refresh clocks of a TCP socket,
 * ensuring monotically increasing values.
 */
void tcp_mstamp_refresh(struct tcp_sock *tp)
{
	u64 val = tcp_clock_ns();

	if (val > tp->tcp_clock_cache)
		tp->tcp_clock_cache = val;

	val = div_u64(val, NSEC_PER_USEC);
	if (val > tp->tcp_mstamp)
		tp->tcp_mstamp = val;
}

static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
			   int push_one, gfp_t gfp);

/* Account for new data that has been sent to the network. */
static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
{
	struct inet_connection_sock *icsk = inet_csk(sk);
	struct tcp_sock *tp = tcp_sk(sk);
	unsigned int prior_packets = tp->packets_out;

	tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;

	__skb_unlink(skb, &sk->sk_write_queue);
	tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);

	tp->packets_out += tcp_skb_pcount(skb);
	if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
		tcp_rearm_rto(sk);

	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
		      tcp_skb_pcount(skb));
}

/* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
 * window scaling factor due to loss of precision.
 * If window has been shrunk, what should we make? It is not clear at all.
 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
 * invalid. OK, let's make this for now:
 */
static inline __u32 tcp_acceptable_seq(const struct sock *sk)
{
	const struct tcp_sock *tp = tcp_sk(sk);

	if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
	    (tp->rx_opt.wscale_ok &&
	     ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
		return tp->snd_nxt;
	else
		return tcp_wnd_end(tp);
}

/* Calculate mss to advertise in SYN segment.
 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
 *
 * 1. It is independent of path mtu.
 * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
 *    attached devices, because some buggy hosts are confused by
 *    large MSS.
 * 4. We do not make 3, we advertise MSS, calculated from first
 *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
 *    This may be overridden via information stored in routing table.
 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
 *    probably even Jumbo".
 */
static __u16 tcp_advertise_mss(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	const struct dst_entry *dst = __sk_dst_get(sk);
	int mss = tp->advmss;

	if (dst) {
		unsigned int metric = dst_metric_advmss(dst);

		if (metric < mss) {
			mss = metric;
			tp->advmss = mss;
		}
	}

	return (__u16)mss;
}

/* RFC2861. Reset CWND after idle period longer RTO to "restart window".
 * This is the first part of cwnd validation mechanism.
 */
void tcp_cwnd_restart(struct sock *sk, s32 delta)
{
	struct tcp_sock *tp = tcp_sk(sk);
	u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
	u32 cwnd = tp->snd_cwnd;

	tcp_ca_event(sk, CA_EVENT_CWND_RESTART);

	tp->snd_ssthresh = tcp_current_ssthresh(sk);
	restart_cwnd = min(restart_cwnd, cwnd);

	while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
		cwnd >>= 1;
	tp->snd_cwnd = max(cwnd, restart_cwnd);
	tp->snd_cwnd_stamp = tcp_jiffies32;
	tp->snd_cwnd_used = 0;
}

/* Congestion state accounting after a packet has been sent. */
static void tcp_event_data_sent(struct tcp_sock *tp,
				struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);
	const u32 now = tcp_jiffies32;

	if (tcp_packets_in_flight(tp) == 0)
		tcp_ca_event(sk, CA_EVENT_TX_START);

	/* If this is the first data packet sent in response to the
	 * previous received data,
	 * and it is a reply for ato after last received packet,
	 * increase pingpong count.
	 */
	if (before(tp->lsndtime, icsk->icsk_ack.lrcvtime) &&
	    (u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
		inet_csk_inc_pingpong_cnt(sk);

	tp->lsndtime = now;
}

/* Account for an ACK we sent. */
static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts,
				      u32 rcv_nxt)
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (unlikely(tp->compressed_ack > TCP_FASTRETRANS_THRESH)) {
		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
			      tp->compressed_ack - TCP_FASTRETRANS_THRESH);
		tp->compressed_ack = TCP_FASTRETRANS_THRESH;
		if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
			__sock_put(sk);
	}

	if (unlikely(rcv_nxt != tp->rcv_nxt))
		return;  /* Special ACK sent by DCTCP to reflect ECN */
	tcp_dec_quickack_mode(sk, pkts);
	inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
}

/* Determine a window scaling and initial window to offer.
 * Based on the assumption that the given amount of space
 * will be offered. Store the results in the tp structure.
 * NOTE: for smooth operation initial space offering should
 * be a multiple of mss if possible. We assume here that mss >= 1.
 * This MUST be enforced by all callers.
 */
void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
			       __u32 *rcv_wnd, __u32 *window_clamp,
			       int wscale_ok, __u8 *rcv_wscale,
			       __u32 init_rcv_wnd)
{
	unsigned int space = (__space < 0 ? 0 : __space);

	/* If no clamp set the clamp to the max possible scaled window */
	if (*window_clamp == 0)
		(*window_clamp) = (U16_MAX << TCP_MAX_WSCALE);
	space = min(*window_clamp, space);

	/* Quantize space offering to a multiple of mss if possible. */
	if (space > mss)
		space = rounddown(space, mss);

	/* NOTE: offering an initial window larger than 32767
	 * will break some buggy TCP stacks. If the admin tells us
	 * it is likely we could be speaking with such a buggy stack
	 * we will truncate our initial window offering to 32K-1
	 * unless the remote has sent us a window scaling option,
	 * which we interpret as a sign the remote TCP is not
	 * misinterpreting the window field as a signed quantity.
	 */
	if (sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
		(*rcv_wnd) = min(space, MAX_TCP_WINDOW);
	else
		(*rcv_wnd) = min_t(u32, space, U16_MAX);

	if (init_rcv_wnd)
		*rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);

	*rcv_wscale = 0;
	if (wscale_ok) {
		/* Set window scaling on max possible window */
		space = max_t(u32, space, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
		space = max_t(u32, space, sysctl_rmem_max);
		space = min_t(u32, space, *window_clamp);
		*rcv_wscale = clamp_t(int, ilog2(space) - 15,
				      0, TCP_MAX_WSCALE);
	}
	/* Set the clamp no higher than max representable value */
	(*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp);
}
EXPORT_SYMBOL(tcp_select_initial_window);

/* Chose a new window to advertise, update state in tcp_sock for the
 * socket, and return result with RFC1323 scaling applied.  The return
 * value can be stuffed directly into th->window for an outgoing
 * frame.
 */
static u16 tcp_select_window(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	u32 old_win = tp->rcv_wnd;
	u32 cur_win = tcp_receive_window(tp);
	u32 new_win = __tcp_select_window(sk);

	/* Never shrink the offered window */
	if (new_win < cur_win) {
		/* Danger Will Robinson!
		 * Don't update rcv_wup/rcv_wnd here or else
		 * we will not be able to advertise a zero
		 * window in time.  --DaveM
		 *
		 * Relax Will Robinson.
		 */
		if (new_win == 0)
			NET_INC_STATS(sock_net(sk),
				      LINUX_MIB_TCPWANTZEROWINDOWADV);
		new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
	}
	tp->rcv_wnd = new_win;
	tp->rcv_wup = tp->rcv_nxt;

	/* Make sure we do not exceed the maximum possible
	 * scaled window.
	 */
	if (!tp->rx_opt.rcv_wscale &&
	    sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
		new_win = min(new_win, MAX_TCP_WINDOW);
	else
		new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));

	/* RFC1323 scaling applied */
	new_win >>= tp->rx_opt.rcv_wscale;

	/* If we advertise zero window, disable fast path. */
	if (new_win == 0) {
		tp->pred_flags = 0;
		if (old_win)
			NET_INC_STATS(sock_net(sk),
				      LINUX_MIB_TCPTOZEROWINDOWADV);
	} else if (old_win == 0) {
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
	}

	return new_win;
}

/* Packet ECN state for a SYN-ACK */
static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
{
	const struct tcp_sock *tp = tcp_sk(sk);

	TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
	if (!(tp->ecn_flags & TCP_ECN_OK))
		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
	else if (tcp_ca_needs_ecn(sk) ||
		 tcp_bpf_ca_needs_ecn(sk))
		INET_ECN_xmit(sk);
}

/* Packet ECN state for a SYN.  */
static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
	bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
	bool use_ecn = sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 ||
		tcp_ca_needs_ecn(sk) || bpf_needs_ecn;

	if (!use_ecn) {
		const struct dst_entry *dst = __sk_dst_get(sk);

		if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
			use_ecn = true;
	}

	tp->ecn_flags = 0;

	if (use_ecn)