aboutsummaryrefslogblamecommitdiffstats
path: root/drivers/infiniband/hw/cxgb4/cm.c
blob: cd2ff5f9518a2b7143d53821672a5d98b9773b17 (plain) (tree)
1
2
  
                                                             





































                                                                      
                          



                          
                    

                          
 

                         
                     
                     
















                         







                                                                   
                         
                                   
                                                            
 
                              
                                             

                                                            
 
















                                                                             
                         
                                   
                                                                  










                                                                      
                       

                                                               
                                                                               
                                          












                                                                              
                                
                                 
                                                                      
 
                                      

                               




                                                                        


                               











                                                 



                                              





                                                           





                                                           
                                            
 
                                                  
                                   
                                                         
                                      


                         














                                                                          
                                     













                                                                          
                                     


















                                                                               
                                                                   


                                                                    
                           
                                 
                                                           

                               

                                                                       

                                                                           




                                                                

                                 
                                
                           
                                  









                                                                           
                                

                                                                           
                                  









                                          
                                        
                                                 










                                                                                

                                                    
                                                          









                                                                              
                                                                               



                                                                            





                                                                         








                                                    































                                                                       
                                                


                   

                                                                     
                                                                 










































                                                                             



                                                                   
                          
                            
 
                                                                            

                                                                      
                       
                            


                                                 

                                              
                                 




                                      












                                                                       

                                    
                                                            
                       

                                                





                                                                             




                                                                     
























                                                                              



                                                                              

                                                          
                                                             
                                                                         










                                                          
                                                         


























































                                                                               































                                                                     
                                                                               


                

                                                                            
 


                                                                             
                
                                                                             



                   
                                                                    
                                                             
 


                                                           
                                                         

                                                                       




                                                                  

                                           





                                                   




                             
                                       







                                                               
                




















                                                                          



                                                            








                                                                  
                                                           
 
                                                                

                                                                     
                                         





                                                                

                                   
 
                                         
                             
                         




                                         
                                


                                            
                                        
                                                            
                                  
                                     
                            
                                  
                                                
                                       



                                                              
                                         
                                                     
                                 
         




                                                                        
                                                                  
 



                                                                             
                                           





























                                                                             


                                                                                

                                                      










                                                                                
                                                                               
                                            































                                                                               


                                                                                
                                                       
                        

                                                                    
                                                                                
                                                           


                                                                        
                 

         
                                                
                                                              
             



                                                                            

 

                                                                 



                                       
                                                





                                                                               

                                                            










                                                               


                                                
                                        

                                           

                                           

                                            



                                                        

                                                                        
                                                 
                                       
                                  
                                          

                                          

                                  

                                                                                   

                                                                       













                                                                           
 







                                                                              











                                                               
                                            
                                   
                              








                                                                          
                                                



                                                                               

                                                                         











                                                                          


                                                
                                        

                                           

                                           

                                            




                                                        
                                             
                                             


                                                                           

                                                                                   

















                                                                               










                                                                   
                              








                                                                         
                                                



                                                                               

                                                                         











                                                                          


                                                
                                        

                                           

                                           

                                            





                                                               
                                             
                                             


                                                                           

                                                                                   






















                                                                               








                                                               
                                            
                              







                                                                   
                                                            






                                                                          
                                   




                                                 
                                                           






                                                
                                                                 
                                 
                                             


                                   



                                               
                                     


                 
                                                                 





                                                           
                              





                                                                        
                                                        





                                                                               
                                        
                                              













                                                                    
                                                          
















                                                                    
                                                        










                                                                             



                                                        

                                                         

                                                       

                                            






                                                                  

                                                                    



                                                           
         


                                                         
                                                   

                                                            


                                                      


         
                                                     

                                 
                



                                                           



                                                        
                                 









                                                                               

                                                            


                                                                              




                                                                               
                                                  
                                         
                   








                                                           

                            


                                                                    
                                                        















                                                                               




                                                                           

                                                             
 




                                                                    
                                                                 

                                                                  
                                                         



                                                

                                 
                                                                     

                                
                                                 
                 

                                            


                                        
                           



                                                           


                                                      
           
                              
                         




















                                                                       
                         


                                                 


                                                                             
































                                                              
                         










                                                            
                                         


                                                                                












                                                                     

                                                                             





                                                                            















                                                                          


                                                   

















                                                                              
                                                            











                                                                               
                                                       



                                                                    















                                                                    









                                                                              
                                    
                                                                 
                                                                    
                              
                               














                                                                         
                                    
                                                                 
                                                                    
                              
                               

                         

                 
                                        
                                        

                                      
                          




                                                                        
                                                 



                                                           




                                                                   
                                        




















                                                                       




                                                 


                                                                             
                                        




                                                              
                                        









                                                      
                                        







                                                      
                                        


















                                                                                














                                                                     

                                                                               















                                                                               





                                                                          








                                                                

                                                            



                                                                      
                                                              

                                                        
         









                                                             
                                  
                           

                                

                         


                                                                         
                                   
 


                                    
                                
                          
                                    
                                                        

                          
                                    

                                             


                                                
                           
                                                                

                                                                            
                                                                 
                                                           
                                                           

                                                                   

                      


                      
                                     

                                                      






                                                               




                                                 



                                                                         
                                                           
                                   

                                
                                                            







                                                      
                                     





                                         





                                                                       
                                
                



                                                                           
                                                              
                                                                              

                                                                     
                                               
                                                               

                                                  
                                                                

                                                  
                                   

                                                                             
                                                
                                                                    
                                                   
                                    
                                                                

                                                                     
                                         





                                                                

                                   
 

                                                         
                              
                          




                                          
                                 

                                             

                                                                         
                                 
                                         
                                                              
                                  
                                                              
                            
                                                           
                                                


                                                                

                                                             


                                                        








                                                                             








                                                        













                                             






                                                                               

                              

                                                                   
                                                                    

































                                                                             

                                                                       







                                                          
                                                                          







                                                            
                                                    

                                                                       

                                              

                                                          


                                                      
                                                     
                                                                          














                                                  


                                             



                                                                





                                                                  















                                                                         














                                                                              



                                                              

                                                                    
                  
                                                             

































                                                                            



                                                                  

                                                                
                                                 
                                                             



                                 

                                  



                                                                 



                                                                         
                                 





                                                                           


                         

                                                






                                    
                               
                                                  
                                            


                                                              
                                                
                                                                    






                                                                        








                                                                                







                                                                          
                      
                










                                                                                                      


                      


                                                       





                                                                             


                                                                               
                                                                 















                                                                   

                                                                     


                                                           
                                                                  
 
    


                 







                                                                       
                                                                  


                 
                                                              






                                                      
                                                   
                
                                                                          


                                                           
 
                     
                          
                                   










                                                                        

                                                                     
                                         





                                                                

                                   
                                         
                             
                         




                                         
                                     



                                                            

                                                        
                                     
                                                
                                  
                                                
                                       



                                               





                                                                               
                                           
                                               
         
                                                              
                                                     
                                         
                                                     
                                 





                                                                               
         
 

                                      
                                                           
                                                               




                                                        
                                                                           
 
                                                              

                                                      



                                            

                                                                            

                                                             





                                                                  
                                                                       
                                                                            


                                                                  














                                                                                







                                                                     
                                                    
                                                      
                                                                  


                                                 
                                       
                                     
                                  
                
                                              
                   
                            

                                         



                                                                                
 





                                                                    

                                                                            
 
                               






                                                                                                     
                                                                        





                                                                                                     
                                                                        



                                                                              



                                                                       



                                                                          




                                 

                                                                    

                                                                           
                                 
                                 
                                

                            
 



                                                                        
 


                                              









                                                                         

                                                                

                                                         


                                                           







                                                                              



                                                          
                                                                               


                                                              







                                                                                



                                                             

                                                                             
                                             
 

                                        
                                                              

                                

                                                                
                                                                       


                                             
                                                                       
                                      
                                                         




                                                                               

                 
                                   















                                                                    


                                                                     





                                          
                                              








                                                                

                           

                                                 
                




                                                           
                                              
                                   
















                                                                       
                                                                
                                                            


                                               
                                                                
                                                            

                       
                                   
                                               
                                                         
                                                                 
                                                                           



                                              








                                                
                                        




                                                                           
                                             









                                            
                                     






                                                      








                                                                

                                                 

                                
                                      






                                                                            

                         

                                                                   
                                              


                                                             

                                                           
           

                                                            

                                   



                                
                                        

                          
                                        
                                                                            













                                                                               

                          

                          


                      
                                  

















                                                                         
                                             







                                            


                                                                      
         
                                     














                                                                               

                                         
                                         








                                                                              
                                                                               





                                                                            







                                                                   


                                                 






                                                           
                                   




                                                
                                        






                                                                     
                                             









                                            
                                     






                                                               
                                                     
                                                 


                                        

                                
                    
 
                               





                                                                              
                                                                                
 



















                                                                               

                                                                    













                                                                           

                                                                           

                           


                                                           


                                             


                                      
                                              
                                              



                                                            
                               
         


                                                            













                                                                               


                                    



                                  
                                              

                    
                                              

                                                                  




                                                       

                                                                           










                                                                              
                 




                                                                    




                                                                       
         


                                  
                                        

                                              


                                                                             
                                                                              

                                    


                                                                            


                              
                   
 






















                                                             
                                         
                               
                                     



                              
                                               

                              
                                     



                              




















































                                                                              
                                                                           






                                                          

                                                                             

                                                          
                    





                                        

                   
                         
 

                                                          


                              




















                                                                         


                                                                            
                           
         
                   









                                                                           
                           
         
                                                         
 







































                                                                              


                                                      
 





                                                                        
                                           


















                                                                             
                                           












                                                                              

                                                                           
                           
         
 

                                                                   
                  
                                                                          
                           







                                                                              






                                          
      

                             
                                                                     
                                                               
      





                              


                                                                          

                                                               
 





                                                                               








                                                                      

                                          


                                                                             


                                                                                                 
         





                                                                          

                                                             







                                                                          



                                                                           












                                                                              

                                                  







                                                                                               




                                                           


                                        
 












                                                                         

                                                       



                                 

                                                    

                                                                              
            

                                                                             
 
                             
                                                                           


                              
                                                         






























                                                                            
                                    



                                                    



                                          

      

                                                              
















                                                        

                                                         



                                                                  
                                          
                                                    


                                                                          



                                                                               


                                                                             
         
                                                                     

                                                              
     







                                                                 


                               
 
                                   






                                                            
                                                












                                                 
                                           






                                                                    
                                                        















                                                               
                    
                             
                                                                 
                                                               
                                                        

                                                                 
                                                      
                 


                                  
                                     




                                         



                                                                             
                                         
 

                                                                   




                              




                                                               
                           




                                                               





                                                                 




                                                             
                                                            
                                  





                                                                             




                                                   








                                                                              
                                                               


                                                                            


                                                      



                                                                          

                                                                  







                                                                      

                                              


























                                                                             



                                     



                                             
                             
                            
 
                                                                 
                                                                 



                               









                                                                  
                                                  


                                                                          

                                                    
                                                
                                           



















                                                                           

                                                           


















                                                                            
                



                                                                               
                                                                              
                                                                              
                                                                 
                                                 






                                              


                                                                            
                                                                




                                                                      
                                                    






                                                        
                                                    
                                     

                                                           






                                                                              





















                                                                     



                                
                                 




                                      
                                                    











                                                                          
                                                                      






                                                                                















                                                                           
                                      



















                                                                         


                                                                               



                                                          

                                               





                                                          







                                                            
                                                
                                                            


                                                           
         
                             







                                                              
                                                                

                                                    


                                                                            













                                                                             


                 
  
















                                                        
                                

                                         






                                               
                                   

                                                                   
                                            
















                                                                         
                                                      
                      









                                                                            
                
                                                                      
                                                                

                          

                                                       
                                     












                                            

                                 











                                                            
                                      


                            
                               








                                                                         
                                       
         






                                                   
                       

                                 
                                                         






                                                                 
         
                                   

                                             


  

























                                                                        
                       


                 








                                                             
                             
                                                                     
                                                                                       
                                                                         

                                                               
                               
                      
                          
                                             
                                
                      


                                                                                
                               




                      







                                                                     





                                                                        
                                      




                                                                            







                                                                   

                                                                   
           




                                                                    



                        















                                                 
                                              


                                        

                                

  

                             
                                      





                                                          


                 
                       
 
                                            


                                 
/*
 * Copyright (c) 2009-2014 Chelsio, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *	  copyright notice, this list of conditions and the following
 *	  disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *	  copyright notice, this list of conditions and the following
 *	  disclaimer in the documentation and/or other materials
 *	  provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#include <linux/module.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/skbuff.h>
#include <linux/timer.h>
#include <linux/notifier.h>
#include <linux/inetdevice.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/if_vlan.h>

#include <net/neighbour.h>
#include <net/netevent.h>
#include <net/route.h>
#include <net/tcp.h>
#include <net/ip6_route.h>
#include <net/addrconf.h>

#include <rdma/ib_addr.h>

#include "iw_cxgb4.h"
#include "clip_tbl.h"

static char *states[] = {
	"idle",
	"listen",
	"connecting",
	"mpa_wait_req",
	"mpa_req_sent",
	"mpa_req_rcvd",
	"mpa_rep_sent",
	"fpdu_mode",
	"aborting",
	"closing",
	"moribund",
	"dead",
	NULL,
};

static int nocong;
module_param(nocong, int, 0644);
MODULE_PARM_DESC(nocong, "Turn of congestion control (default=0)");

static int enable_ecn;
module_param(enable_ecn, int, 0644);
MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)");

static int dack_mode = 1;
module_param(dack_mode, int, 0644);
MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)");

uint c4iw_max_read_depth = 32;
module_param(c4iw_max_read_depth, int, 0644);
MODULE_PARM_DESC(c4iw_max_read_depth,
		 "Per-connection max ORD/IRD (default=32)");

static int enable_tcp_timestamps;
module_param(enable_tcp_timestamps, int, 0644);
MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)");

static int enable_tcp_sack;
module_param(enable_tcp_sack, int, 0644);
MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)");

static int enable_tcp_window_scaling = 1;
module_param(enable_tcp_window_scaling, int, 0644);
MODULE_PARM_DESC(enable_tcp_window_scaling,
		 "Enable tcp window scaling (default=1)");

int c4iw_debug;
module_param(c4iw_debug, int, 0644);
MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)");

static int peer2peer = 1;
module_param(peer2peer, int, 0644);
MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=1)");

static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ;
module_param(p2p_type, int, 0644);
MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: "
			   "1=RDMA_READ 0=RDMA_WRITE (default 1)");

static int ep_timeout_secs = 60;
module_param(ep_timeout_secs, int, 0644);
MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "
				   "in seconds (default=60)");

static int mpa_rev = 2;
module_param(mpa_rev, int, 0644);
MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, "
		"1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft"
		" compliant (default=2)");

static int markers_enabled;
module_param(markers_enabled, int, 0644);
MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)");

static int crc_enabled = 1;
module_param(crc_enabled, int, 0644);
MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)");

static int rcv_win = 256 * 1024;
module_param(rcv_win, int, 0644);
MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)");

static int snd_win = 128 * 1024;
module_param(snd_win, int, 0644);
MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)");

static struct workqueue_struct *workq;

static struct sk_buff_head rxq;

static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp);
static void ep_timeout(unsigned long arg);
static void connect_reply_upcall(struct c4iw_ep *ep, int status);

static LIST_HEAD(timeout_list);
static spinlock_t timeout_lock;

static void deref_qp(struct c4iw_ep *ep)
{
	c4iw_qp_rem_ref(&ep->com.qp->ibqp);
	clear_bit(QP_REFERENCED, &ep->com.flags);
}

static void ref_qp(struct c4iw_ep *ep)
{
	set_bit(QP_REFERENCED, &ep->com.flags);
	c4iw_qp_add_ref(&ep->com.qp->ibqp);
}

static void start_ep_timer(struct c4iw_ep *ep)
{
	PDBG("%s ep %p\n", __func__, ep);
	if (timer_pending(&ep->timer)) {
		pr_err("%s timer already started! ep %p\n",
		       __func__, ep);
		return;
	}
	clear_bit(TIMEOUT, &ep->com.flags);
	c4iw_get_ep(&ep->com);
	ep->timer.expires = jiffies + ep_timeout_secs * HZ;
	ep->timer.data = (unsigned long)ep;
	ep->timer.function = ep_timeout;
	add_timer(&ep->timer);
}

static int stop_ep_timer(struct c4iw_ep *ep)
{
	PDBG("%s ep %p stopping\n", __func__, ep);
	del_timer_sync(&ep->timer);
	if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) {
		c4iw_put_ep(&ep->com);
		return 0;
	}
	return 1;
}

static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb,
		  struct l2t_entry *l2e)
{
	int	error = 0;

	if (c4iw_fatal_error(rdev)) {
		kfree_skb(skb);
		PDBG("%s - device in error state - dropping\n", __func__);
		return -EIO;
	}
	error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e);
	if (error < 0)
		kfree_skb(skb);
	return error < 0 ? error : 0;
}

int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb)
{
	int	error = 0;

	if (c4iw_fatal_error(rdev)) {
		kfree_skb(skb);
		PDBG("%s - device in error state - dropping\n", __func__);
		return -EIO;
	}
	error = cxgb4_ofld_send(rdev->lldi.ports[0], skb);
	if (error < 0)
		kfree_skb(skb);
	return error < 0 ? error : 0;
}

static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb)
{
	struct cpl_tid_release *req;

	skb = get_skb(skb, sizeof *req, GFP_KERNEL);
	if (!skb)
		return;
	req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req));
	INIT_TP_WR(req, hwtid);
	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));
	set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
	c4iw_ofld_send(rdev, skb);
	return;
}

static void set_emss(struct c4iw_ep *ep, u16 opt)
{
	ep->emss = ep->com.dev->rdev.lldi.mtus[TCPOPT_MSS_G(opt)] -
		   ((AF_INET == ep->com.remote_addr.ss_family) ?
		    sizeof(struct iphdr) : sizeof(struct ipv6hdr)) -
		   sizeof(struct tcphdr);
	ep->mss = ep->emss;
	if (TCPOPT_TSTAMP_G(opt))
		ep->emss -= round_up(TCPOLEN_TIMESTAMP, 4);
	if (ep->emss < 128)
		ep->emss = 128;
	if (ep->emss & 7)
		PDBG("Warning: misaligned mtu idx %u mss %u emss=%u\n",
		     TCPOPT_MSS_G(opt), ep->mss, ep->emss);
	PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, TCPOPT_MSS_G(opt),
	     ep->mss, ep->emss);
}

static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc)
{
	enum c4iw_ep_state state;

	mutex_lock(&epc->mutex);
	state = epc->state;
	mutex_unlock(&epc->mutex);
	return state;
}

static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new)
{
	epc->state = new;
}

static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new)
{
	mutex_lock(&epc->mutex);
	PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]);
	__state_set(epc, new);
	mutex_unlock(&epc->mutex);
	return;
}

static void *alloc_ep(int size, gfp_t gfp)
{
	struct c4iw_ep_common *epc;

	epc = kzalloc(size, gfp);
	if (epc) {
		kref_init(&epc->kref);
		mutex_init(&epc->mutex);
		c4iw_init_wr_wait(&epc->wr_wait);
	}
	PDBG("%s alloc ep %p\n", __func__, epc);
	return epc;
}

void _c4iw_free_ep(struct kref *kref)
{
	struct c4iw_ep *ep;

	ep = container_of(kref, struct c4iw_ep, com.kref);
	PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]);
	if (test_bit(QP_REFERENCED, &ep->com.flags))
		deref_qp(ep);
	if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) {
		if (ep->com.remote_addr.ss_family == AF_INET6) {
			struct sockaddr_in6 *sin6 =
					(struct sockaddr_in6 *)
					&ep->com.mapped_local_addr;

			cxgb4_clip_release(
					ep->com.dev->rdev.lldi.ports[0],
					(const u32 *)&sin6->sin6_addr.s6_addr,
					1);
		}
		remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid);
		cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid);
		dst_release(ep->dst);
		cxgb4_l2t_release(ep->l2t);
	}
	if (test_bit(RELEASE_MAPINFO, &ep->com.flags)) {
		print_addr(&ep->com, __func__, "remove_mapinfo/mapping");
		iwpm_remove_mapinfo(&ep->com.local_addr,
				    &ep->com.mapped_local_addr);
		iwpm_remove_mapping(&ep->com.local_addr, RDMA_NL_C4IW);
	}
	kfree(ep);
}

static void release_ep_resources(struct c4iw_ep *ep)
{
	set_bit(RELEASE_RESOURCES, &ep->com.flags);
	c4iw_put_ep(&ep->com);
}

static int status2errno(int status)
{
	switch (status) {
	case CPL_ERR_NONE:
		return 0;
	case CPL_ERR_CONN_RESET:
		return -ECONNRESET;
	case CPL_ERR_ARP_MISS:
		return -EHOSTUNREACH;
	case CPL_ERR_CONN_TIMEDOUT:
		return -ETIMEDOUT;
	case CPL_ERR_TCAM_FULL:
		return -ENOMEM;
	case CPL_ERR_CONN_EXIST:
		return -EADDRINUSE;
	default:
		return -EIO;
	}
}

/*
 * Try and reuse skbs already allocated...
 */
static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)
{
	if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) {
		skb_trim(skb, 0);
		skb_get(skb);
		skb_reset_transport_header(skb);
	} else {
		skb = alloc_skb(len, gfp);
	}
	t4_set_arp_err_handler(skb, NULL, NULL);
	return skb;
}

static struct net_device *get_real_dev(struct net_device *egress_dev)
{
	return rdma_vlan_dev_real_dev(egress_dev) ? : egress_dev;
}

static int our_interface(struct c4iw_dev *dev, struct net_device *egress_dev)
{
	int i;

	egress_dev = get_real_dev(egress_dev);
	for (i = 0; i < dev->rdev.lldi.nports; i++)
		if (dev->rdev.lldi.ports[i] == egress_dev)
			return 1;
	return 0;
}

static struct dst_entry *find_route6(struct c4iw_dev *dev, __u8 *local_ip,
				     __u8 *peer_ip, __be16 local_port,
				     __be16 peer_port, u8 tos,
				     __u32 sin6_scope_id)
{
	struct dst_entry *dst = NULL;

	if (IS_ENABLED(CONFIG_IPV6)) {
		struct flowi6 fl6;

		memset(&fl6, 0, sizeof(fl6));
		memcpy(&fl6.daddr, peer_ip, 16);
		memcpy(&fl6.saddr, local_ip, 16);
		if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
			fl6.flowi6_oif = sin6_scope_id;
		dst = ip6_route_output(&init_net, NULL, &fl6);
		if (!dst)
			goto out;
		if (!our_interface(dev, ip6_dst_idev(dst)->dev) &&
		    !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) {
			dst_release(dst);
			dst = NULL;
		}
	}

out:
	return dst;
}

static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip,
				 __be32 peer_ip, __be16 local_port,
				 __be16 peer_port, u8 tos)
{
	struct rtable *rt;
	struct flowi4 fl4;
	struct neighbour *n;

	rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
				   peer_port, local_port, IPPROTO_TCP,
				   tos, 0);
	if (IS_ERR(rt))
		return NULL;
	n = dst_neigh_lookup(&rt->dst, &peer_ip);
	if (!n)
		return NULL;
	if (!our_interface(dev, n->dev) &&
	    !(n->dev->flags & IFF_LOOPBACK)) {
		neigh_release(n);
		dst_release(&rt->dst);
		return NULL;
	}
	neigh_release(n);
	return &rt->dst;
}

static void arp_failure_discard(void *handle, struct sk_buff *skb)
{
	PDBG("%s c4iw_dev %p\n", __func__, handle);
	kfree_skb(skb);
}

/*
 * Handle an ARP failure for an active open.
 */
static void act_open_req_arp_failure(void *handle, struct sk_buff *skb)
{
	struct c4iw_ep *ep = handle;

	printk(KERN_ERR MOD "ARP failure during connect\n");
	kfree_skb(skb);
	connect_reply_upcall(ep, -EHOSTUNREACH);
	state_set(&ep->com, DEAD);
	if (ep->com.remote_addr.ss_family == AF_INET6) {
		struct sockaddr_in6 *sin6 =
			(struct sockaddr_in6 *)&ep->com.mapped_local_addr;
		cxgb4_clip_release(ep->com.dev->rdev.lldi.ports[0],
				   (const u32 *)&sin6->sin6_addr.s6_addr, 1);
	}
	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid);
	cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid);
	dst_release(ep->dst);
	cxgb4_l2t_release(ep->l2t);
	c4iw_put_ep(&ep->com);
}

/*
 * Handle an ARP failure for a CPL_ABORT_REQ.  Change it into a no RST variant
 * and send it along.
 */
static void abort_arp_failure(void *handle, struct sk_buff *skb)
{
	struct c4iw_rdev *rdev = handle;
	struct cpl_abort_req *req = cplhdr(skb);

	PDBG("%s rdev %p\n", __func__, rdev);
	req->cmd = CPL_ABORT_NO_RST;
	c4iw_ofld_send(rdev, skb);
}

static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb)
{
	unsigned int flowclen = 80;
	struct fw_flowc_wr *flowc;
	int i;

	skb = get_skb(skb, flowclen, GFP_KERNEL);
	flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen);

	flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |
					   FW_FLOWC_WR_NPARAMS_V(8));
	flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(flowclen,
					  16)) | FW_WR_FLOWID_V(ep->hwtid));

	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
	flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V
					    (ep->com.dev->rdev.lldi.pf));
	flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
	flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan);
	flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
	flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan);
	flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
	flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid);
	flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
	flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq);
	flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
	flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq);
	flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
	flowc->mnemval[6].val = cpu_to_be32(ep->snd_win);
	flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
	flowc->mnemval[7].val = cpu_to_be32(ep->emss);
	/* Pad WR to 16 byte boundary */
	flowc->mnemval[8].mnemonic = 0;
	flowc->mnemval[8].val = 0;
	for (i = 0; i < 9; i++) {
		flowc->mnemval[i].r4[0] = 0;
		flowc->mnemval[i].r4[1] = 0;
		flowc->mnemval[i].r4[2] = 0;
	}

	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
	c4iw_ofld_send(&ep->com.dev->rdev, skb);
}

static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp)
{
	struct cpl_close_con_req *req;
	struct sk_buff *skb;
	int wrlen = roundup(sizeof *req, 16);

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	skb = get_skb(NULL, wrlen, gfp);
	if (!skb) {
		printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
		return -ENOMEM;
	}
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
	t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
	req = (struct cpl_close_con_req *) skb_put(skb, wrlen);
	memset(req, 0, wrlen);
	INIT_TP_WR(req, ep->hwtid);
	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ,
						    ep->hwtid));
	return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
}

static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp)
{
	struct cpl_abort_req *req;
	int wrlen = roundup(sizeof *req, 16);

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	skb = get_skb(skb, wrlen, gfp);
	if (!skb) {
		printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
		       __func__);
		return -ENOMEM;
	}
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
	t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure);
	req = (struct cpl_abort_req *) skb_put(skb, wrlen);
	memset(req, 0, wrlen);
	INIT_TP_WR(req, ep->hwtid);
	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid));
	req->cmd = CPL_ABORT_SEND_RST;
	return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
}

/*
 * c4iw_form_pm_msg - Form a port mapper message with mapping info
 */
static void c4iw_form_pm_msg(struct c4iw_ep *ep,
				struct iwpm_sa_data *pm_msg)
{
	memcpy(&pm_msg->loc_addr, &ep->com.local_addr,
		sizeof(ep->com.local_addr));
	memcpy(&pm_msg->rem_addr, &ep->com.remote_addr,
		sizeof(ep->com.remote_addr));
}

/*
 * c4iw_form_reg_msg - Form a port mapper message with dev info
 */
static void c4iw_form_reg_msg(struct c4iw_dev *dev,
				struct iwpm_dev_data *pm_msg)
{
	memcpy(pm_msg->dev_name, dev->ibdev.name, IWPM_DEVNAME_SIZE);
	memcpy(pm_msg->if_name, dev->rdev.lldi.ports[0]->name,
				IWPM_IFNAME_SIZE);
}

static void c4iw_record_pm_msg(struct c4iw_ep *ep,
			struct iwpm_sa_data *pm_msg)
{
	memcpy(&ep->com.mapped_local_addr, &pm_msg->mapped_loc_addr,
		sizeof(ep->com.mapped_local_addr));
	memcpy(&ep->com.mapped_remote_addr, &pm_msg->mapped_rem_addr,
		sizeof(ep->com.mapped_remote_addr));
}

static int get_remote_addr(struct c4iw_ep *parent_ep, struct c4iw_ep *child_ep)
{
	int ret;

	print_addr(&parent_ep->com, __func__, "get_remote_addr parent_ep ");
	print_addr(&child_ep->com, __func__, "get_remote_addr child_ep ");

	ret = iwpm_get_remote_info(&parent_ep->com.mapped_local_addr,
				   &child_ep->com.mapped_remote_addr,
				   &child_ep->com.remote_addr, RDMA_NL_C4IW);
	if (ret)
		PDBG("Unable to find remote peer addr info - err %d\n", ret);

	return ret;
}

static void best_mtu(const unsigned short *mtus, unsigned short mtu,
		     unsigned int *idx, int use_ts, int ipv6)
{
	unsigned short hdr_size = (ipv6 ?
				   sizeof(struct ipv6hdr) :
				   sizeof(struct iphdr)) +
				  sizeof(struct tcphdr) +
				  (use_ts ?
				   round_up(TCPOLEN_TIMESTAMP, 4) : 0);
	unsigned short data_size = mtu - hdr_size;

	cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx);
}

static int send_connect(struct c4iw_ep *ep)
{
	struct cpl_act_open_req *req = NULL;
	struct cpl_t5_act_open_req *t5req = NULL;
	struct cpl_t6_act_open_req *t6req = NULL;
	struct cpl_act_open_req6 *req6 = NULL;
	struct cpl_t5_act_open_req6 *t5req6 = NULL;
	struct cpl_t6_act_open_req6 *t6req6 = NULL;
	struct sk_buff *skb;
	u64 opt0;
	u32 opt2;
	unsigned int mtu_idx;
	int wscale;
	int win, sizev4, sizev6, wrlen;
	struct sockaddr_in *la = (struct sockaddr_in *)
				 &ep->com.mapped_local_addr;
	struct sockaddr_in *ra = (struct sockaddr_in *)
				 &ep->com.mapped_remote_addr;
	struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)
				   &ep->com.mapped_local_addr;
	struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)
				   &ep->com.mapped_remote_addr;
	int ret;
	enum chip_type adapter_type = ep->com.dev->rdev.lldi.adapter_type;
	u32 isn = (prandom_u32() & ~7UL) - 1;

	switch (CHELSIO_CHIP_VERSION(adapter_type)) {
	case CHELSIO_T4:
		sizev4 = sizeof(struct cpl_act_open_req);
		sizev6 = sizeof(struct cpl_act_open_req6);
		break;
	case CHELSIO_T5:
		sizev4 = sizeof(struct cpl_t5_act_open_req);
		sizev6 = sizeof(struct cpl_t5_act_open_req6);
		break;
	case CHELSIO_T6:
		sizev4 = sizeof(struct cpl_t6_act_open_req);
		sizev6 = sizeof(struct cpl_t6_act_open_req6);
		break;
	default:
		pr_err("T%d Chip is not supported\n",
		       CHELSIO_CHIP_VERSION(adapter_type));
		return -EINVAL;
	}

	wrlen = (ep->com.remote_addr.ss_family == AF_INET) ?
			roundup(sizev4, 16) :
			roundup(sizev6, 16);

	PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid);

	skb = get_skb(NULL, wrlen, GFP_KERNEL);
	if (!skb) {
		printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
		       __func__);
		return -ENOMEM;
	}
	set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx);

	best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
		 enable_tcp_timestamps,
		 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
	wscale = compute_wscale(rcv_win);

	/*
	 * Specify the largest window that will fit in opt0. The
	 * remainder will be specified in the rx_data_ack.
	 */
	win = ep->rcv_win >> 10;
	if (win > RCV_BUFSIZ_M)
		win = RCV_BUFSIZ_M;

	opt0 = (nocong ? NO_CONG_F : 0) |
	       KEEP_ALIVE_F |
	       DELACK_F |
	       WND_SCALE_V(wscale) |
	       MSS_IDX_V(mtu_idx) |
	       L2T_IDX_V(ep->l2t->idx) |
	       TX_CHAN_V(ep->tx_chan) |
	       SMAC_SEL_V(ep->smac_idx) |
	       DSCP_V(ep->tos) |
	       ULP_MODE_V(ULP_MODE_TCPDDP) |
	       RCV_BUFSIZ_V(win);
	opt2 = RX_CHANNEL_V(0) |
	       CCTRL_ECN_V(enable_ecn) |
	       RSS_QUEUE_VALID_F | RSS_QUEUE_V(ep->rss_qid);
	if (enable_tcp_timestamps)
		opt2 |= TSTAMPS_EN_F;
	if (enable_tcp_sack)
		opt2 |= SACK_EN_F;
	if (wscale && enable_tcp_window_scaling)
		opt2 |= WND_SCALE_EN_F;
	if (CHELSIO_CHIP_VERSION(adapter_type) > CHELSIO_T4) {
		if (peer2peer)
			isn += 4;

		opt2 |= T5_OPT_2_VALID_F;
		opt2 |= CONG_CNTRL_V(CONG_ALG_TAHOE);
		opt2 |= T5_ISS_F;
	}

	if (ep->com.remote_addr.ss_family == AF_INET6)
		cxgb4_clip_get(ep->com.dev->rdev.lldi.ports[0],
			       (const u32 *)&la6->sin6_addr.s6_addr, 1);

	t4_set_arp_err_handler(skb, ep, act_open_req_arp_failure);

	if (ep->com.remote_addr.ss_family == AF_INET) {
		switch (CHELSIO_CHIP_VERSION(adapter_type)) {
		case CHELSIO_T4:
			req = (struct cpl_act_open_req *)skb_put(skb, wrlen);
			INIT_TP_WR(req, 0);
			break;
		case CHELSIO_T5:
			t5req = (struct cpl_t5_act_open_req *)skb_put(skb,
					wrlen);
			INIT_TP_WR(t5req, 0);
			req = (struct cpl_act_open_req *)t5req;
			break;
		case CHELSIO_T6:
			t6req = (struct cpl_t6_act_open_req *)skb_put(skb,
					wrlen);
			INIT_TP_WR(t6req, 0);
			req = (struct cpl_act_open_req *)t6req;
			t5req = (struct cpl_t5_act_open_req *)t6req;
			break;
		default:
			pr_err("T%d Chip is not supported\n",
			       CHELSIO_CHIP_VERSION(adapter_type));
			ret = -EINVAL;
			goto clip_release;
		}

		OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
					((ep->rss_qid<<14) | ep->atid)));
		req->local_port = la->sin_port;
		req->peer_port = ra->sin_port;
		req->local_ip = la->sin_addr.s_addr;
		req->peer_ip = ra->sin_addr.s_addr;
		req->opt0 = cpu_to_be64(opt0);

		if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) {
			req->params = cpu_to_be32(cxgb4_select_ntuple(
						ep->com.dev->rdev.lldi.ports[0],
						ep->l2t));
			req->opt2 = cpu_to_be32(opt2);
		} else {
			t5req->params = cpu_to_be64(FILTER_TUPLE_V(
						cxgb4_select_ntuple(
						ep->com.dev->rdev.lldi.ports[0],
						ep->l2t)));
			t5req->rsvd = cpu_to_be32(isn);
			PDBG("%s snd_isn %u\n", __func__, t5req->rsvd);
			t5req->opt2 = cpu_to_be32(opt2);
		}
	} else {
		switch (CHELSIO_CHIP_VERSION(adapter_type)) {
		case CHELSIO_T4:
			req6 = (struct cpl_act_open_req6 *)skb_put(skb, wrlen);
			INIT_TP_WR(req6, 0);
			break;
		case CHELSIO_T5:
			t5req6 = (struct cpl_t5_act_open_req6 *)skb_put(skb,
					wrlen);
			INIT_TP_WR(t5req6, 0);
			req6 = (struct cpl_act_open_req6 *)t5req6;
			break;
		case CHELSIO_T6:
			t6req6 = (struct cpl_t6_act_open_req6 *)skb_put(skb,
					wrlen);
			INIT_TP_WR(t6req6, 0);
			req6 = (struct cpl_act_open_req6 *)t6req6;
			t5req6 = (struct cpl_t5_act_open_req6 *)t6req6;
			break;
		default:
			pr_err("T%d Chip is not supported\n",
			       CHELSIO_CHIP_VERSION(adapter_type));
			ret = -EINVAL;
			goto clip_release;
		}

		OPCODE_TID(req6) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
					((ep->rss_qid<<14)|ep->atid)));
		req6->local_port = la6->sin6_port;
		req6->peer_port = ra6->sin6_port;
		req6->local_ip_hi = *((__be64 *)(la6->sin6_addr.s6_addr));
		req6->local_ip_lo = *((__be64 *)(la6->sin6_addr.s6_addr + 8));
		req6->peer_ip_hi = *((__be64 *)(ra6->sin6_addr.s6_addr));
		req6->peer_ip_lo = *((__be64 *)(ra6->sin6_addr.s6_addr + 8));
		req6->opt0 = cpu_to_be64(opt0);

		if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) {
			req6->params = cpu_to_be32(cxgb4_select_ntuple(
						ep->com.dev->rdev.lldi.ports[0],
						ep->l2t));
			req6->opt2 = cpu_to_be32(opt2);
		} else {
			t5req6->params = cpu_to_be64(FILTER_TUPLE_V(
						cxgb4_select_ntuple(
						ep->com.dev->rdev.lldi.ports[0],
						ep->l2t)));
			t5req6->rsvd = cpu_to_be32(isn);
			PDBG("%s snd_isn %u\n", __func__, t5req6->rsvd);
			t5req6->opt2 = cpu_to_be32(opt2);
		}
	}

	set_bit(ACT_OPEN_REQ, &ep->com.history);
	ret = c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
clip_release:
	if (ret && ep->com.remote_addr.ss_family == AF_INET6)
		cxgb4_clip_release(ep->com.dev->rdev.lldi.ports[0],
				   (const u32 *)&la6->sin6_addr.s6_addr, 1);
	return ret;
}

static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb,
		u8 mpa_rev_to_use)
{
	int mpalen, wrlen;
	struct fw_ofld_tx_data_wr *req;
	struct mpa_message *mpa;
	struct mpa_v2_conn_params mpa_v2_params;

	PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);

	BUG_ON(skb_cloned(skb));

	mpalen = sizeof(*mpa) + ep->plen;
	if (mpa_rev_to_use == 2)
		mpalen += sizeof(struct mpa_v2_conn_params);
	wrlen = roundup(mpalen + sizeof *req, 16);
	skb = get_skb(skb, wrlen, GFP_KERNEL);
	if (!skb) {
		connect_reply_upcall(ep, -ENOMEM);
		return;
	}
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);

	req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen);
	memset(req, 0, wrlen);
	req->op_to_immdlen = cpu_to_be32(
		FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
		FW_WR_COMPL_F |
		FW_WR_IMMDLEN_V(mpalen));
	req->flowid_len16 = cpu_to_be32(
		FW_WR_FLOWID_V(ep->hwtid) |
		FW_WR_LEN16_V(wrlen >> 4));
	req->plen = cpu_to_be32(mpalen);
	req->tunnel_to_proxy = cpu_to_be32(
		FW_OFLD_TX_DATA_WR_FLUSH_F |
		FW_OFLD_TX_DATA_WR_SHOVE_F);

	mpa = (struct mpa_message *)(req + 1);
	memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
	mpa->flags = (crc_enabled ? MPA_CRC : 0) |
		     (markers_enabled ? MPA_MARKERS : 0) |
		     (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0);
	mpa->private_data_size = htons(ep->plen);
	mpa->revision = mpa_rev_to_use;
	if (mpa_rev_to_use == 1) {
		ep->tried_with_mpa_v1 = 1;
		ep->retry_with_mpa_v1 = 0;
	}

	if (mpa_rev_to_use == 2) {
		mpa->private_data_size = htons(ntohs(mpa->private_data_size) +
					       sizeof (struct mpa_v2_conn_params));
		PDBG("%s initiator ird %u ord %u\n", __func__, ep->ird,
		     ep->ord);
		mpa_v2_params.ird = htons((u16)ep->ird);
		mpa_v2_params.ord = htons((u16)ep->ord);

		if (peer2peer) {
			mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL);
			if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE)
				mpa_v2_params.ord |=
					htons(MPA_V2_RDMA_WRITE_RTR);
			else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ)
				mpa_v2_params.ord |=
					htons(MPA_V2_RDMA_READ_RTR);
		}
		memcpy(mpa->private_data, &mpa_v2_params,
		       sizeof(struct mpa_v2_conn_params));

		if (ep->plen)
			memcpy(mpa->private_data +
			       sizeof(struct mpa_v2_conn_params),
			       ep->mpa_pkt + sizeof(*mpa), ep->plen);
	} else
		if (ep->plen)
			memcpy(mpa->private_data,
					ep->mpa_pkt + sizeof(*mpa), ep->plen);

	/*
	 * Reference the mpa skb.  This ensures the data area
	 * will remain in memory until the hw acks the tx.
	 * Function fw4_ack() will deref it.
	 */
	skb_get(skb);
	t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
	BUG_ON(ep->mpa_skb);
	ep->mpa_skb = skb;
	c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
	start_ep_timer(ep);
	__state_set(&ep->com, MPA_REQ_SENT);
	ep->mpa_attr.initiator = 1;
	ep->snd_seq += mpalen;
	return;
}

static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)
{
	int mpalen, wrlen;
	struct fw_ofld_tx_data_wr *req;
	struct mpa_message *mpa;
	struct sk_buff *skb;
	struct mpa_v2_conn_params mpa_v2_params;

	PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);

	mpalen = sizeof(*mpa) + plen;
	if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn)
		mpalen += sizeof(struct mpa_v2_conn_params);
	wrlen = roundup(mpalen + sizeof *req, 16);

	skb = get_skb(NULL, wrlen, GFP_KERNEL);
	if (!skb) {
		printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
		return -ENOMEM;
	}
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);

	req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen);
	memset(req, 0, wrlen);
	req->op_to_immdlen = cpu_to_be32(
		FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
		FW_WR_COMPL_F |
		FW_WR_IMMDLEN_V(mpalen));
	req->flowid_len16 = cpu_to_be32(
		FW_WR_FLOWID_V(ep->hwtid) |
		FW_WR_LEN16_V(wrlen >> 4));
	req->plen = cpu_to_be32(mpalen);
	req->tunnel_to_proxy = cpu_to_be32(
		FW_OFLD_TX_DATA_WR_FLUSH_F |
		FW_OFLD_TX_DATA_WR_SHOVE_F);

	mpa = (struct mpa_message *)(req + 1);
	memset(mpa, 0, sizeof(*mpa));
	memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
	mpa->flags = MPA_REJECT;
	mpa->revision = ep->mpa_attr.version;
	mpa->private_data_size = htons(plen);

	if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
		mpa->flags |= MPA_ENHANCED_RDMA_CONN;
		mpa->private_data_size = htons(ntohs(mpa->private_data_size) +
					       sizeof (struct mpa_v2_conn_params));
		mpa_v2_params.ird = htons(((u16)ep->ird) |
					  (peer2peer ? MPA_V2_PEER2PEER_MODEL :
					   0));
		mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ?
					  (p2p_type ==
					   FW_RI_INIT_P2PTYPE_RDMA_WRITE ?
					   MPA_V2_RDMA_WRITE_RTR : p2p_type ==
					   FW_RI_INIT_P2PTYPE_READ_REQ ?
					   MPA_V2_RDMA_READ_RTR : 0) : 0));
		memcpy(mpa->private_data, &mpa_v2_params,
		       sizeof(struct mpa_v2_conn_params));

		if (ep->plen)
			memcpy(mpa->private_data +
			       sizeof(struct mpa_v2_conn_params), pdata, plen);
	} else
		if (plen)
			memcpy(mpa->private_data, pdata, plen);

	/*
	 * Reference the mpa skb again.  This ensures the data area
	 * will remain in memory until the hw acks the tx.
	 * Function fw4_ack() will deref it.
	 */
	skb_get(skb);
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
	t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
	BUG_ON(ep->mpa_skb);
	ep->mpa_skb = skb;
	ep->snd_seq += mpalen;
	return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
}

static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)
{
	int mpalen, wrlen;
	struct fw_ofld_tx_data_wr *req;
	struct mpa_message *mpa;
	struct sk_buff *skb;
	struct mpa_v2_conn_params mpa_v2_params;

	PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);

	mpalen = sizeof(*mpa) + plen;
	if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn)
		mpalen += sizeof(struct mpa_v2_conn_params);
	wrlen = roundup(mpalen + sizeof *req, 16);

	skb = get_skb(NULL, wrlen, GFP_KERNEL);
	if (!skb) {
		printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
		return -ENOMEM;
	}
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);

	req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen);
	memset(req, 0, wrlen);
	req->op_to_immdlen = cpu_to_be32(
		FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
		FW_WR_COMPL_F |
		FW_WR_IMMDLEN_V(mpalen));
	req->flowid_len16 = cpu_to_be32(
		FW_WR_FLOWID_V(ep->hwtid) |
		FW_WR_LEN16_V(wrlen >> 4));
	req->plen = cpu_to_be32(mpalen);
	req->tunnel_to_proxy = cpu_to_be32(
		FW_OFLD_TX_DATA_WR_FLUSH_F |
		FW_OFLD_TX_DATA_WR_SHOVE_F);

	mpa = (struct mpa_message *)(req + 1);
	memset(mpa, 0, sizeof(*mpa));
	memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
	mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
		     (markers_enabled ? MPA_MARKERS : 0);
	mpa->revision = ep->mpa_attr.version;
	mpa->private_data_size = htons(plen);

	if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
		mpa->flags |= MPA_ENHANCED_RDMA_CONN;
		mpa->private_data_size = htons(ntohs(mpa->private_data_size) +
					       sizeof (struct mpa_v2_conn_params));
		mpa_v2_params.ird = htons((u16)ep->ird);
		mpa_v2_params.ord = htons((u16)ep->ord);
		if (peer2peer && (ep->mpa_attr.p2p_type !=
					FW_RI_INIT_P2PTYPE_DISABLED)) {
			mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL);

			if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE)
				mpa_v2_params.ord |=
					htons(MPA_V2_RDMA_WRITE_RTR);
			else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ)
				mpa_v2_params.ord |=
					htons(MPA_V2_RDMA_READ_RTR);
		}

		memcpy(mpa->private_data, &mpa_v2_params,
		       sizeof(struct mpa_v2_conn_params));

		if (ep->plen)
			memcpy(mpa->private_data +
			       sizeof(struct mpa_v2_conn_params), pdata, plen);
	} else
		if (plen)
			memcpy(mpa->private_data, pdata, plen);

	/*
	 * Reference the mpa skb.  This ensures the data area
	 * will remain in memory until the hw acks the tx.
	 * Function fw4_ack() will deref it.
	 */
	skb_get(skb);
	t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
	ep->mpa_skb = skb;
	__state_set(&ep->com, MPA_REP_SENT);
	ep->snd_seq += mpalen;
	return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
}

static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb)
{
	struct c4iw_ep *ep;
	struct cpl_act_establish *req = cplhdr(skb);
	unsigned int tid = GET_TID(req);
	unsigned int atid = TID_TID_G(ntohl(req->tos_atid));
	struct tid_info *t = dev->rdev.lldi.tids;

	ep = lookup_atid(t, atid);

	PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid,
	     be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn));

	mutex_lock(&ep->com.mutex);
	dst_confirm(ep->dst);

	/* setup the hwtid for this connection */
	ep->hwtid = tid;
	cxgb4_insert_tid(t, ep, tid);
	insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid);

	ep->snd_seq = be32_to_cpu(req->snd_isn);
	ep->rcv_seq = be32_to_cpu(req->rcv_isn);

	set_emss(ep, ntohs(req->tcp_opt));

	/* dealloc the atid */
	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid);
	cxgb4_free_atid(t, atid);
	set_bit(ACT_ESTAB, &ep->com.history);

	/* start MPA negotiation */
	send_flowc(ep, NULL);
	if (ep->retry_with_mpa_v1)
		send_mpa_req(ep, skb, 1);
	else
		send_mpa_req(ep, skb, mpa_rev);
	mutex_unlock(&ep->com.mutex);
	return 0;
}

static void close_complete_upcall(struct c4iw_ep *ep, int status)
{
	struct iw_cm_event event;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	memset(&event, 0, sizeof(event));
	event.event = IW_CM_EVENT_CLOSE;
	event.status = status;
	if (ep->com.cm_id) {
		PDBG("close complete delivered ep %p cm_id %p tid %u\n",
		     ep, ep->com.cm_id, ep->hwtid);
		ep->com.cm_id->event_handler(ep->com.cm_id, &event);
		ep->com.cm_id->rem_ref(ep->com.cm_id);
		ep->com.cm_id = NULL;
		set_bit(CLOSE_UPCALL, &ep->com.history);
	}
}

static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp)
{
	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	__state_set(&ep->com, ABORTING);
	set_bit(ABORT_CONN, &ep->com.history);
	return send_abort(ep, skb, gfp);
}

static void peer_close_upcall(struct c4iw_ep *ep)
{
	struct iw_cm_event event;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	memset(&event, 0, sizeof(event));
	event.event = IW_CM_EVENT_DISCONNECT;
	if (ep->com.cm_id) {
		PDBG("peer close delivered ep %p cm_id %p tid %u\n",
		     ep, ep->com.cm_id, ep->hwtid);
		ep->com.cm_id->event_handler(ep->com.cm_id, &event);
		set_bit(DISCONN_UPCALL, &ep->com.history);
	}
}

static void peer_abort_upcall(struct c4iw_ep *ep)
{
	struct iw_cm_event event;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	memset(&event, 0, sizeof(event));
	event.event = IW_CM_EVENT_CLOSE;
	event.status = -ECONNRESET;
	if (ep->com.cm_id) {
		PDBG("abort delivered ep %p cm_id %p tid %u\n", ep,
		     ep->com.cm_id, ep->hwtid);
		ep->com.cm_id->event_handler(ep->com.cm_id, &event);
		ep->com.cm_id->rem_ref(ep->com.cm_id);
		ep->com.cm_id = NULL;
		set_bit(ABORT_UPCALL, &ep->com.history);
	}
}

static void connect_reply_upcall(struct c4iw_ep *ep, int status)
{
	struct iw_cm_event event;

	PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status);
	memset(&event, 0, sizeof(event));
	event.event = IW_CM_EVENT_CONNECT_REPLY;
	event.status = status;
	memcpy(&event.local_addr, &ep->com.local_addr,
	       sizeof(ep->com.local_addr));
	memcpy(&event.remote_addr, &ep->com.remote_addr,
	       sizeof(ep->com.remote_addr));

	if ((status == 0) || (status == -ECONNREFUSED)) {
		if (!ep->tried_with_mpa_v1) {
			/* this means MPA_v2 is used */
			event.ord = ep->ird;
			event.ird = ep->ord;
			event.private_data_len = ep->plen -
				sizeof(struct mpa_v2_conn_params);
			event.private_data = ep->mpa_pkt +
				sizeof(struct mpa_message) +
				sizeof(struct mpa_v2_conn_params);
		} else {
			/* this means MPA_v1 is used */
			event.ord = cur_max_read_depth(ep->com.dev);
			event.ird = cur_max_read_depth(ep->com.dev);
			event.private_data_len = ep->plen;
			event.private_data = ep->mpa_pkt +
				sizeof(struct mpa_message);
		}
	}

	PDBG("%s ep %p tid %u status %d\n", __func__, ep,
	     ep->hwtid, status);
	set_bit(CONN_RPL_UPCALL, &ep->com.history);
	ep->com.cm_id->event_handler(ep->com.cm_id, &event);

	if (status < 0) {
		ep->com.cm_id->rem_ref(ep->com.cm_id);
		ep->com.cm_id = NULL;
	}
}

static int connect_request_upcall(struct c4iw_ep *ep)
{
	struct iw_cm_event event;
	int ret;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	memset(&event, 0, sizeof(event));
	event.event = IW_CM_EVENT_CONNECT_REQUEST;
	memcpy(&event.local_addr, &ep->com.local_addr,
	       sizeof(ep->com.local_addr));
	memcpy(&event.remote_addr, &ep->com.remote_addr,
	       sizeof(ep->com.remote_addr));
	event.provider_data = ep;
	if (!ep->tried_with_mpa_v1) {
		/* this means MPA_v2 is used */
		event.ord = ep->ord;
		event.ird = ep->ird;
		event.private_data_len = ep->plen -
			sizeof(struct mpa_v2_conn_params);
		event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) +
			sizeof(struct mpa_v2_conn_params);
	} else {
		/* this means MPA_v1 is used. Send max supported */
		event.ord = cur_max_read_depth(ep->com.dev);
		event.ird = cur_max_read_depth(ep->com.dev);
		event.private_data_len = ep->plen;
		event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
	}
	c4iw_get_ep(&ep->com);
	ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id,
						      &event);
	if (ret)
		c4iw_put_ep(&ep->com);
	set_bit(CONNREQ_UPCALL, &ep->com.history);
	c4iw_put_ep(&ep->parent_ep->com);
	return ret;
}

static void established_upcall(struct c4iw_ep *ep)
{
	struct iw_cm_event event;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	memset(&event, 0, sizeof(event));
	event.event = IW_CM_EVENT_ESTABLISHED;
	event.ird = ep->ord;
	event.ord = ep->ird;
	if (ep->com.cm_id) {
		PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
		ep->com.cm_id->event_handler(ep->com.cm_id, &event);
		set_bit(ESTAB_UPCALL, &ep->com.history);
	}
}

static int update_rx_credits(struct c4iw_ep *ep, u32 credits)
{
	struct cpl_rx_data_ack *req;
	struct sk_buff *skb;
	int wrlen = roundup(sizeof *req, 16);

	PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits);
	skb = get_skb(NULL, wrlen, GFP_KERNEL);
	if (!skb) {
		printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n");
		return 0;
	}

	/*
	 * If we couldn't specify the entire rcv window at connection setup
	 * due to the limit in the number of bits in the RCV_BUFSIZ field,
	 * then add the overage in to the credits returned.
	 */
	if (ep->rcv_win > RCV_BUFSIZ_M * 1024)
		credits += ep->rcv_win - RCV_BUFSIZ_M * 1024;

	req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen);
	memset(req, 0, wrlen);
	INIT_TP_WR(req, ep->hwtid);
	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
						    ep->hwtid));
	req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK_F |
				       RX_DACK_CHANGE_F |
				       RX_DACK_MODE_V(dack_mode));
	set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx);
	c4iw_ofld_send(&ep->com.dev->rdev, skb);
	return credits;
}

#define RELAXED_IRD_NEGOTIATION 1

static int process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb)
{
	struct mpa_message *mpa;
	struct mpa_v2_conn_params *mpa_v2_params;
	u16 plen;
	u16 resp_ird, resp_ord;
	u8 rtr_mismatch = 0, insuff_ird = 0;
	struct c4iw_qp_attributes attrs;
	enum c4iw_qp_attr_mask mask;
	int err;
	int disconnect = 0;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);

	/*
	 * Stop mpa timer.  If it expired, then
	 * we ignore the MPA reply.  process_timeout()
	 * will abort the connection.
	 */
	if (stop_ep_timer(ep))
		return 0;

	/*
	 * If we get more than the supported amount of private data
	 * then we must fail this connection.
	 */
	if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
		err = -EINVAL;
		goto err;
	}

	/*
	 * copy the new data into our accumulation buffer.
	 */
	skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
				  skb->len);
	ep->mpa_pkt_len += skb->len;

	/*
	 * if we don't even have the mpa message, then bail.
	 */
	if (ep->mpa_pkt_len < sizeof(*mpa))
		return 0;
	mpa = (struct mpa_message *) ep->mpa_pkt;

	/* Validate MPA header. */
	if (mpa->revision > mpa_rev) {
		printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d,"
		       " Received = %d\n", __func__, mpa_rev, mpa->revision);
		err = -EPROTO;
		goto err;
	}
	if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
		err = -EPROTO;
		goto err;
	}

	plen = ntohs(mpa->private_data_size);

	/*
	 * Fail if there's too much private data.
	 */
	if (plen > MPA_MAX_PRIVATE_DATA) {
		err = -EPROTO;
		goto err;
	}

	/*
	 * If plen does not account for pkt size
	 */
	if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
		err = -EPROTO;
		goto err;
	}

	ep->plen = (u8) plen;

	/*
	 * If we don't have all the pdata yet, then bail.
	 * We'll continue process when more data arrives.
	 */
	if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
		return 0;

	if (mpa->flags & MPA_REJECT) {
		err = -ECONNREFUSED;
		goto err;
	}

	/*
	 * If we get here we have accumulated the entire mpa
	 * start reply message including private data. And
	 * the MPA header is valid.
	 */
	__state_set(&ep->com, FPDU_MODE);
	ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
	ep->mpa_attr.recv_marker_enabled = markers_enabled;
	ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
	ep->mpa_attr.version = mpa->revision;
	ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED;

	if (mpa->revision == 2) {
		ep->mpa_attr.enhanced_rdma_conn =
			mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0;
		if (ep->mpa_attr.enhanced_rdma_conn) {
			mpa_v2_params = (struct mpa_v2_conn_params *)
				(ep->mpa_pkt + sizeof(*mpa));
			resp_ird = ntohs(mpa_v2_params->ird) &
				MPA_V2_IRD_ORD_MASK;
			resp_ord = ntohs(mpa_v2_params->ord) &
				MPA_V2_IRD_ORD_MASK;
			PDBG("%s responder ird %u ord %u ep ird %u ord %u\n",
			     __func__, resp_ird, resp_ord, ep->ird, ep->ord);

			/*
			 * This is a double-check. Ideally, below checks are
			 * not required since ird/ord stuff has been taken
			 * care of in c4iw_accept_cr
			 */
			if (ep->ird < resp_ord) {
				if (RELAXED_IRD_NEGOTIATION && resp_ord <=
				    ep->com.dev->rdev.lldi.max_ordird_qp)
					ep->ird = resp_ord;
				else
					insuff_ird = 1;
			} else if (ep->ird > resp_ord) {
				ep->ird = resp_ord;
			}
			if (ep->ord > resp_ird) {
				if (RELAXED_IRD_NEGOTIATION)
					ep->ord = resp_ird;
				else
					insuff_ird = 1;
			}
			if (insuff_ird) {
				err = -ENOMEM;
				ep->ird = resp_ord;
				ep->ord = resp_ird;
			}

			if (ntohs(mpa_v2_params->ird) &
					MPA_V2_PEER2PEER_MODEL) {
				if (ntohs(mpa_v2_params->ord) &
						MPA_V2_RDMA_WRITE_RTR)
					ep->mpa_attr.p2p_type =
						FW_RI_INIT_P2PTYPE_RDMA_WRITE;
				else if (ntohs(mpa_v2_params->ord) &
						MPA_V2_RDMA_READ_RTR)
					ep->mpa_attr.p2p_type =
						FW_RI_INIT_P2PTYPE_READ_REQ;
			}
		}
	} else if (mpa->revision == 1)
		if (peer2peer)
			ep->mpa_attr.p2p_type = p2p_type;

	PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
	     "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = "
	     "%d\n", __func__, ep->mpa_attr.crc_enabled,
	     ep->mpa_attr.recv_marker_enabled,
	     ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version,
	     ep->mpa_attr.p2p_type, p2p_type);

	/*
	 * If responder's RTR does not match with that of initiator, assign
	 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not
	 * generated when moving QP to RTS state.
	 * A TERM message will be sent after QP has moved to RTS state
	 */
	if ((ep->mpa_attr.version == 2) && peer2peer &&
			(ep->mpa_attr.p2p_type != p2p_type)) {
		ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED;
		rtr_mismatch = 1;
	}

	attrs.mpa_attr = ep->mpa_attr;
	attrs.max_ird = ep->ird;
	attrs.max_ord = ep->ord;
	attrs.llp_stream_handle = ep;
	attrs.next_state = C4IW_QP_STATE_RTS;

	mask = C4IW_QP_ATTR_NEXT_STATE |
	    C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR |
	    C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD;

	/* bind QP and TID with INIT_WR */
	err = c4iw_modify_qp(ep->com.qp->rhp,
			     ep->com.qp, mask, &attrs, 1);
	if (err)
		goto err;

	/*
	 * If responder's RTR requirement did not match with what initiator
	 * supports, generate TERM message
	 */
	if (rtr_mismatch) {
		printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__);
		attrs.layer_etype = LAYER_MPA | DDP_LLP;
		attrs.ecode = MPA_NOMATCH_RTR;
		attrs.next_state = C4IW_QP_STATE_TERMINATE;
		attrs.send_term = 1;
		err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
				C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
		err = -ENOMEM;
		disconnect = 1;
		goto out;
	}

	/*
	 * Generate TERM if initiator IRD is not sufficient for responder
	 * provided ORD. Currently, we do the same behaviour even when
	 * responder provided IRD is also not sufficient as regards to
	 * initiator ORD.
	 */
	if (insuff_ird) {
		printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n",
				__func__);
		attrs.layer_etype = LAYER_MPA | DDP_LLP;
		attrs.ecode = MPA_INSUFF_IRD;
		attrs.next_state = C4IW_QP_STATE_TERMINATE;
		attrs.send_term = 1;
		err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
				C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
		err = -ENOMEM;
		disconnect = 1;
		goto out;
	}
	goto out;
err:
	__state_set(&ep->com, ABORTING);
	send_abort(ep, skb, GFP_KERNEL);
out:
	connect_reply_upcall(ep, err);
	return disconnect;
}

static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
{
	struct mpa_message *mpa;
	struct mpa_v2_conn_params *mpa_v2_params;
	u16 plen;

	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);

	/*
	 * If we get more than the supported amount of private data
	 * then we must fail this connection.
	 */
	if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
		(void)stop_ep_timer(ep);
		abort_connection(ep, skb, GFP_KERNEL);
		return;
	}

	PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);

	/*
	 * Copy the new data into our accumulation buffer.
	 */
	skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
				  skb->len);
	ep->mpa_pkt_len += skb->len;

	/*
	 * If we don't even have the mpa message, then bail.
	 * We'll continue process when more data arrives.
	 */
	if (ep->mpa_pkt_len < sizeof(*mpa))
		return;

	PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
	mpa = (struct mpa_message *) ep->mpa_pkt;

	/*
	 * Validate MPA Header.
	 */
	if (mpa->revision > mpa_rev) {
		printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d,"
		       " Received = %d\n", __func__, mpa_rev, mpa->revision);
		(void)stop_ep_timer(ep);
		abort_connection(ep, skb, GFP_KERNEL);
		return;
	}

	if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
		(void)stop_ep_timer(ep);
		abort_connection(ep, skb, GFP_KERNEL);
		return;
	}

	plen = ntohs(mpa->private_data_size);

	/*
	 * Fail if there's too much private data.
	 */
	if (plen > MPA_MAX_PRIVATE_DATA) {
		(void)stop_ep_timer(ep);
		abort_connection(ep, skb, GFP_KERNEL);
		return;
	}

	/*
	 * If plen does not account for pkt size
	 */
	if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
		(void)stop_ep_timer(ep);
		abort_connection(ep, skb, GFP_KERNEL);
		return;
	}
	ep->plen = (u8) plen;

	/*
	 * If we don't have all the pdata yet, then bail.
	 */
	if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
		return;

	/*
	 * If we get here we have accumulated the entire mpa
	 * start reply message including private data.
	 */
	ep->mpa_attr.initiator = 0;
	ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
	ep->mpa_attr.recv_marker_enabled = markers_enabled;
	ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
	ep->mpa_attr.version = mpa->revision;
	if (mpa->revision == 1)
		ep->tried_with_mpa_v1 = 1;
	ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED;

	if (mpa->revision == 2) {
		ep->mpa_attr.enhanced_rdma_conn =
			mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0;
		if (ep->mpa_attr.enhanced_rdma_conn) {
			mpa_v2_params = (struct mpa_v2_conn_params *)
				(ep->mpa_pkt + sizeof(*mpa));
			ep->ird = ntohs(mpa_v2_params->ird) &
				MPA_V2_IRD_ORD_MASK;
			ep->ord = ntohs(mpa_v2_params->ord) &
				MPA_V2_IRD_ORD_MASK;
			PDBG("%s initiator ird %u ord %u\n", __func__, ep->ird,
			     ep->ord);
			if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL)
				if (peer2peer) {
					if (ntohs(mpa_v2_params->ord) &
							MPA_V2_RDMA_WRITE_RTR)
						ep->mpa_attr.p2p_type =
						FW_RI_INIT_P2PTYPE_RDMA_WRITE;
					else if (ntohs(mpa_v2_params->ord) &
							MPA_V2_RDMA_READ_RTR)
						ep->mpa_attr.p2p_type =
						FW_RI_INIT_P2PTYPE_READ_REQ;
				}
		}
	} else if (mpa->revision == 1)
		if (peer2peer)
			ep->mpa_attr.p2p_type = p2p_type;

	PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
	     "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__,
	     ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
	     ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version,
	     ep->mpa_attr.p2p_type);

	/*
	 * If the endpoint timer already expired, then we ignore
	 * the start request.  process_timeout() will abort
	 * the connection.
	 */
	if (!stop_ep_timer(ep)) {
		__state_set(&ep->com, MPA_REQ_RCVD);

		/* drive upcall */
		mutex_lock_nested(&ep->parent_ep->com.mutex,
				  SINGLE_DEPTH_NESTING);
		if (ep->parent_ep->com.state != DEAD) {
			if (connect_request_upcall(ep))
				abort_connection(ep, skb, GFP_KERNEL);
		} else {
			abort_connection(ep, skb, GFP_KERNEL);
		}
		mutex_unlock(&ep->parent_ep->com.mutex);
	}
	return;
}

static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb)
{
	struct c4iw_ep *ep;
	struct cpl_rx_data *hdr = cplhdr(skb);
	unsigned int dlen = ntohs(hdr->len);
	unsigned int tid = GET_TID(hdr);
	struct tid_info *t = dev->rdev.lldi.tids;
	__u8 status = hdr->status;
	int disconnect = 0;

	ep = lookup_tid(t, tid);
	if (!ep)
		return 0;
	PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen);
	skb_pull(skb, sizeof(*hdr));
	skb_trim(skb, dlen);
	mutex_lock(&ep->com.mutex);

	/* update RX credits */
	update_rx_credits(ep, dlen);

	switch (ep->com.state) {
	case MPA_REQ_SENT:
		ep->rcv_seq += dlen;
		disconnect = process_mpa_reply(ep, skb);
		break;
	case MPA_REQ_WAIT:
		ep->rcv_seq += dlen;
		process_mpa_request(ep, skb);
		break;
	case FPDU_MODE: {
		struct c4iw_qp_attributes attrs;
		BUG_ON(!ep->com.qp);
		if (status)
			pr_err("%s Unexpected streaming data." \
			       " qpid %u ep %p state %d tid %u status %d\n",
			       __func__, ep->com.qp->wq.sq.qid, ep,
			       ep->com.state, ep->hwtid, status);
		attrs.next_state = C4IW_QP_STATE_TERMINATE;
		c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
			       C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
		disconnect = 1;
		break;
	}
	default:
		break;
	}
	mutex_unlock(&ep->com.mutex);
	if (disconnect)
		c4iw_ep_disconnect(ep, 0, GFP_KERNEL);
	return 0;
}

static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
{
	struct c4iw_ep *ep;
	struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
	int release = 0;
	unsigned int tid = GET_TID(rpl);
	struct tid_info *t = dev->rdev.lldi.tids;

	ep = lookup_tid(t, tid);
	if (!ep) {
		printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n");
		return 0;
	}
	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
	mutex_lock(&ep->com.mutex);
	switch (ep->com.state) {
	case ABORTING:
		c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET);
		__state_set(&ep->com, DEAD);
		release = 1;
		break;
	default:
		printk(KERN_ERR "%s ep %p state %d\n",
		     __func__, ep, ep->com.state);
		break;
	}
	mutex_unlock(&ep->com.mutex);

	if (release)
		release_ep_resources(ep);
	return 0;
}

static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)
{
	struct sk_buff *skb;
	struct fw_ofld_connection_wr *req;
	unsigned int mtu_idx;
	int wscale;
	struct sockaddr_in *sin;
	int win;

	skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
	req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req));
	memset(req, 0, sizeof(*req));
	req->op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR));
	req->len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)));
	req->le.filter = cpu_to_be32(cxgb4_select_ntuple(
				     ep->com.dev->rdev.lldi.ports[0],
				     ep->l2t));
	sin = (struct sockaddr_in *)&ep->com.mapped_local_addr;
	req->le.lport = sin->sin_port;
	req->le.u.ipv4.lip = sin->sin_addr.s_addr;
	sin = (struct sockaddr_in *)&ep->com.mapped_remote_addr;
	req->le.pport = sin->sin_port;
	req->le.u.ipv4.pip = sin->sin_addr.s_addr;
	req->tcb.t_state_to_astid =
			htonl(FW_OFLD_CONNECTION_WR_T_STATE_V(TCP_SYN_SENT) |
			FW_OFLD_CONNECTION_WR_ASTID_V(atid));
	req->tcb.cplrxdataack_cplpassacceptrpl =
			htons(FW_OFLD_CONNECTION_WR_CPLRXDATAACK_F);
	req->tcb.tx_max = (__force __be32) jiffies;
	req->tcb.rcv_adv = htons(1);
	best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
		 enable_tcp_timestamps,
		 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
	wscale = compute_wscale(rcv_win);

	/*
	 * Specify the largest window that will fit in opt0. The
	 * remainder will be specified in the rx_data_ack.
	 */
	win = ep->rcv_win >> 10;
	if (win > RCV_BUFSIZ_M)
		win = RCV_BUFSIZ_M;

	req->tcb.opt0 = (__force __be64) (TCAM_BYPASS_F |
		(nocong ? NO_CONG_F : 0) |
		KEEP_ALIVE_F |
		DELACK_F |
		WND_SCALE_V(wscale) |
		MSS_IDX_V(mtu_idx) |
		L2T_IDX_V(ep->l2t->idx) |
		TX_CHAN_V(ep->tx_chan) |
		SMAC_SEL_V(ep->smac_idx) |
		DSCP_V(ep->tos) |
		ULP_MODE_V(ULP_MODE_TCPDDP) |
		RCV_BUFSIZ_V(win));
	req->tcb.opt2 = (__force __be32) (PACE_V(1) |
		TX_QUEUE_V(ep->com.dev->rdev.lldi.tx_modq[ep->tx_chan]) |
		RX_CHANNEL_V(0) |
		CCTRL_ECN_V(enable_ecn) |
		RSS_QUEUE_VALID_F | RSS_QUEUE_V(ep->rss_qid));
	if (enable_tcp_timestamps)
		req->tcb.opt2 |= (__force __be32)TSTAMPS_EN_F;
	if (enable_tcp_sack)
		req->tcb.opt2 |= (__force __be32)SACK_EN_F;
	if (wscale && enable_tcp_window_scaling)
		req->tcb.opt2 |= (__force __be32)WND_SCALE_EN_F;
	req->tcb.opt0 = cpu_to_be64((__force u64)req->tcb.opt0);
	req->tcb.opt2 = cpu_to_be32((__force u32)req->tcb.opt2);
	set_wr_txq(skb, CPL_PRIORITY_CONTROL, ep->ctrlq_idx);
	set_bit(ACT_OFLD_CONN, &ep->com.history);
	c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
}

/*
 * Return whether a failed active open has allocated a TID
 */
static inline int act_open_has_tid(int status)
{
	return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
	       status != CPL_ERR_ARP_MISS;
}

/* Returns whether a CPL status conveys negative advice.
 */
static int is_neg_adv(unsigned int status)
{
	return status == CPL_ERR_RTX_NEG_ADVICE ||
	       status == CPL_ERR_PERSIST_NEG_ADVICE ||
	       status == CPL_ERR_KEEPALV_NEG_ADVICE;
}

static char *neg_adv_str(unsigned int status)
{
	switch (status) {
	case CPL_ERR_RTX_NEG_ADVICE:
		return "Retransmit timeout";
	case CPL_ERR_PERSIST_NEG_ADVICE:
		return "Persist timeout";
	case CPL_ERR_KEEPALV_NEG_ADVICE:
		return "Keepalive timeout";
	default:
		return "Unknown";
	}
}

static void set_tcp_window(struct c4iw_ep *ep, struct port_info *pi)
{
	ep->snd_win = snd_win;
	ep->rcv_win = rcv_win;
	PDBG("%s snd_win %d rcv_win %d\n", __func__, ep->snd_win, ep->rcv_win);
}

#define ACT_OPEN_RETRY_COUNT 2

static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip,
		     struct dst_entry *dst, struct c4iw_dev *cdev,
		     bool clear_mpa_v1, enum chip_type adapter_type)
{
	struct neighbour *n;
	int err, step;
	struct net_device *pdev;

	n = dst_neigh_lookup(dst, peer_ip);
	if (!n)
		return -ENODEV;

	rcu_read_lock();
	err = -ENOMEM;
	if (n->dev->flags & IFF_LOOPBACK) {