aboutsummaryrefslogblamecommitdiffstats
path: root/drivers/infiniband/hw/mlx4/qp.c
blob: 429a59c5801cc129cc2ac477495fb49c8adbdecc (plain) (tree)
1
2
3

                                                              
                                                                       





























                                                                      
                       
                              
                   
                       
                            
 

                          
                         
                        
                              
 
                              


                          
                          
 



                                                           

                                                                             
 





                                               


                                               



          
                                                                  


                                                                     
           
                                             
                                              








                                                                   
                                             

  
      

                                     

  




                                     


                         
 
                                        









                                                                                          
                                                                               

                                                                                            

  




                            




                                                          




                                                                       


                                                                     

 

                                                                 











                                                                             

                                                                                  




                                              



                                                      

 
                                    

                                                                 











                                                                             
                                                                                  





                                              



                                                       
                                                 











                                                                    

                                                                    

                                                                      
   
                                                        
 
                    
              
              
                  
                                       
 





                                                         
         

 




































                                                                      
                                                           







                                                              





                                                                                                        
                                                                  







                                                                   
                            
                                                          
                                                               
                                                                                  










                                                                   

                                                           
                            
                                                          
                                                                    
                                                           

                             

                                                          

                                                                  









                                                                  
                                                                      
                                                                       
                                       
 
                                                    

                                                                                        

                               
                      
                                                    
                                       
 
                                                   
                



                                                                
                                                                      

                                                                          

                                       
                                                                                 
                                                                                  

                                                                            
         
 










                                                                                            




                                                                             
                                                                               
 

              
                                                    

                                                                                          
                                                                       






                                                                                

                                                                                 


                                                             

                                                                             
                                                   
 


                                              

                                                        
          

                                                                
           
                                                                  






                                                               
                                                  


                                                             

                                                  
                                                                   
                
                                                                   


                                  
                                             
                                                   


                                                               

                                                                



                 

                                                    

                                                           






                                                                           
                                                      

                                               

                                                             



                 




                                                                         

                                                                         











                                                                                



                                                                          




























                                                                          







                                                                               



                                                        

                                                                



                 











                                                               















                                                                               






                                                                         

                                                                         




                                                                               
























                                                                                                       
                                                   
                                                            
                                                   
                                                            
                                                                          










                                                                                                               
                                                   
                                                            
                                                   
                                                            
                                                                          


                                                                                                               





                                                                                                      













                                                                                 


                 
                                                 





















                                                                            
                                                     






























































                                                                                


                                                                
































                                                                      
                                                                       









                                      


















































































                                                                                
                                                                      
                                                         
                                                              

                                                             
 
                
                
                                       
                              

                                                                     
                                                                                 

                               
                           







                                                                       






                                                                              













                                                                                    


                                                                             


                                                            



                                                                                






                                                                                      
                                                                              


                                               

                                             
                        
                                                                            

                                               

                                             




                                      



                                     
                                      
                                            

                                        

                                                                          
 
 
                    




                                                    

                          



                                                                               
 
                                                                 



                                      
                                             

                                                                         



                                                                           
 










                                                                                             
 










                                                                         
                                                              
                                                                             


                                 


















                                                                             




                                                                                




                                                



                                                                            






                                                                      
                                           
                                                  
                                               
                                                                           

                                                                          


                                             
                                                          
                
                                                              



                                                               

                                       


                                                                        







                                                                      
                                                                            


                                 
                                           
                                                                  

                                         
 

                                       
 
                                                                          
                                               

                                      






                                                                                 
                                                                       


                                     



                                                                      



                                                   
                                                      

         
                   






                                                                        
                                            
                                                                       

                                      
                


                                                                             
                                                            
                                                                         


                                                                                  

                                                                             
                    



                                                                           
                                                                                    
                        
                                       

         


                                                                            
                                                     
                
                             
 


                                                 






                                                                      


                                                                    

                                

















                                                                      

                 
        


                                                           
                                                
                                                            


                                                                


                                                         
         
                    
                                         
                                                                
                

                                    





                                             
                     




                                                                
                                           
                                                

    
                                
                          

                   

















                                                                                    
                                                             
 
                                 
                                          

                                                         
                                          

                                                                       
                                          




                                                                                      
                                                             
 

                                          
                                            
                                                         
                                            
                                            

                                            
                                            


         









                                                                







                                                           
                                                                        











                                                                             



                                                                               



                      





























                                                                              
                                                                             
                                                                         

                                             
                            
 
                                        

                                                                            
                                                                    
                                            
                                                                           

                                                                                       
                                              

















                                                                                       
 
                                             
 
                                                                 

                                           



                                                                               









                                                                             
                                                                      

                                         
 


                                                                   


                                                                               


                                                                        
 


                                             





                                                                         

                                          

                                    


                                                                        
                                                                
                                   
                                                        
         

                            

 











                                                                             
                                                                             
            
                                                                             

 


                                                                          
 
                                     
                
                                                                     
                      
 


                                                                    
          

                                                                              
           

                                                                             

                                                                 
                                                          
                                                                       
                                        
 




                                                              




                                                                             








                                                                                        
 
                                     









                                                                                    

                       
                               
                                                      

                                                

                                     


                                  

                                                                               

                                  
                                            
                 

                                              
                                  





                        

                         
                                                                     
                          
                                                
                                                                              


                                                                                 





                                                                           
 

                                                                               
                        
                                            

                                                      

                                                                                           









                                           


































                                                                                                                






                                                      

                                                            




                                                                  


                                                  


                                         
                                                                         
         








                                    













                                                        
                                                                         

                       

















                                                                         


         
                                                                                        




























                                                          
                                                                                   














                                                                           

                                                        
                                                                            
                                                                              
 
                 
                       
                
 




                                                                 




                                                                                      
 


                                                                         
                                                                             
 

                                               
                                                                            
                                                                        
                                                                                        



                                          
                                                   
                                                  
                                        


                                                                

         
                                                 
                                                            

                                  
                                                                 
                                                                            
 
                                                     
                                        

























                                                                                                  
                                                              






                                                                                       
                 





                                                                        

                                                                  











                                                                                    
                                                     



                                                           
                                                                 
                                                                              
         
 


                 

                                                                              
                                                

                                                            

                                                

                                                    
                                                     




                                                               
                                                    


                                                                

                                     
                                                     

 











                                                                           

                                                            

                                                                    


                       
                                                              

                                                                                            
                                                  












                                                                             











                                                                               
                                                                            




















                                                                            

















                                                    



                                                                             

                                                                          
 
                    
              



















                                                                              
                                                                          




















                                                                             

                                                                          










                                                                                 

                                                                 









                                                                         
                                                                          




























                                                                          
                                                                            
                                                                            


                                                          
 
                              
                                                  
                                             


                                

                                             

                                                                      

                                        
                      
                         
                          
                          
 


                                          
                                                  





                                                           


                                   
                                                  





                                                    

         





                                                               



                                                       
                                                                       
                                                                                   

















                                                                                 


                                                        


                                                        
                                                           
                                                              
                                              
                                                                                  
                                        



                                                                               
                                                                      
                                                
                                                                                  
                                                            
                                               
                                 
                 

                                                             

         




                                                                             
 

                                                                     

                                                        


                                                           
                                                                    
                                                                     
                                                             
                                                 
                                                                
         
 
                     
                                                
                                                                             
            

                                                                             











                                                                           
                                                                  



                                                    

                                                                          


                                                                 

                                                                        
                                                               





                                                                           
                      

                                                                  




                                                         
 
                                            





                                                                                        

         
                                           

                                                                    



                                                                
                                   
                                                            
                                                                           

                                  


                                                                         
 
                             


                                                                         
                 
 
                                                                               
                                                        
                                 


                                                             


                                                                            
                                                                               







                                                                            


                                        
                                                              



                                                     

                                                                  
                                 
 

                                                                      
                                 
 

                                                               
                                                          
                                 





                                                                    


                                           
                                                                           




                                                                     

                                                          
 
                                                                
                      

                                                         




                                                                       














                                                                                











                                                                                       
                  








                                                                                   
                                                                             
                                     















                                                                              


                                               


                                                                       
 


                                        



                                                               

                                                                     
                                                                    


                                                                                
                                                                                         




                                                                        
                                                                                     
                 








                                                                                
                                                                                 



                                                      

                                                                                 

                         

         
                                           

                                                                            

                                                                                            
                                         


                                                                     
 
                                                                








                                                                        





                                                                               
                        

                                        
                                                     
 

                                                                   


                                                                 
           
                        

                                        


                                               
                                                      

                                                                  


                                                            


                 






                                                                             











                                                                          
                                     
                                          

                                         











                                                                       
                                                               
                                                                         
                                                 









                                                                           
                                        
                                
                                                              
                                                                        









                                                                             
 


                                                                 
                                                                           

                                                                                       
                                              










                                                                                       
 





                                                                                       
         
    

                                                 

                                                 
                       

                                                                       


                                                                                                           
                                                                                 

























































                                                                                                           


                   




                                                                 

                                                                          




                                                        




                                                                                 
                                                                    
                                             




                                                                      
                         
         
 




















                                                                                                              















                                                                                
                                       
                                                                       



                                                                        


                         




                                                                               

                                                                           




                                                                               
                                 
                 

         

                                                                    



                                                                      




                                                                         



                                                                           







                                                                  
                                                              

                                                                         



                                 
                                                                        
                                                               
 
                                     
                                                              
 


                                                                 




                                 




















                                                                              



                                                                    


                                                               





                                 
                                                          
                                                            





                                                                   
                                               






                        
                                        



                               

                                                      





                                                                        
                                                                           
















                                                                              
                                                                                       


                                                                 
                                                                                 

                                                    
                                                                                           

                                                                                  






                                                                           
















































                                                                             















                                                                  












                                                                       
                                  




                               
                                    
                                                                               
                                                             
 
                                                       
                                                    
                                           
                                             
                                                            
                                               
                          


                        
                
              
                    
                          


                             

                            

                      

                                                      
 

                                                                                                       
                     
                                          



                                                                          




                                                                                                
                         


                                                                       
                                   
                                                                                







                                                                                         
                                           
                         
                 
                                                             


                                                                     
         
                                                                            
                                                                  

                           






                                                                                     
 
                                          
                                                  
                                                                                  
                                                  

                                                                             
                             
                                                                                






                                                                                    


                                                                                            



                                                                                       

                                                                
                         
                 
                                                              
                                           

         


















                                                                                  
                                                           





                                                                                          

                                                                            

                                                               
 
                                






                                                                                         
                                                                      




                               
                     
                                    
                               

                                                                                   
                                                    

                                                                  
                                                   
 
                                                                             
                                                                     


                                                                   
 
 

                                                                                     
                               
                                                                          
                        
                                                                           

                                                                          
                





                                                                                                  


                                                                            
                                                                                       


                                                                                 
                                                                                
                                                    
                                                                         
                                                                                  

                                                                             




                                                                          
                                                                     

                                                       


                                                                              
                                             
                                              
                 
                             

         


































                                                                             
 


                                                                                 







                                                                                 
                                              






                                  
                                          

 

                                     





                                                                           



                                                                                                  
                                                      
                                                   













                                                                         

                                                                              

                                          

 







                                                                          
                                                            
                                                         
 





                                                                        
                
                                                              




                                   
                                                                          
                                                                
 



                                                                    

 
                                                                
                                                       
 




                                                                       

 

                                                                       
                                                              
                                                             
 
                                                    









                                                                  
                                         
                                                                                       
            
                                                                                       

                                                                  

 

                                                                     


                                              
                                               



                                                


                                                     

                                           























                                                                                  




















                                                                           
 

                                                 











                                                                     

 






                                                                             


                                                                                
 
                                                            
 

                                                       

                                                     
                                                                     

                               
                                                  
 
                                                             



                              
                                                    













                                                           






                                               

                                                                              



                                             
                                       


                            
                     
                                    
                                           


                                             
                   
              
                                                         
 




                                                                      
                                                  

                                         





                                                                               

                                                                                                        


                 
                                               

                                                                           




                             
 
                              

                                                   
                                 
                        
 











                                                                        
                                                                          
                                                                                     





                                                                    


                                                                       

                                           
                                          



                                         


                                              


                                                        
                                                               

                                                                              

                                                                           
                                                                   
                                                                            
 

                                                                                   



                                                             

                                                                              

                                                                           
                                                                          



                                                                                          





                                                       

                                                                            

                                                                                
                                      

                                             

                                                                                




                                                                                      







                                                                                





                                                                          
                                               

                                                                             









                                                                              
                                                         





                                                                            
                                                         

                                                                           

                                                      

                                                                                



                                                     
                                                         


                                                    

                              
                                                 

                                                                            









                                                                             
                                                                     



                                            




                                                                            

                                                                           
                                                                     

                                                                           
                                                                     





                                            

                                                                            
                                            


                                             

                                            





                              





                                                                 
 


                                                                               

                                                                             



                                                                                          
                                                   


                                                                       


                                                              







                                                                    

                                                                             







                                                               
                                                                                 
                                     




                                                                 
                                                                                





                                                                        


                                                                    











                                                                

                                                                             





                                                                    
 
                                                                
 
                                      

         
                                                    



                   

                                                                      



                                                           

                                                                              






                                             
                   
              
                                                         
 
                               

                                               

                                                                           





                             
                                                 

                                                   
                                                                        












                                                            














                                                                                            

                                                                  
 
                                 






                                                                            
                                                       


















                                                                
 

                                                                      



                                                           






































                                                                            
                                                      
                                                         
                                                      
 
                                          
                                                       
 
                                             
                                                            
                       
                                                                    
 
                                                    

                                                                          
            
                                                                        
                                                
 











                                                                               









                                                                                      

                    


                                   
                               






                                                          



                              


                                                      

                                                                  










                                                                                       

                                                                               
                                                                             

                                                                    


                                                                 



                                                                                
















                                                                                   


                                                      
                             




                                                           

         







                                                                     






                                                                                    


                                                                    



                                                                            


                                 

 









                                                                  
                   

                                        

                                                             





















                                                                         
                                                                 



















                                                                         


                                                                         

























                                                                          

                                                                             






















                                                                              

                                                                               










                                                                                       
                                                                      



                                                                    

                                                                             




















































                                                                                            




                                                                          



                                                                              
                                                                 



                                        

                                 
















                                                             





































































                                                                                  





































































                                                                                
                                         

































                                                                              

                                         





















                                                                              
/*
 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/log2.h>
#include <linux/etherdevice.h>
#include <net/ip.h>
#include <linux/slab.h>
#include <linux/netdevice.h>

#include <rdma/ib_cache.h>
#include <rdma/ib_pack.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_mad.h>
#include <rdma/uverbs_ioctl.h>

#include <linux/mlx4/driver.h>
#include <linux/mlx4/qp.h>

#include "mlx4_ib.h"
#include <rdma/mlx4-abi.h>

static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
			     struct mlx4_ib_cq *recv_cq);
static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
			       struct mlx4_ib_cq *recv_cq);
static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
			      struct ib_udata *udata);

enum {
	MLX4_IB_ACK_REQ_FREQ	= 8,
};

enum {
	MLX4_IB_DEFAULT_SCHED_QUEUE	= 0x83,
	MLX4_IB_DEFAULT_QP0_SCHED_QUEUE	= 0x3f,
	MLX4_IB_LINK_TYPE_IB		= 0,
	MLX4_IB_LINK_TYPE_ETH		= 1
};

enum {
	/*
	 * Largest possible UD header: send with GRH and immediate
	 * data plus 18 bytes for an Ethernet header with VLAN/802.1Q
	 * tag.  (LRH would only use 8 bytes, so Ethernet is the
	 * biggest case)
	 */
	MLX4_IB_UD_HEADER_SIZE		= 82,
	MLX4_IB_LSO_HEADER_SPARE	= 128,
};

struct mlx4_ib_sqp {
	struct mlx4_ib_qp	qp;
	int			pkey_index;
	u32			qkey;
	u32			send_psn;
	struct ib_ud_header	ud_header;
	u8			header_buf[MLX4_IB_UD_HEADER_SIZE];
	struct ib_qp		*roce_v2_gsi;
};

enum {
	MLX4_IB_MIN_SQ_STRIDE	= 6,
	MLX4_IB_CACHE_LINE_SIZE	= 64,
};

enum {
	MLX4_RAW_QP_MTU		= 7,
	MLX4_RAW_QP_MSGMAX	= 31,
};

#ifndef ETH_ALEN
#define ETH_ALEN        6
#endif

static const __be32 mlx4_ib_opcode[] = {
	[IB_WR_SEND]				= cpu_to_be32(MLX4_OPCODE_SEND),
	[IB_WR_LSO]				= cpu_to_be32(MLX4_OPCODE_LSO),
	[IB_WR_SEND_WITH_IMM]			= cpu_to_be32(MLX4_OPCODE_SEND_IMM),
	[IB_WR_RDMA_WRITE]			= cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
	[IB_WR_RDMA_WRITE_WITH_IMM]		= cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
	[IB_WR_RDMA_READ]			= cpu_to_be32(MLX4_OPCODE_RDMA_READ),
	[IB_WR_ATOMIC_CMP_AND_SWP]		= cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
	[IB_WR_ATOMIC_FETCH_AND_ADD]		= cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
	[IB_WR_SEND_WITH_INV]			= cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
	[IB_WR_LOCAL_INV]			= cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
	[IB_WR_REG_MR]				= cpu_to_be32(MLX4_OPCODE_FMR),
	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
};

enum mlx4_ib_source_type {
	MLX4_IB_QP_SRC	= 0,
	MLX4_IB_RWQ_SRC	= 1,
};

static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
{
	return container_of(mqp, struct mlx4_ib_sqp, qp);
}

static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
{
	if (!mlx4_is_master(dev->dev))
		return 0;

	return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
	       qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
		8 * MLX4_MFUNC_MAX;
}

static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
{
	int proxy_sqp = 0;
	int real_sqp = 0;
	int i;
	/* PPF or Native -- real SQP */
	real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
		    qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
		    qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
	if (real_sqp)
		return 1;
	/* VF or PF -- proxy SQP */
	if (mlx4_is_mfunc(dev->dev)) {
		for (i = 0; i < dev->dev->caps.num_ports; i++) {
			if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy ||
			    qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) {
				proxy_sqp = 1;
				break;
			}
		}
	}
	if (proxy_sqp)
		return 1;

	return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
}

/* used for INIT/CLOSE port logic */
static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
{
	int proxy_qp0 = 0;
	int real_qp0 = 0;
	int i;
	/* PPF or Native -- real QP0 */
	real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
		    qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
		    qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
	if (real_qp0)
		return 1;
	/* VF or PF -- proxy QP0 */
	if (mlx4_is_mfunc(dev->dev)) {
		for (i = 0; i < dev->dev->caps.num_ports; i++) {
			if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) {
				proxy_qp0 = 1;
				break;
			}
		}
	}
	return proxy_qp0;
}

static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
{
	return mlx4_buf_offset(&qp->buf, offset);
}

static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
{
	return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
}

static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
{
	return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
}

/*
 * Stamp a SQ WQE so that it is invalid if prefetched by marking the
 * first four bytes of every 64 byte chunk with 0xffffffff, except for
 * the very first chunk of the WQE.
 */
static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n)
{
	__be32 *wqe;
	int i;
	int s;
	void *buf;
	struct mlx4_wqe_ctrl_seg *ctrl;

	buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
	ctrl = (struct mlx4_wqe_ctrl_seg *)buf;
	s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4;
	for (i = 64; i < s; i += 64) {
		wqe = buf + i;
		*wqe = cpu_to_be32(0xffffffff);
	}
}

static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
{
	struct ib_event event;
	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;

	if (type == MLX4_EVENT_TYPE_PATH_MIG)
		to_mibqp(qp)->port = to_mibqp(qp)->alt_port;

	if (ibqp->event_handler) {
		event.device     = ibqp->device;
		event.element.qp = ibqp;
		switch (type) {
		case MLX4_EVENT_TYPE_PATH_MIG:
			event.event = IB_EVENT_PATH_MIG;
			break;
		case MLX4_EVENT_TYPE_COMM_EST:
			event.event = IB_EVENT_COMM_EST;
			break;
		case MLX4_EVENT_TYPE_SQ_DRAINED:
			event.event = IB_EVENT_SQ_DRAINED;
			break;
		case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
			event.event = IB_EVENT_QP_LAST_WQE_REACHED;
			break;
		case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
			event.event = IB_EVENT_QP_FATAL;
			break;
		case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
			event.event = IB_EVENT_PATH_MIG_ERR;
			break;
		case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
			event.event = IB_EVENT_QP_REQ_ERR;
			break;
		case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
			event.event = IB_EVENT_QP_ACCESS_ERR;
			break;
		default:
			pr_warn("Unexpected event type %d "
			       "on QP %06x\n", type, qp->qpn);
			return;
		}

		ibqp->event_handler(&event, ibqp->qp_context);
	}
}

static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
{
	pr_warn_ratelimited("Unexpected event type %d on WQ 0x%06x. Events are not supported for WQs\n",
			    type, qp->qpn);
}

static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
{
	/*
	 * UD WQEs must have a datagram segment.
	 * RC and UC WQEs might have a remote address segment.
	 * MLX WQEs need two extra inline data segments (for the UD
	 * header and space for the ICRC).
	 */
	switch (type) {
	case MLX4_IB_QPT_UD:
		return sizeof (struct mlx4_wqe_ctrl_seg) +
			sizeof (struct mlx4_wqe_datagram_seg) +
			((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
	case MLX4_IB_QPT_PROXY_SMI_OWNER:
	case MLX4_IB_QPT_PROXY_SMI:
	case MLX4_IB_QPT_PROXY_GSI:
		return sizeof (struct mlx4_wqe_ctrl_seg) +
			sizeof (struct mlx4_wqe_datagram_seg) + 64;
	case MLX4_IB_QPT_TUN_SMI_OWNER:
	case MLX4_IB_QPT_TUN_GSI:
		return sizeof (struct mlx4_wqe_ctrl_seg) +
			sizeof (struct mlx4_wqe_datagram_seg);

	case MLX4_IB_QPT_UC:
		return sizeof (struct mlx4_wqe_ctrl_seg) +
			sizeof (struct mlx4_wqe_raddr_seg);
	case MLX4_IB_QPT_RC:
		return sizeof (struct mlx4_wqe_ctrl_seg) +
			sizeof (struct mlx4_wqe_masked_atomic_seg) +
			sizeof (struct mlx4_wqe_raddr_seg);
	case MLX4_IB_QPT_SMI:
	case MLX4_IB_QPT_GSI:
		return sizeof (struct mlx4_wqe_ctrl_seg) +
			ALIGN(MLX4_IB_UD_HEADER_SIZE +
			      DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
					   MLX4_INLINE_ALIGN) *
			      sizeof (struct mlx4_wqe_inline_seg),
			      sizeof (struct mlx4_wqe_data_seg)) +
			ALIGN(4 +
			      sizeof (struct mlx4_wqe_inline_seg),
			      sizeof (struct mlx4_wqe_data_seg));
	default:
		return sizeof (struct mlx4_wqe_ctrl_seg);
	}
}

static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
		       bool is_user, int has_rq, struct mlx4_ib_qp *qp,
		       u32 inl_recv_sz)
{
	/* Sanity check RQ size before proceeding */
	if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
	    cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
		return -EINVAL;

	if (!has_rq) {
		if (cap->max_recv_wr || inl_recv_sz)
			return -EINVAL;

		qp->rq.wqe_cnt = qp->rq.max_gs = 0;
	} else {
		u32 max_inl_recv_sz = dev->dev->caps.max_rq_sg *
			sizeof(struct mlx4_wqe_data_seg);
		u32 wqe_size;

		/* HW requires >= 1 RQ entry with >= 1 gather entry */
		if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge ||
				inl_recv_sz > max_inl_recv_sz))
			return -EINVAL;

		qp->rq.wqe_cnt	 = roundup_pow_of_two(max(1U, cap->max_recv_wr));
		qp->rq.max_gs	 = roundup_pow_of_two(max(1U, cap->max_recv_sge));
		wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg);
		qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz));
	}

	/* leave userspace return values as they were, so as not to break ABI */
	if (is_user) {
		cap->max_recv_wr  = qp->rq.max_post = qp->rq.wqe_cnt;
		cap->max_recv_sge = qp->rq.max_gs;
	} else {
		cap->max_recv_wr  = qp->rq.max_post =
			min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
		cap->max_recv_sge = min(qp->rq.max_gs,
					min(dev->dev->caps.max_sq_sg,
					    dev->dev->caps.max_rq_sg));
	}

	return 0;
}

static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
			      enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp)
{
	int s;

	/* Sanity check SQ size before proceeding */
	if (cap->max_send_wr  > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
	    cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
	    cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
	    sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
		return -EINVAL;

	/*
	 * For MLX transport we need 2 extra S/G entries:
	 * one for the header and one for the checksum at the end
	 */
	if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
	     type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
	    cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
		return -EINVAL;

	s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
		cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
		send_wqe_overhead(type, qp->flags);

	if (s > dev->dev->caps.max_sq_desc_sz)
		return -EINVAL;

	qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));

	/*
	 * We need to leave 2 KB + 1 WR of headroom in the SQ to
	 * allow HW to prefetch.
	 */
	qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);
	qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr +
					    qp->sq_spare_wqes);

	qp->sq.max_gs =
		(min(dev->dev->caps.max_sq_desc_sz,
		     (1 << qp->sq.wqe_shift)) -
		 send_wqe_overhead(type, qp->flags)) /
		sizeof (struct mlx4_wqe_data_seg);

	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
		(qp->sq.wqe_cnt << qp->sq.wqe_shift);
	if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
		qp->rq.offset = 0;
		qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
	} else {
		qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
		qp->sq.offset = 0;
	}

	cap->max_send_wr  = qp->sq.max_post =
		qp->sq.wqe_cnt - qp->sq_spare_wqes;
	cap->max_send_sge = min(qp->sq.max_gs,
				min(dev->dev->caps.max_sq_sg,
				    dev->dev->caps.max_rq_sg));
	/* We don't support inline sends for kernel QPs (yet) */
	cap->max_inline_data = 0;

	return 0;
}

static int set_user_sq_size(struct mlx4_ib_dev *dev,
			    struct mlx4_ib_qp *qp,
			    struct mlx4_ib_create_qp *ucmd)
{
	/* Sanity check SQ size before proceeding */
	if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes	 ||
	    ucmd->log_sq_stride >
		ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
	    ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
		return -EINVAL;

	qp->sq.wqe_cnt   = 1 << ucmd->log_sq_bb_count;
	qp->sq.wqe_shift = ucmd->log_sq_stride;

	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
		(qp->sq.wqe_cnt << qp->sq.wqe_shift);

	return 0;
}

static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
{
	int i;

	qp->sqp_proxy_rcv =
		kmalloc_array(qp->rq.wqe_cnt, sizeof(struct mlx4_ib_buf),
			      GFP_KERNEL);
	if (!qp->sqp_proxy_rcv)
		return -ENOMEM;
	for (i = 0; i < qp->rq.wqe_cnt; i++) {
		qp->sqp_proxy_rcv[i].addr =
			kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
				GFP_KERNEL);
		if (!qp->sqp_proxy_rcv[i].addr)
			goto err;
		qp->sqp_proxy_rcv[i].map =
			ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
					  sizeof (struct mlx4_ib_proxy_sqp_hdr),
					  DMA_FROM_DEVICE);
		if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) {
			kfree(qp->sqp_proxy_rcv[i].addr);
			goto err;
		}
	}
	return 0;

err:
	while (i > 0) {
		--i;
		ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
				    sizeof (struct mlx4_ib_proxy_sqp_hdr),
				    DMA_FROM_DEVICE);
		kfree(qp->sqp_proxy_rcv[i].addr);
	}
	kfree(qp->sqp_proxy_rcv);
	qp->sqp_proxy_rcv = NULL;
	return -ENOMEM;
}

static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
{
	int i;

	for (i = 0; i < qp->rq.wqe_cnt; i++) {
		ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
				    sizeof (struct mlx4_ib_proxy_sqp_hdr),
				    DMA_FROM_DEVICE);
		kfree(qp->sqp_proxy_rcv[i].addr);
	}
	kfree(qp->sqp_proxy_rcv);
}

static int qp_has_rq(struct ib_qp_init_attr *attr)
{
	if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
		return 0;

	return !attr->srq;
}

static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
{
	int i;
	for (i = 0; i < dev->caps.num_ports; i++) {
		if (qpn == dev->caps.spec_qps[i].qp0_proxy)
			return !!dev->caps.spec_qps[i].qp0_qkey;
	}
	return 0;
}

static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
				    struct mlx4_ib_qp *qp)
{
	mutex_lock(&dev->counters_table[qp->port - 1].mutex);
	mlx4_counter_free(dev->dev, qp->counter_index->index);
	list_del(&qp->counter_index->list);
	mutex_unlock(&dev->counters_table[qp->port - 1].mutex);

	kfree(qp->counter_index);
	qp->counter_index = NULL;
}

static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
		      struct ib_qp_init_attr *init_attr,
		      struct mlx4_ib_create_qp_rss *ucmd)
{
	rss_ctx->base_qpn_tbl_sz = init_attr->rwq_ind_tbl->ind_tbl[0]->wq_num |
		(init_attr->rwq_ind_tbl->log_ind_tbl_size << 24);

	if ((ucmd->rx_hash_function == MLX4_IB_RX_HASH_FUNC_TOEPLITZ) &&
	    (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP)) {
		memcpy(rss_ctx->rss_key, ucmd->rx_hash_key,
		       MLX4_EN_RSS_KEY_SIZE);
	} else {
		pr_debug("RX Hash function is not supported\n");
		return (-EOPNOTSUPP);
	}

	if (ucmd->rx_hash_fields_mask & ~(MLX4_IB_RX_HASH_SRC_IPV4	|
					  MLX4_IB_RX_HASH_DST_IPV4	|
					  MLX4_IB_RX_HASH_SRC_IPV6	|
					  MLX4_IB_RX_HASH_DST_IPV6	|
					  MLX4_IB_RX_HASH_SRC_PORT_TCP	|
					  MLX4_IB_RX_HASH_DST_PORT_TCP	|
					  MLX4_IB_RX_HASH_SRC_PORT_UDP	|
					  MLX4_IB_RX_HASH_DST_PORT_UDP  |
					  MLX4_IB_RX_HASH_INNER)) {
		pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",
			 ucmd->rx_hash_fields_mask);
		return (-EOPNOTSUPP);
	}

	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) &&
	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
		rss_ctx->flags = MLX4_RSS_IPV4;
	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) ||
		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
		pr_debug("RX Hash fields_mask is not supported - both IPv4 SRC and DST must be set\n");
		return (-EOPNOTSUPP);
	}

	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) &&
	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
		rss_ctx->flags |= MLX4_RSS_IPV6;
	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) ||
		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
		pr_debug("RX Hash fields_mask is not supported - both IPv6 SRC and DST must be set\n");
		return (-EOPNOTSUPP);
	}

	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) &&
	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
		if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) {
			pr_debug("RX Hash fields_mask for UDP is not supported\n");
			return (-EOPNOTSUPP);
		}

		if (rss_ctx->flags & MLX4_RSS_IPV4)
			rss_ctx->flags |= MLX4_RSS_UDP_IPV4;
		if (rss_ctx->flags & MLX4_RSS_IPV6)
			rss_ctx->flags |= MLX4_RSS_UDP_IPV6;
		if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
			pr_debug("RX Hash fields_mask is not supported - UDP must be set with IPv4 or IPv6\n");
			return (-EOPNOTSUPP);
		}
	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) ||
		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
		pr_debug("RX Hash fields_mask is not supported - both UDP SRC and DST must be set\n");
		return (-EOPNOTSUPP);
	}

	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) &&
	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
		if (rss_ctx->flags & MLX4_RSS_IPV4)
			rss_ctx->flags |= MLX4_RSS_TCP_IPV4;
		if (rss_ctx->flags & MLX4_RSS_IPV6)
			rss_ctx->flags |= MLX4_RSS_TCP_IPV6;
		if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
			pr_debug("RX Hash fields_mask is not supported - TCP must be set with IPv4 or IPv6\n");
			return (-EOPNOTSUPP);
		}
	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) ||
		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
		pr_debug("RX Hash fields_mask is not supported - both TCP SRC and DST must be set\n");
		return (-EOPNOTSUPP);
	}

	if (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_INNER) {
		if (dev->dev->caps.tunnel_offload_mode ==
		    MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
			/*
			 * Hash according to inner headers if exist, otherwise
			 * according to outer headers.
			 */
			rss_ctx->flags |= MLX4_RSS_BY_INNER_HEADERS_IPONLY;
		} else {
			pr_debug("RSS Hash for inner headers isn't supported\n");
			return (-EOPNOTSUPP);
		}
	}

	return 0;
}

static int create_qp_rss(struct mlx4_ib_dev *dev,
			 struct ib_qp_init_attr *init_attr,
			 struct mlx4_ib_create_qp_rss *ucmd,
			 struct mlx4_ib_qp *qp)
{
	int qpn;
	int err;

	qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;

	err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage);
	if (err)
		return err;

	err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
	if (err)
		goto err_qpn;

	mutex_init(&qp->mutex);

	INIT_LIST_HEAD(&qp->gid_list);
	INIT_LIST_HEAD(&qp->steering_rules);

	qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
	qp->state = IB_QPS_RESET;

	/* Set dummy send resources to be compatible with HV and PRM */
	qp->sq_no_prefetch = 1;
	qp->sq.wqe_cnt = 1;
	qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
	qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE;
	qp->mtt = (to_mqp(
		   (struct ib_qp *)init_attr->rwq_ind_tbl->ind_tbl[0]))->mtt;

	qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL);
	if (!qp->rss_ctx) {
		err = -ENOMEM;
		goto err_qp_alloc;
	}

	err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd);
	if (err)
		goto err;

	return 0;

err:
	kfree(qp->rss_ctx);

err_qp_alloc:
	mlx4_qp_remove(dev->dev, &qp->mqp);
	mlx4_qp_free(dev->dev, &qp->mqp);

err_qpn:
	mlx4_qp_release_range(dev->dev, qpn, 1);
	return err;
}

static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,
					    struct ib_qp_init_attr *init_attr,
					    struct ib_udata *udata)
{
	struct mlx4_ib_qp *qp;
	struct mlx4_ib_create_qp_rss ucmd = {};
	size_t required_cmd_sz;
	int err;

	if (!udata) {
		pr_debug("RSS QP with NULL udata\n");
		return ERR_PTR(-EINVAL);
	}

	if (udata->outlen)
		return ERR_PTR(-EOPNOTSUPP);

	required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
					sizeof(ucmd.reserved1);
	if (udata->inlen < required_cmd_sz) {
		pr_debug("invalid inlen\n");
		return ERR_PTR(-EINVAL);
	}

	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
		pr_debug("copy failed\n");
		return ERR_PTR(-EFAULT);
	}

	if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
		return ERR_PTR(-EOPNOTSUPP);

	if (ucmd.comp_mask || ucmd.reserved1)
		return ERR_PTR(-EOPNOTSUPP);

	if (udata->inlen > sizeof(ucmd) &&
	    !ib_is_udata_cleared(udata, sizeof(ucmd),
				 udata->inlen - sizeof(ucmd))) {
		pr_debug("inlen is not supported\n");
		return ERR_PTR(-EOPNOTSUPP);
	}

	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
		pr_debug("RSS QP with unsupported QP type %d\n",
			 init_attr->qp_type);
		return ERR_PTR(-EOPNOTSUPP);
	}

	if (init_attr->create_flags) {
		pr_debug("RSS QP doesn't support create flags\n");
		return ERR_PTR(-EOPNOTSUPP);
	}

	if (init_attr->send_cq || init_attr->cap.max_send_wr) {
		pr_debug("RSS QP with unsupported send attributes\n");
		return ERR_PTR(-EOPNOTSUPP);
	}

	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
	if (!qp)
		return ERR_PTR(-ENOMEM);

	qp->pri.vid = 0xFFFF;
	qp->alt.vid = 0xFFFF;

	err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
	if (err) {
		kfree(qp);
		return ERR_PTR(err);
	}

	qp->ibqp.qp_num = qp->mqp.qpn;

	return &qp->ibqp;
}

/*
 * This function allocates a WQN from a range which is consecutive and aligned
 * to its size. In case the range is full, then it creates a new range and
 * allocates WQN from it. The new range will be used for following allocations.
 */
static int mlx4_ib_alloc_wqn(struct mlx4_ib_ucontext *context,
			     struct mlx4_ib_qp *qp, int range_size, int *wqn)
{
	struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
	struct mlx4_wqn_range *range;
	int err = 0;

	mutex_lock(&context->wqn_ranges_mutex);

	range = list_first_entry_or_null(&context->wqn_ranges_list,
					 struct mlx4_wqn_range, list);

	if (!range || (range->refcount == range->size) || range->dirty) {
		range = kzalloc(sizeof(*range), GFP_KERNEL);
		if (!range) {
			err = -ENOMEM;
			goto out;
		}

		err = mlx4_qp_reserve_range(dev->dev, range_size,
					    range_size, &range->base_wqn, 0,
					    qp->mqp.usage);
		if (err) {
			kfree(range);
			goto out;
		}

		range->size = range_size;
		list_add(&range->list, &context->wqn_ranges_list);
	} else if (range_size != 1) {
		/*
		 * Requesting a new range (>1) when last range is still open, is
		 * not valid.
		 */
		err = -EINVAL;
		goto out;
	}

	qp->wqn_range = range;

	*wqn = range->base_wqn + range->refcount;

	range->refcount++;

out:
	mutex_unlock(&context->wqn_ranges_mutex);

	return err;
}

static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,
				struct mlx4_ib_qp *qp, bool dirty_release)
{
	struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
	struct mlx4_wqn_range *range;

	mutex_lock(&context->wqn_ranges_mutex);

	range = qp->wqn_range;

	range->refcount--;
	if (!range->refcount) {
		mlx4_qp_release_range(dev->dev, range->base_wqn,
				      range->size);
		list_del(&range->list);
		kfree(range);
	} else if (dirty_release) {
	/*
	 * A range which one of its WQNs is destroyed, won't be able to be
	 * reused for further WQN allocations.
	 * The next created WQ will allocate a new range.
	 */
		range->dirty = 1;
	}

	mutex_unlock(&context->wqn_ranges_mutex);
}

static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
			    enum mlx4_ib_source_type src,
			    struct ib_qp_init_attr *init_attr,
			    struct ib_udata *udata, int sqpn,
			    struct mlx4_ib_qp **caller_qp)
{
	int qpn;
	int err;
	struct mlx4_ib_sqp *sqp = NULL;
	struct mlx4_ib_qp *qp;
	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
		udata, struct mlx4_ib_ucontext, ibucontext);
	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
	struct mlx4_ib_cq *mcq;
	unsigned long flags;
	int range_size = 0;

	/* When tunneling special qps, we use a plain UD qp */
	if (sqpn) {
		if (mlx4_is_mfunc(dev->dev) &&
		    (!mlx4_is_master(dev->dev) ||
		     !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
			if (init_attr->qp_type == IB_QPT_GSI)
				qp_type = MLX4_IB_QPT_PROXY_GSI;
			else {
				if (mlx4_is_master(dev->dev) ||
				    qp0_enabled_vf(dev->dev, sqpn))
					qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
				else
					qp_type = MLX4_IB_QPT_PROXY_SMI;
			}
		}
		qpn = sqpn;
		/* add extra sg entry for tunneling */
		init_attr->cap.max_recv_sge++;
	} else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
		struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
			container_of(init_attr,
				     struct mlx4_ib_qp_tunnel_init_attr, init_attr);
		if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
		     tnl_init->proxy_qp_type != IB_QPT_GSI)   ||
		    !mlx4_is_master(dev->dev))
			return -EINVAL;
		if (tnl_init->proxy_qp_type == IB_QPT_GSI)
			qp_type = MLX4_IB_QPT_TUN_GSI;
		else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
			 mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
					     tnl_init->port))
			qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
		else
			qp_type = MLX4_IB_QPT_TUN_SMI;
		/* we are definitely in the PPF here, since we are creating
		 * tunnel QPs. base_tunnel_sqpn is therefore valid. */
		qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
			+ tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
		sqpn = qpn;
	}

	if (!*caller_qp) {
		if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||
		    (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
				MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
			sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
			if (!sqp)
				return -ENOMEM;
			qp = &sqp->qp;
			qp->pri.vid = 0xFFFF;
			qp->alt.vid = 0xFFFF;
		} else {
			qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);
			if (!qp)
				return -ENOMEM;
			qp->pri.vid = 0xFFFF;
			qp->alt.vid = 0xFFFF;
		}
	} else
		qp = *caller_qp;

	qp->mlx4_ib_qp_type = qp_type;

	mutex_init(&qp->mutex);
	spin_lock_init(&qp->sq.lock);
	spin_lock_init(&qp->rq.lock);
	INIT_LIST_HEAD(&qp->gid_list);
	INIT_LIST_HEAD(&qp->steering_rules);

	qp->state	 = IB_QPS_RESET;
	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
		qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);


	if (udata) {
		union {
			struct mlx4_ib_create_qp qp;
			struct mlx4_ib_create_wq wq;
		} ucmd;
		size_t copy_len;
		int shift;
		int n;

		copy_len = (src == MLX4_IB_QP_SRC) ?
			   sizeof(struct mlx4_ib_create_qp) :
			   min(sizeof(struct mlx4_ib_create_wq), udata->inlen);

		if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
			err = -EFAULT;
			goto err;
		}

		if (src == MLX4_IB_RWQ_SRC) {
			if (ucmd.wq.comp_mask || ucmd.wq.reserved[0] ||
			    ucmd.wq.reserved[1] || ucmd.wq.reserved[2]) {
				pr_debug("user command isn't supported\n");
				err = -EOPNOTSUPP;
				goto err;
			}

			if (ucmd.wq.log_range_size >
			    ilog2(dev->dev->caps.max_rss_tbl_sz)) {
				pr_debug("WQN range size must be equal or smaller than %d\n",
					 dev->dev->caps.max_rss_tbl_sz);
				err = -EOPNOTSUPP;
				goto err;
			}
			range_size = 1 << ucmd.wq.log_range_size;
		} else {
			qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
		}

		if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
			if (!(dev->dev->caps.flags &
			      MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
				pr_debug("scatter FCS is unsupported\n");
				err = -EOPNOTSUPP;
				goto err;
			}

			qp->flags |= MLX4_IB_QP_SCATTER_FCS;
		}

		err = set_rq_size(dev, &init_attr->cap, udata,
				  qp_has_rq(init_attr), qp, qp->inl_recv_sz);
		if (err)
			goto err;

		if (src == MLX4_IB_QP_SRC) {
			qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;

			err = set_user_sq_size(dev, qp,
					       (struct mlx4_ib_create_qp *)
					       &ucmd);
			if (err)
				goto err;
		} else {
			qp->sq_no_prefetch = 1;
			qp->sq.wqe_cnt = 1;
			qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
			/* Allocated buffer expects to have at least that SQ
			 * size.
			 */
			qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
				(qp->sq.wqe_cnt << qp->sq.wqe_shift);
		}

		qp->umem =
			ib_umem_get(udata,
				    (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
							      ucmd.wq.buf_addr,
				    qp->buf_size, 0, 0);
		if (IS_ERR(qp->umem)) {
			err = PTR_ERR(qp->umem);
			goto err;
		}

		n = ib_umem_page_count(qp->umem);
		shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
		err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);

		if (err)
			goto err_buf;

		err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
		if (err)
			goto err_mtt;

		if (qp_has_rq(init_attr)) {
			err = mlx4_ib_db_map_user(
				context, udata,
				(src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
							  ucmd.wq.db_addr,
				&qp->db);
			if (err)
				goto err_mtt;
		}
		qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
	} else {
		err = set_rq_size(dev, &init_attr->cap, udata,
				  qp_has_rq(init_attr), qp, 0);
		if (err)
			goto err;

		qp->sq_no_prefetch = 0;

		if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
			qp->flags |= MLX4_IB_QP_LSO;

		if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
			if (dev->steering_support ==
			    MLX4_STEERING_MODE_DEVICE_MANAGED)
				qp->flags |= MLX4_IB_QP_NETIF;
			else
				goto err;
		}

		err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp);
		if (err)
			goto err;

		if (qp_has_rq(init_attr)) {
			err = mlx4_db_alloc(dev->dev, &qp->db, 0);
			if (err)
				goto err;

			*qp->db.db = 0;
		}

		if (mlx4_buf_alloc(dev->dev, qp->buf_size,  PAGE_SIZE * 2,
				   &qp->buf)) {
			err = -ENOMEM;
			goto err_db;
		}

		err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
				    &qp->mtt);
		if (err)
			goto err_buf;

		err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
		if (err)
			goto err_mtt;

		qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
					     sizeof(u64), GFP_KERNEL);
		qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
					     sizeof(u64), GFP_KERNEL);
		if (!qp->sq.wrid || !qp->rq.wrid) {
			err = -ENOMEM;
			goto err_wrid;
		}
		qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
	}

	if (sqpn) {
		if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
		    MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
			if (alloc_proxy_bufs(pd->device, qp)) {
				err = -ENOMEM;
				goto err_wrid;
			}
		}
	} else if (src == MLX4_IB_RWQ_SRC) {
		err = mlx4_ib_alloc_wqn(context, qp, range_size, &qpn);
		if (err)
			goto err_wrid;
	} else {
		/* Raw packet QPNs may not have bits 6,7 set in their qp_num;
		 * otherwise, the WQE BlueFlame setup flow wrongly causes
		 * VLAN insertion. */
		if (init_attr->qp_type == IB_QPT_RAW_PACKET)
			err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
						    (init_attr->cap.max_send_wr ?
						     MLX4_RESERVE_ETH_BF_QP : 0) |
						    (init_attr->cap.max_recv_wr ?
						     MLX4_RESERVE_A0_QP : 0),
						    qp->mqp.usage);
		else
			if (qp->flags & MLX4_IB_QP_NETIF)
				err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
			else
				err = mlx4_qp_reserve_range(dev->dev, 1, 1,
							    &qpn, 0, qp->mqp.usage);
		if (err)
			goto err_proxy;
	}

	if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
		qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;

	err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
	if (err)
		goto err_qpn;

	if (init_attr->qp_type == IB_QPT_XRC_TGT)
		qp->mqp.qpn |= (1 << 23);

	/*
	 * Hardware wants QPN written in big-endian order (after
	 * shifting) for send doorbell.  Precompute this value to save
	 * a little bit when posting sends.
	 */
	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);

	qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
						  mlx4_ib_wq_event;

	if (!*caller_qp)
		*caller_qp = qp;

	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
	mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
			 to_mcq(init_attr->recv_cq));
	/* Maintain device to QPs access, needed for further handling
	 * via reset flow
	 */
	list_add_tail(&qp->qps_list, &dev->qp_list);
	/* Maintain CQ to QPs access, needed for further handling
	 * via reset flow
	 */
	mcq = to_mcq(init_attr->send_cq);
	list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
	mcq = to_mcq(init_attr->recv_cq);
	list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
	mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
			   to_mcq(init_attr->recv_cq));
	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
	return 0;

err_qpn:
	if (!sqpn) {
		if (qp->flags & MLX4_IB_QP_NETIF)
			mlx4_ib_steer_qp_free(dev, qpn, 1);
		else if (src == MLX4_IB_RWQ_SRC)
			mlx4_ib_release_wqn(context, qp, 0);
		else
			mlx4_qp_release_range(dev->dev, qpn, 1);
	}
err_proxy:
	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
		free_proxy_bufs(pd->device, qp);
err_wrid:
	if (udata) {
		if (qp_has_rq(init_attr))
			mlx4_ib_db_unmap_user(context, &qp->db);
	} else {
		kvfree(qp->sq.wrid);
		kvfree(qp->rq.wrid);
	}

err_mtt:
	mlx4_mtt_cleanup(dev->dev, &qp->mtt);

err_buf:
	if (qp->umem)
		ib_umem_release(qp->umem);
	else
		mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);

err_db:
	if (!udata && qp_has_rq(init_attr))
		mlx4_db_free(dev->dev, &qp->db);

err:
	if (!sqp && !*caller_qp)
		kfree(qp);
	kfree(sqp);

	return err;
}

static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
{
	switch (state) {
	case IB_QPS_RESET:	return MLX4_QP_STATE_RST;
	case IB_QPS_INIT:	return MLX4_QP_STATE_INIT;
	case IB_QPS_RTR:	return MLX4_QP_STATE_RTR;
	case IB_QPS_RTS:	return MLX4_QP_STATE_RTS;
	case IB_QPS_SQD:	return MLX4_QP_STATE_SQD;
	case IB_QPS_SQE:	return MLX4_QP_STATE_SQER;
	case IB_QPS_ERR:	return MLX4_QP_STATE_ERR;
	default:		return -1;
	}
}

static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
{
	if (send_cq == recv_cq) {
		spin_lock(&send_cq->lock);
		__acquire(&recv_cq->lock);
	} else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
		spin_lock(&send_cq->lock);
		spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
	} else {
		spin_lock(&recv_cq->lock);
		spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
	}
}

static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
	__releases(&send_cq->lock) __releases(&recv_cq->lock)
{
	if (send_cq == recv_cq) {
		__release(&recv_cq->lock);
		spin_unlock(&send_cq->lock);
	} else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
		spin_unlock(&recv_cq->lock);
		spin_unlock(&send_cq->lock);
	} else {
		spin_unlock(&send_cq->lock);
		spin_unlock(&recv_cq->lock);
	}
}

static void del_gid_entries(struct mlx4_ib_qp *qp)
{
	struct mlx4_ib_gid_entry *ge, *tmp;

	list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
		list_del(&ge->list);
		kfree(ge);
	}
}

static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
{
	if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
		return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
	else
		return to_mpd(qp->ibqp.pd);
}

static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src,
		    struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
{
	switch (qp->ibqp.qp_type) {
	case IB_QPT_XRC_TGT:
		*send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
		*recv_cq = *send_cq;
		break;
	case IB_QPT_XRC_INI:
		*send_cq = to_mcq(qp->ibqp.send_cq);
		*recv_cq = *send_cq;
		break;
	default:
		*recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) :
						     to_mcq(qp->ibwq.cq);
		*send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) :
						     *recv_cq;
		break;
	}
}

static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
{
	if (qp->state != IB_QPS_RESET) {
		int i;

		for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size);
		     i++) {
			struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i];
			struct mlx4_ib_qp *wq =	to_mqp((struct ib_qp *)ibwq);

			mutex_lock(&wq->mutex);

			wq->rss_usecnt--;

			mutex_unlock(&wq->mutex);
		}

		if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
				   MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
			pr_warn("modify QP %06x to RESET failed.\n",
				qp->mqp.qpn);
	}

	mlx4_qp_remove(dev->dev, &qp->mqp);
	mlx4_qp_free(dev->dev, &qp->mqp);
	mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
	del_gid_entries(qp);
	kfree(qp->rss_ctx);
}

static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
			      enum mlx4_ib_source_type src, bool is_user)
{
	struct mlx4_ib_cq *send_cq, *recv_cq;
	unsigned long flags;

	if (qp->state != IB_QPS_RESET) {
		if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
				   MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
			pr_warn("modify QP %06x to RESET failed.\n",
			       qp->mqp.qpn);
		if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
			mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
			qp->pri.smac = 0;
			qp->pri.smac_port = 0;
		}
		if (qp->alt.smac) {
			mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
			qp->alt.smac = 0;
		}
		if (qp->pri.vid < 0x1000) {
			mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
			qp->pri.vid = 0xFFFF;
			qp->pri.candidate_vid = 0xFFFF;
			qp->pri.update_vid = 0;
		}
		if (qp->alt.vid < 0x1000) {
			mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
			qp->alt.vid = 0xFFFF;
			qp->alt.candidate_vid = 0xFFFF;
			qp->alt.update_vid = 0;
		}
	}

	get_cqs(qp, src, &send_cq, &recv_cq);

	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
	mlx4_ib_lock_cqs(send_cq, recv_cq);

	/* del from lists under both locks above to protect reset flow paths */
	list_del(&qp->qps_list);
	list_del(&qp->cq_send_list);
	list_del(&qp->cq_recv_list);
	if (!is_user) {
		__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
				 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
		if (send_cq != recv_cq)
			__mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
	}

	mlx4_qp_remove(dev->dev, &qp->mqp);

	mlx4_ib_unlock_cqs(send_cq, recv_cq);
	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);

	mlx4_qp_free(dev->dev, &qp->mqp);

	if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
		if (qp->flags & MLX4_IB_QP_NETIF)
			mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
		else if (src == MLX4_IB_RWQ_SRC)
			mlx4_ib_release_wqn(to_mucontext(
					    qp->ibwq.uobject->context), qp, 1);
		else
			mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
	}

	mlx4_mtt_cleanup(dev->dev, &qp->mtt);

	if (is_user) {
		if (qp->rq.wqe_cnt) {
			struct mlx4_ib_ucontext *mcontext = !src ?
				to_mucontext(qp->ibqp.uobject->context) :
				to_mucontext(qp->ibwq.uobject->context);
			mlx4_ib_db_unmap_user(mcontext, &qp->db);
		}
		ib_umem_release(qp->umem);
	} else {
		kvfree(qp->sq.wrid);
		kvfree(qp->rq.wrid);
		if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
		    MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
			free_proxy_bufs(&dev->ib_dev, qp);
		mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
		if (qp->rq.wqe_cnt)
			mlx4_db_free(dev->dev, &qp->db);
	}

	del_gid_entries(qp);
}

static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
{
	/* Native or PPF */
	if (!mlx4_is_mfunc(dev->dev) ||
	    (mlx4_is_master(dev->dev) &&
	     attr->create_flags & MLX4_IB_SRIOV_SQP)) {
		return  dev->dev->phys_caps.base_sqpn +
			(attr->qp_type == IB_QPT_SMI ? 0 : 2) +
			attr->port_num - 1;
	}
	/* PF or VF -- creating proxies */
	if (attr->qp_type == IB_QPT_SMI)
		return dev->dev->caps.spec_qps[attr->port_num - 1].qp0_proxy;
	else
		return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
}

static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
					struct ib_qp_init_attr *init_attr,
					struct ib_udata *udata)
{
	struct mlx4_ib_qp *qp = NULL;
	int err;
	int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
	u16 xrcdn = 0;

	if (init_attr->rwq_ind_tbl)
		return _mlx4_ib_create_qp_rss(pd, init_attr, udata);

	/*
	 * We only support LSO, vendor flag1, and multicast loopback blocking,
	 * and only for kernel UD QPs.
	 */
	if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
					MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
					MLX4_IB_SRIOV_TUNNEL_QP |
					MLX4_IB_SRIOV_SQP |
					MLX4_IB_QP_NETIF |
					MLX4_IB_QP_CREATE_ROCE_V2_GSI))
		return ERR_PTR(-EINVAL);

	if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
		if (init_attr->qp_type != IB_QPT_UD)
			return ERR_PTR(-EINVAL);
	}

	if (init_attr->create_flags) {
		if (udata && init_attr->create_flags & ~(sup_u_create_flags))
			return ERR_PTR(-EINVAL);

		if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
						 MLX4_IB_QP_CREATE_ROCE_V2_GSI  |
						 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
		     init_attr->qp_type != IB_QPT_UD) ||
		    (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
		     init_attr->qp_type > IB_QPT_GSI) ||
		    (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
		     init_attr->qp_type != IB_QPT_GSI))
			return ERR_PTR(-EINVAL);
	}

	switch (init_attr->qp_type) {
	case IB_QPT_XRC_TGT:
		pd = to_mxrcd(init_attr->xrcd)->pd;
		xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
		init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
		/* fall through */
	case IB_QPT_XRC_INI:
		if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
			return ERR_PTR(-ENOSYS);
		init_attr->recv_cq = init_attr->send_cq;
		/* fall through */
	case IB_QPT_RC:
	case IB_QPT_UC:
	case IB_QPT_RAW_PACKET:
		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
		if (!qp)
			return ERR_PTR(-ENOMEM);
		qp->pri.vid = 0xFFFF;
		qp->alt.vid = 0xFFFF;
		/* fall through */
	case IB_QPT_UD:
	{
		err = create_qp_common(to_mdev(pd->device), pd,	MLX4_IB_QP_SRC,
				       init_attr, udata, 0, &qp);
		if (err) {
			kfree(qp);
			return ERR_PTR(err);
		}

		qp->ibqp.qp_num = qp->mqp.qpn;
		qp->xrcdn = xrcdn;

		break;
	}
	case IB_QPT_SMI:
	case IB_QPT_GSI:
	{
		int sqpn;

		/* Userspace is not allowed to create special QPs: */
		if (udata)
			return ERR_PTR(-EINVAL);
		if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
			int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
							1, 1, &sqpn, 0,
							MLX4_RES_USAGE_DRIVER);

			if (res)
				return ERR_PTR(res);
		} else {
			sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
		}

		err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
				       init_attr, udata, sqpn, &qp);
		if (err)
			return ERR_PTR(err);

		qp->port	= init_attr->port_num;
		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
			init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
		break;
	}
	default:
		/* Don't support raw QPs */
		return ERR_PTR(-EINVAL);
	}

	return &qp->ibqp;
}

struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
				struct ib_qp_init_attr *init_attr,
				struct ib_udata *udata) {
	struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
	struct ib_qp *ibqp;
	struct mlx4_ib_dev *dev = to_mdev(device);

	ibqp = _mlx4_ib_create_qp(pd