diff options
Diffstat (limited to 'drivers/net/ethernet/intel/igc')
23 files changed, 5593 insertions, 1906 deletions
diff --git a/drivers/net/ethernet/intel/igc/Makefile b/drivers/net/ethernet/intel/igc/Makefile index 49fb1e1965cd..95d1e8c490a4 100644 --- a/drivers/net/ethernet/intel/igc/Makefile +++ b/drivers/net/ethernet/intel/igc/Makefile @@ -8,4 +8,4 @@ obj-$(CONFIG_IGC) += igc.o igc-objs := igc_main.o igc_mac.o igc_i225.o igc_base.o igc_nvm.o igc_phy.o \ -igc_ethtool.o igc_ptp.o +igc_diag.o igc_ethtool.o igc_ptp.o igc_dump.o igc_tsn.o igc_xdp.o diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 52066bdbbad0..1e7e7071f64d 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -16,14 +16,243 @@ #include "igc_hw.h" -/* forward declaration */ -void igc_set_ethtool_ops(struct net_device *); +void igc_ethtool_set_ops(struct net_device *); -struct igc_adapter; -struct igc_ring; +/* Transmit and receive queues */ +#define IGC_MAX_RX_QUEUES 4 +#define IGC_MAX_TX_QUEUES 4 + +#define MAX_Q_VECTORS 8 +#define MAX_STD_JUMBO_FRAME_SIZE 9216 + +#define MAX_ETYPE_FILTER 8 +#define IGC_RETA_SIZE 128 + +/* SDP support */ +#define IGC_N_EXTTS 2 +#define IGC_N_PEROUT 2 +#define IGC_N_SDP 4 + +#define MAX_FLEX_FILTER 32 + +enum igc_mac_filter_type { + IGC_MAC_FILTER_TYPE_DST = 0, + IGC_MAC_FILTER_TYPE_SRC +}; + +struct igc_tx_queue_stats { + u64 packets; + u64 bytes; + u64 restart_queue; + u64 restart_queue2; +}; + +struct igc_rx_queue_stats { + u64 packets; + u64 bytes; + u64 drops; + u64 csum_err; + u64 alloc_failed; +}; + +struct igc_rx_packet_stats { + u64 ipv4_packets; /* IPv4 headers processed */ + u64 ipv4e_packets; /* IPv4E headers with extensions processed */ + u64 ipv6_packets; /* IPv6 headers processed */ + u64 ipv6e_packets; /* IPv6E headers with extensions processed */ + u64 tcp_packets; /* TCP headers processed */ + u64 udp_packets; /* UDP headers processed */ + u64 sctp_packets; /* SCTP headers processed */ + u64 nfs_packets; /* NFS headers processe */ + u64 other_packets; +}; + +struct igc_ring_container { + struct igc_ring *ring; /* pointer to linked list of rings */ + unsigned int total_bytes; /* total bytes processed this int */ + unsigned int total_packets; /* total packets processed this int */ + u16 work_limit; /* total work allowed per interrupt */ + u8 count; /* total number of rings in vector */ + u8 itr; /* current ITR setting for ring */ +}; + +struct igc_ring { + struct igc_q_vector *q_vector; /* backlink to q_vector */ + struct net_device *netdev; /* back pointer to net_device */ + struct device *dev; /* device for dma mapping */ + union { /* array of buffer info structs */ + struct igc_tx_buffer *tx_buffer_info; + struct igc_rx_buffer *rx_buffer_info; + }; + void *desc; /* descriptor ring memory */ + unsigned long flags; /* ring specific flags */ + void __iomem *tail; /* pointer to ring tail register */ + dma_addr_t dma; /* phys address of the ring */ + unsigned int size; /* length of desc. ring in bytes */ + + u16 count; /* number of desc. in the ring */ + u8 queue_index; /* logical index of the ring*/ + u8 reg_idx; /* physical index of the ring */ + bool launchtime_enable; /* true if LaunchTime is enabled */ + + u32 start_time; + u32 end_time; + + /* CBS parameters */ + bool cbs_enable; /* indicates if CBS is enabled */ + s32 idleslope; /* idleSlope in kbps */ + s32 sendslope; /* sendSlope in kbps */ + s32 hicredit; /* hiCredit in bytes */ + s32 locredit; /* loCredit in bytes */ + + /* everything past this point are written often */ + u16 next_to_clean; + u16 next_to_use; + u16 next_to_alloc; + + union { + /* TX */ + struct { + struct igc_tx_queue_stats tx_stats; + struct u64_stats_sync tx_syncp; + struct u64_stats_sync tx_syncp2; + }; + /* RX */ + struct { + struct igc_rx_queue_stats rx_stats; + struct igc_rx_packet_stats pkt_stats; + struct u64_stats_sync rx_syncp; + struct sk_buff *skb; + }; + }; + + struct xdp_rxq_info xdp_rxq; + struct xsk_buff_pool *xsk_pool; +} ____cacheline_internodealigned_in_smp; + +/* Board specific private data structure */ +struct igc_adapter { + struct net_device *netdev; + + struct ethtool_eee eee; + u16 eee_advert; + + unsigned long state; + unsigned int flags; + unsigned int num_q_vectors; + + struct msix_entry *msix_entries; + + /* TX */ + u16 tx_work_limit; + u32 tx_timeout_count; + int num_tx_queues; + struct igc_ring *tx_ring[IGC_MAX_TX_QUEUES]; + + /* RX */ + int num_rx_queues; + struct igc_ring *rx_ring[IGC_MAX_RX_QUEUES]; + + struct timer_list watchdog_timer; + struct timer_list dma_err_timer; + struct timer_list phy_info_timer; + + u32 wol; + u32 en_mng_pt; + u16 link_speed; + u16 link_duplex; + + u8 port_num; + + u8 __iomem *io_addr; + /* Interrupt Throttle Rate */ + u32 rx_itr_setting; + u32 tx_itr_setting; + + struct work_struct reset_task; + struct work_struct watchdog_task; + struct work_struct dma_err_task; + bool fc_autoneg; + + u8 tx_timeout_factor; + + int msg_enable; + u32 max_frame_size; + u32 min_frame_size; + + ktime_t base_time; + ktime_t cycle_time; + + /* OS defined structs */ + struct pci_dev *pdev; + /* lock for statistics */ + spinlock_t stats64_lock; + struct rtnl_link_stats64 stats64; + + /* structs defined in igc_hw.h */ + struct igc_hw hw; + struct igc_hw_stats stats; + + struct igc_q_vector *q_vector[MAX_Q_VECTORS]; + u32 eims_enable_mask; + u32 eims_other; + + u16 tx_ring_count; + u16 rx_ring_count; + + u32 tx_hwtstamp_timeouts; + u32 tx_hwtstamp_skipped; + u32 rx_hwtstamp_cleared; + + u32 rss_queues; + u32 rss_indir_tbl_init; + + /* Any access to elements in nfc_rule_list is protected by the + * nfc_rule_lock. + */ + struct mutex nfc_rule_lock; + struct list_head nfc_rule_list; + unsigned int nfc_rule_count; + + u8 rss_indir_tbl[IGC_RETA_SIZE]; + + unsigned long link_check_timeout; + struct igc_info ei; + + u32 test_icr; + + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_caps; + struct work_struct ptp_tx_work; + struct sk_buff *ptp_tx_skb; + struct hwtstamp_config tstamp_config; + unsigned long ptp_tx_start; + unsigned int ptp_flags; + /* System time value lock */ + spinlock_t tmreg_lock; + struct cyclecounter cc; + struct timecounter tc; + struct timespec64 prev_ptp_time; /* Pre-reset PTP clock */ + ktime_t ptp_reset_start; /* Reset time in clock mono */ + struct system_time_snapshot snapshot; + + char fw_version[32]; + + struct bpf_prog *xdp_prog; + + bool pps_sys_wrap_on; + + struct ptp_pin_desc sdp_config[IGC_N_SDP]; + struct { + struct timespec64 start; + struct timespec64 period; + } perout[IGC_N_PEROUT]; +}; void igc_up(struct igc_adapter *adapter); void igc_down(struct igc_adapter *adapter); +int igc_open(struct net_device *netdev); +int igc_close(struct net_device *netdev); int igc_setup_tx_resources(struct igc_ring *ring); int igc_setup_rx_resources(struct igc_ring *ring); void igc_free_tx_resources(struct igc_ring *ring); @@ -35,34 +264,42 @@ int igc_reinit_queues(struct igc_adapter *adapter); void igc_write_rss_indir_tbl(struct igc_adapter *adapter); bool igc_has_link(struct igc_adapter *adapter); void igc_reset(struct igc_adapter *adapter); -int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx); -int igc_add_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags); -int igc_del_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags); void igc_update_stats(struct igc_adapter *adapter); +void igc_disable_rx_ring(struct igc_ring *ring); +void igc_enable_rx_ring(struct igc_ring *ring); +void igc_disable_tx_ring(struct igc_ring *ring); +void igc_enable_tx_ring(struct igc_ring *ring); +int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); + +/* igc_dump declarations */ +void igc_rings_dump(struct igc_adapter *adapter); +void igc_regs_dump(struct igc_adapter *adapter); extern char igc_driver_name[]; -extern char igc_driver_version[]; #define IGC_REGS_LEN 740 -#define IGC_RETA_SIZE 128 /* flags controlling PTP/1588 function */ #define IGC_PTP_ENABLED BIT(0) -/* Interrupt defines */ -#define IGC_START_ITR 648 /* ~6000 ints/sec */ +/* Flags definitions */ #define IGC_FLAG_HAS_MSI BIT(0) #define IGC_FLAG_QUEUE_PAIRS BIT(3) #define IGC_FLAG_DMAC BIT(4) #define IGC_FLAG_PTP BIT(8) +#define IGC_FLAG_WOL_SUPPORTED BIT(8) #define IGC_FLAG_NEED_LINK_UPDATE BIT(9) #define IGC_FLAG_MEDIA_RESET BIT(10) #define IGC_FLAG_MAS_ENABLE BIT(12) #define IGC_FLAG_HAS_MSIX BIT(13) +#define IGC_FLAG_EEE BIT(14) #define IGC_FLAG_VLAN_PROMISC BIT(15) #define IGC_FLAG_RX_LEGACY BIT(16) +#define IGC_FLAG_TSN_QBV_ENABLED BIT(17) +#define IGC_FLAG_TSN_QAV_ENABLED BIT(18) + +#define IGC_FLAG_TSN_ANY_ENABLED \ + (IGC_FLAG_TSN_QBV_ENABLED | IGC_FLAG_TSN_QAV_ENABLED) #define IGC_FLAG_RSS_FIELD_IPV4_UDP BIT(6) #define IGC_FLAG_RSS_FIELD_IPV6_UDP BIT(7) @@ -71,6 +308,7 @@ extern char igc_driver_version[]; #define IGC_MRQC_RSS_FIELD_IPV4_UDP 0x00400000 #define IGC_MRQC_RSS_FIELD_IPV6_UDP 0x00800000 +/* Interrupt defines */ #define IGC_START_ITR 648 /* ~6000 ints/sec */ #define IGC_4K_ITR 980 #define IGC_20K_ITR 196 @@ -92,13 +330,6 @@ extern char igc_driver_version[]; #define IGC_MIN_RXD 80 #define IGC_MAX_RXD 4096 -/* Transmit and receive queues */ -#define IGC_MAX_RX_QUEUES 4 -#define IGC_MAX_TX_QUEUES 4 - -#define MAX_Q_VECTORS 8 -#define MAX_STD_JUMBO_FRAME_SIZE 9216 - /* Supported Rx Buffer Sizes */ #define IGC_RXBUFFER_256 256 #define IGC_RXBUFFER_2048 2048 @@ -108,18 +339,14 @@ extern char igc_driver_version[]; #define IGC_RX_HDR_LEN IGC_RXBUFFER_256 /* Transmit and receive latency (for PTP timestamps) */ -/* FIXME: These values were estimated using the ones that i210 has as - * basis, they seem to provide good numbers with ptp4l/phc2sys, but we - * need to confirm them. - */ -#define IGC_I225_TX_LATENCY_10 9542 -#define IGC_I225_TX_LATENCY_100 1024 -#define IGC_I225_TX_LATENCY_1000 178 -#define IGC_I225_TX_LATENCY_2500 64 -#define IGC_I225_RX_LATENCY_10 20662 -#define IGC_I225_RX_LATENCY_100 2213 -#define IGC_I225_RX_LATENCY_1000 448 -#define IGC_I225_RX_LATENCY_2500 160 +#define IGC_I225_TX_LATENCY_10 240 +#define IGC_I225_TX_LATENCY_100 58 +#define IGC_I225_TX_LATENCY_1000 80 +#define IGC_I225_TX_LATENCY_2500 1325 +#define IGC_I225_RX_LATENCY_10 6450 +#define IGC_I225_RX_LATENCY_100 185 +#define IGC_I225_RX_LATENCY_1000 300 +#define IGC_I225_RX_LATENCY_2500 1485 /* RX and TX descriptor control thresholds. * PTHRESH - MAC will consider prefetch if it has fewer than this number of @@ -158,6 +385,7 @@ extern char igc_driver_version[]; /* VLAN info */ #define IGC_TX_FLAGS_VLAN_MASK 0xffff0000 +#define IGC_TX_FLAGS_VLAN_SHIFT 16 /* igc_test_staterr - tests bits within Rx descriptor status and error fields */ static inline __le32 igc_test_staterr(union igc_adv_rx_desc *rx_desc, @@ -198,13 +426,23 @@ enum igc_boards { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IGC_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) +enum igc_tx_buffer_type { + IGC_TX_BUFFER_TYPE_SKB, + IGC_TX_BUFFER_TYPE_XDP, + IGC_TX_BUFFER_TYPE_XSK, +}; + /* wrapper around a pointer to a socket buffer, * so a DMA handle can be stored along with the buffer */ struct igc_tx_buffer { union igc_adv_tx_desc *next_to_watch; unsigned long time_stamp; - struct sk_buff *skb; + enum igc_tx_buffer_type type; + union { + struct sk_buff *skb; + struct xdp_frame *xdpf; + }; unsigned int bytecount; u16 gso_segs; __be16 protocol; @@ -215,92 +453,20 @@ struct igc_tx_buffer { }; struct igc_rx_buffer { - dma_addr_t dma; - struct page *page; + union { + struct { + dma_addr_t dma; + struct page *page; #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) - __u32 page_offset; + __u32 page_offset; #else - __u16 page_offset; + __u16 page_offset; #endif - __u16 pagecnt_bias; -}; - -struct igc_tx_queue_stats { - u64 packets; - u64 bytes; - u64 restart_queue; - u64 restart_queue2; -}; - -struct igc_rx_queue_stats { - u64 packets; - u64 bytes; - u64 drops; - u64 csum_err; - u64 alloc_failed; -}; - -struct igc_rx_packet_stats { - u64 ipv4_packets; /* IPv4 headers processed */ - u64 ipv4e_packets; /* IPv4E headers with extensions processed */ - u64 ipv6_packets; /* IPv6 headers processed */ - u64 ipv6e_packets; /* IPv6E headers with extensions processed */ - u64 tcp_packets; /* TCP headers processed */ - u64 udp_packets; /* UDP headers processed */ - u64 sctp_packets; /* SCTP headers processed */ - u64 nfs_packets; /* NFS headers processe */ - u64 other_packets; -}; - -struct igc_ring_container { - struct igc_ring *ring; /* pointer to linked list of rings */ - unsigned int total_bytes; /* total bytes processed this int */ - unsigned int total_packets; /* total packets processed this int */ - u16 work_limit; /* total work allowed per interrupt */ - u8 count; /* total number of rings in vector */ - u8 itr; /* current ITR setting for ring */ -}; - -struct igc_ring { - struct igc_q_vector *q_vector; /* backlink to q_vector */ - struct net_device *netdev; /* back pointer to net_device */ - struct device *dev; /* device for dma mapping */ - union { /* array of buffer info structs */ - struct igc_tx_buffer *tx_buffer_info; - struct igc_rx_buffer *rx_buffer_info; - }; - void *desc; /* descriptor ring memory */ - unsigned long flags; /* ring specific flags */ - void __iomem *tail; /* pointer to ring tail register */ - dma_addr_t dma; /* phys address of the ring */ - unsigned int size; /* length of desc. ring in bytes */ - - u16 count; /* number of desc. in the ring */ - u8 queue_index; /* logical index of the ring*/ - u8 reg_idx; /* physical index of the ring */ - bool launchtime_enable; /* true if LaunchTime is enabled */ - - /* everything past this point are written often */ - u16 next_to_clean; - u16 next_to_use; - u16 next_to_alloc; - - union { - /* TX */ - struct { - struct igc_tx_queue_stats tx_stats; - struct u64_stats_sync tx_syncp; - struct u64_stats_sync tx_syncp2; - }; - /* RX */ - struct { - struct igc_rx_queue_stats rx_stats; - struct igc_rx_packet_stats pkt_stats; - struct u64_stats_sync rx_syncp; - struct sk_buff *skb; + __u16 pagecnt_bias; }; + struct xdp_buff *xdp; }; -} ____cacheline_internodealigned_in_smp; +}; struct igc_q_vector { struct igc_adapter *adapter; /* backlink */ @@ -319,154 +485,56 @@ struct igc_q_vector { struct net_device poll_dev; /* for dynamic allocation of rings associated with this q_vector */ - struct igc_ring ring[0] ____cacheline_internodealigned_in_smp; + struct igc_ring ring[] ____cacheline_internodealigned_in_smp; }; -#define MAX_ETYPE_FILTER (4 - 1) - enum igc_filter_match_flags { - IGC_FILTER_FLAG_ETHER_TYPE = 0x1, - IGC_FILTER_FLAG_VLAN_TCI = 0x2, - IGC_FILTER_FLAG_SRC_MAC_ADDR = 0x4, - IGC_FILTER_FLAG_DST_MAC_ADDR = 0x8, + IGC_FILTER_FLAG_ETHER_TYPE = BIT(0), + IGC_FILTER_FLAG_VLAN_TCI = BIT(1), + IGC_FILTER_FLAG_SRC_MAC_ADDR = BIT(2), + IGC_FILTER_FLAG_DST_MAC_ADDR = BIT(3), + IGC_FILTER_FLAG_USER_DATA = BIT(4), + IGC_FILTER_FLAG_VLAN_ETYPE = BIT(5), }; -/* RX network flow classification data structure */ -struct igc_nfc_input { - /* Byte layout in order, all values with MSB first: - * match_flags - 1 byte - * etype - 2 bytes - * vlan_tci - 2 bytes - */ +struct igc_nfc_filter { u8 match_flags; - __be16 etype; - __be16 vlan_tci; + u16 etype; + __be16 vlan_etype; + u16 vlan_tci; u8 src_addr[ETH_ALEN]; u8 dst_addr[ETH_ALEN]; + u8 user_data[8]; + u8 user_mask[8]; + u8 flex_index; + u8 rx_queue; + u8 prio; + u8 immediate_irq; + u8 drop; }; -struct igc_nfc_filter { - struct hlist_node nfc_node; - struct igc_nfc_input filter; - unsigned long cookie; - u16 etype_reg_index; - u16 sw_idx; +struct igc_nfc_rule { + struct list_head list; + struct igc_nfc_filter filter; + u32 location; u16 action; + bool flex; }; -struct igc_mac_addr { - u8 addr[ETH_ALEN]; - u8 queue; - u8 state; /* bitmask */ -}; - -#define IGC_MAC_STATE_DEFAULT 0x1 -#define IGC_MAC_STATE_IN_USE 0x2 -#define IGC_MAC_STATE_SRC_ADDR 0x4 -#define IGC_MAC_STATE_QUEUE_STEERING 0x8 - -#define IGC_MAX_RXNFC_FILTERS 16 - -/* Board specific private data structure */ -struct igc_adapter { - struct net_device *netdev; - - unsigned long state; - unsigned int flags; - unsigned int num_q_vectors; - - struct msix_entry *msix_entries; - - /* TX */ - u16 tx_work_limit; - u32 tx_timeout_count; - int num_tx_queues; - struct igc_ring *tx_ring[IGC_MAX_TX_QUEUES]; - - /* RX */ - int num_rx_queues; - struct igc_ring *rx_ring[IGC_MAX_RX_QUEUES]; - - struct timer_list watchdog_timer; - struct timer_list dma_err_timer; - struct timer_list phy_info_timer; - - u32 wol; - u32 en_mng_pt; - u16 link_speed; - u16 link_duplex; - - u8 port_num; - - u8 __iomem *io_addr; - /* Interrupt Throttle Rate */ - u32 rx_itr_setting; - u32 tx_itr_setting; - - struct work_struct reset_task; - struct work_struct watchdog_task; - struct work_struct dma_err_task; - bool fc_autoneg; - - u8 tx_timeout_factor; - - int msg_enable; - u32 max_frame_size; - u32 min_frame_size; - - /* OS defined structs */ - struct pci_dev *pdev; - /* lock for statistics */ - spinlock_t stats64_lock; - struct rtnl_link_stats64 stats64; - - /* structs defined in igc_hw.h */ - struct igc_hw hw; - struct igc_hw_stats stats; - - struct igc_q_vector *q_vector[MAX_Q_VECTORS]; - u32 eims_enable_mask; - u32 eims_other; - - u16 tx_ring_count; - u16 rx_ring_count; - - u32 tx_hwtstamp_timeouts; - u32 tx_hwtstamp_skipped; - u32 rx_hwtstamp_cleared; - - u32 rss_queues; - u32 rss_indir_tbl_init; - - /* RX network flow classification support */ - struct hlist_head nfc_filter_list; - struct hlist_head cls_flower_list; - unsigned int nfc_filter_count; - - /* lock for RX network flow classification filter */ - spinlock_t nfc_lock; - bool etype_bitmap[MAX_ETYPE_FILTER]; - - struct igc_mac_addr *mac_table; - - u8 rss_indir_tbl[IGC_RETA_SIZE]; - - unsigned long link_check_timeout; - struct igc_info ei; - - struct ptp_clock *ptp_clock; - struct ptp_clock_info ptp_caps; - struct work_struct ptp_tx_work; - struct sk_buff *ptp_tx_skb; - struct hwtstamp_config tstamp_config; - unsigned long ptp_tx_start; - unsigned long last_rx_ptp_check; - unsigned long last_rx_timestamp; - unsigned int ptp_flags; - /* System time value lock */ - spinlock_t tmreg_lock; - struct cyclecounter cc; - struct timecounter tc; +/* IGC supports a total of 32 NFC rules: 16 MAC address based, 8 VLAN priority + * based, 8 ethertype based and 32 Flex filter based rules. + */ +#define IGC_MAX_RXNFC_RULES 64 + +struct igc_flex_filter { + u8 index; + u8 data[128]; + u8 mask[16]; + u8 length; + u8 rx_queue; + u8 prio; + u8 immediate_irq; + u8 drop; }; /* igc_desc_unused - calculate if we have unused descriptors */ @@ -505,11 +573,16 @@ enum igc_ring_flags_t { IGC_RING_FLAG_RX_SCTP_CSUM, IGC_RING_FLAG_RX_LB_VLAN_BSWAP, IGC_RING_FLAG_TX_CTX_IDX, - IGC_RING_FLAG_TX_DETECT_HANG + IGC_RING_FLAG_TX_DETECT_HANG, + IGC_RING_FLAG_AF_XDP_ZC, }; #define ring_uses_large_buffer(ring) \ test_bit(IGC_RING_FLAG_RX_3K_BUFFER, &(ring)->flags) +#define set_ring_uses_large_buffer(ring) \ + set_bit(IGC_RING_FLAG_RX_3K_BUFFER, &(ring)->flags) +#define clear_ring_uses_large_buffer(ring) \ + clear_bit(IGC_RING_FLAG_RX_3K_BUFFER, &(ring)->flags) #define ring_uses_build_skb(ring) \ test_bit(IGC_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags) @@ -540,25 +613,24 @@ static inline s32 igc_read_phy_reg(struct igc_hw *hw, u32 offset, u16 *data) if (hw->phy.ops.read_reg) return hw->phy.ops.read_reg(hw, offset, data); - return 0; + return -EOPNOTSUPP; } -/* forward declaration */ void igc_reinit_locked(struct igc_adapter *); -int igc_add_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input); -int igc_erase_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input); +struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, + u32 location); +int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule); +void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule); void igc_ptp_init(struct igc_adapter *adapter); void igc_ptp_reset(struct igc_adapter *adapter); +void igc_ptp_suspend(struct igc_adapter *adapter); void igc_ptp_stop(struct igc_adapter *adapter); -void igc_ptp_rx_rgtstamp(struct igc_q_vector *q_vector, struct sk_buff *skb); -void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va, - struct sk_buff *skb); +ktime_t igc_ptp_rx_pktstamp(struct igc_adapter *adapter, __le32 *buf); int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr); int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr); void igc_ptp_tx_hang(struct igc_adapter *adapter); +void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts); #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring)) diff --git a/drivers/net/ethernet/intel/igc/igc_base.c b/drivers/net/ethernet/intel/igc/igc_base.c index 5a506440560a..a15927e77272 100644 --- a/drivers/net/ethernet/intel/igc/igc_base.c +++ b/drivers/net/ethernet/intel/igc/igc_base.c @@ -26,7 +26,7 @@ static s32 igc_reset_hw_base(struct igc_hw *hw) */ ret_val = igc_disable_pcie_master(hw); if (ret_val) - hw_dbg("PCI-E Master disable polling has failed.\n"); + hw_dbg("PCI-E Master disable polling has failed\n"); hw_dbg("Masking off all interrupts\n"); wr32(IGC_IMC, 0xffffffff); @@ -40,7 +40,7 @@ static s32 igc_reset_hw_base(struct igc_hw *hw) ctrl = rd32(IGC_CTRL); hw_dbg("Issuing a global reset to MAC\n"); - wr32(IGC_CTRL, ctrl | IGC_CTRL_DEV_RST); + wr32(IGC_CTRL, ctrl | IGC_CTRL_RST); ret_val = igc_get_auto_rd_done(hw); if (ret_val) { @@ -158,11 +158,6 @@ static s32 igc_init_phy_params_base(struct igc_hw *hw) struct igc_phy_info *phy = &hw->phy; s32 ret_val = 0; - if (hw->phy.media_type != igc_media_type_copper) { - phy->type = igc_phy_none; - goto out; - } - phy->autoneg_mask = AUTONEG_ADVERTISE_SPEED_DEFAULT_2500; phy->reset_delay_us = 100; @@ -177,7 +172,7 @@ static s32 igc_init_phy_params_base(struct igc_hw *hw) */ ret_val = hw->phy.ops.reset(hw); if (ret_val) { - hw_dbg("Error resetting the PHY.\n"); + hw_dbg("Error resetting the PHY\n"); goto out; } @@ -187,16 +182,6 @@ static s32 igc_init_phy_params_base(struct igc_hw *hw) igc_check_for_copper_link(hw); - /* Verify phy id and set remaining function pointers */ - switch (phy->id) { - case I225_I_PHY_ID: - phy->type = igc_phy_i225; - break; - default: - ret_val = -IGC_ERR_PHY; - goto out; - } - out: return ret_val; } @@ -212,6 +197,16 @@ static s32 igc_get_invariants_base(struct igc_hw *hw) case IGC_DEV_ID_I225_I: case IGC_DEV_ID_I220_V: case IGC_DEV_ID_I225_K: + case IGC_DEV_ID_I225_K2: + case IGC_DEV_ID_I226_K: + case IGC_DEV_ID_I225_LMVP: + case IGC_DEV_ID_I226_LMVP: + case IGC_DEV_ID_I225_IT: + case IGC_DEV_ID_I226_LM: + case IGC_DEV_ID_I226_V: + case IGC_DEV_ID_I226_IT: + case IGC_DEV_ID_I221_V: + case IGC_DEV_ID_I226_BLANK_NVM: case IGC_DEV_ID_I225_BLANK_NVM: mac->type = igc_i225; break; @@ -364,7 +359,7 @@ void igc_rx_fifo_flush_base(struct igc_hw *hw) } if (ms_wait == 10) - pr_debug("Queue disable timed out after 10ms\n"); + hw_dbg("Queue disable timed out after 10ms\n"); /* Clear RLPML, RCTL.SBP, RFCTL.LEF, and set RCTL.LPE so that all * incoming packets are rejected. Set enable and wait 2ms so that diff --git a/drivers/net/ethernet/intel/igc/igc_base.h b/drivers/net/ethernet/intel/igc/igc_base.h index ea627ce52525..ce530f5fd7bd 100644 --- a/drivers/net/ethernet/intel/igc/igc_base.h +++ b/drivers/net/ethernet/intel/igc/igc_base.h @@ -78,9 +78,11 @@ union igc_adv_rx_desc { /* Additional Transmit Descriptor Control definitions */ #define IGC_TXDCTL_QUEUE_ENABLE 0x02000000 /* Ena specific Tx Queue */ +#define IGC_TXDCTL_SWFLUSH 0x04000000 /* Transmit Software Flush */ /* Additional Receive Descriptor Control definitions */ #define IGC_RXDCTL_QUEUE_ENABLE 0x02000000 /* Ena specific Rx Queue */ +#define IGC_RXDCTL_SWFLUSH 0x04000000 /* Receive Software Flush */ /* SRRCTL bit definitions */ #define IGC_SRRCTL_BSIZEPKT_SHIFT 10 /* Shift _right_ */ diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 58efa7a02c68..4f9d7f013a95 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -8,6 +8,8 @@ #define REQ_TX_DESCRIPTOR_MULTIPLE 8 #define REQ_RX_DESCRIPTOR_MULTIPLE 8 +#define IGC_CTRL_EXT_SDP2_DIR 0x00000400 /* SDP2 Data direction */ +#define IGC_CTRL_EXT_SDP3_DIR 0x00000800 /* SDP3 Data direction */ #define IGC_CTRL_EXT_DRV_LOAD 0x10000000 /* Drv loaded bit for FW */ /* Definitions for power management and wakeup registers */ @@ -15,8 +17,22 @@ #define IGC_WUC_PME_EN 0x00000002 /* PME Enable */ /* Wake Up Filter Control */ -#define IGC_WUFC_LNKC 0x00000001 /* Link Status Change Wakeup Enable */ -#define IGC_WUFC_MC 0x00000008 /* Directed Multicast Wakeup Enable */ +#define IGC_WUFC_LNKC 0x00000001 /* Link Status Change Wakeup Enable */ +#define IGC_WUFC_MAG 0x00000002 /* Magic Packet Wakeup Enable */ +#define IGC_WUFC_EX 0x00000004 /* Directed Exact Wakeup Enable */ +#define IGC_WUFC_MC 0x00000008 /* Directed Multicast Wakeup Enable */ +#define IGC_WUFC_BC 0x00000010 /* Broadcast Wakeup Enable */ +#define IGC_WUFC_FLEX_HQ BIT(14) /* Flex Filters Host Queuing */ +#define IGC_WUFC_FLX0 BIT(16) /* Flexible Filter 0 Enable */ +#define IGC_WUFC_FLX1 BIT(17) /* Flexible Filter 1 Enable */ +#define IGC_WUFC_FLX2 BIT(18) /* Flexible Filter 2 Enable */ +#define IGC_WUFC_FLX3 BIT(19) /* Flexible Filter 3 Enable */ +#define IGC_WUFC_FLX4 BIT(20) /* Flexible Filter 4 Enable */ +#define IGC_WUFC_FLX5 BIT(21) /* Flexible Filter 5 Enable */ +#define IGC_WUFC_FLX6 BIT(22) /* Flexible Filter 6 Enable */ +#define IGC_WUFC_FLX7 BIT(23) /* Flexible Filter 7 Enable */ + +#define IGC_WUFC_FILTER_MASK GENMASK(23, 14) #define IGC_CTRL_ADVD3WUC 0x00100000 /* D3 WUC */ @@ -41,13 +57,37 @@ /* Wake Up Packet Memory stores the first 128 bytes of the wake up packet */ #define IGC_WUPM_BYTES 128 -/* Physical Func Reset Done Indication */ -#define IGC_CTRL_EXT_LINK_MODE_MASK 0x00C00000 +/* Wakeup Filter Control Extended */ +#define IGC_WUFC_EXT_FLX8 BIT(8) /* Flexible Filter 8 Enable */ +#define IGC_WUFC_EXT_FLX9 BIT(9) /* Flexible Filter 9 Enable */ +#define IGC_WUFC_EXT_FLX10 BIT(10) /* Flexible Filter 10 Enable */ +#define IGC_WUFC_EXT_FLX11 BIT(11) /* Flexible Filter 11 Enable */ +#define IGC_WUFC_EXT_FLX12 BIT(12) /* Flexible Filter 12 Enable */ +#define IGC_WUFC_EXT_FLX13 BIT(13) /* Flexible Filter 13 Enable */ +#define IGC_WUFC_EXT_FLX14 BIT(14) /* Flexible Filter 14 Enable */ +#define IGC_WUFC_EXT_FLX15 BIT(15) /* Flexible Filter 15 Enable */ +#define IGC_WUFC_EXT_FLX16 BIT(16) /* Flexible Filter 16 Enable */ +#define IGC_WUFC_EXT_FLX17 BIT(17) /* Flexible Filter 17 Enable */ +#define IGC_WUFC_EXT_FLX18 BIT(18) /* Flexible Filter 18 Enable */ +#define IGC_WUFC_EXT_FLX19 BIT(19) /* Flexible Filter 19 Enable */ +#define IGC_WUFC_EXT_FLX20 BIT(20) /* Flexible Filter 20 Enable */ +#define IGC_WUFC_EXT_FLX21 BIT(21) /* Flexible Filter 21 Enable */ +#define IGC_WUFC_EXT_FLX22 BIT(22) /* Flexible Filter 22 Enable */ +#define IGC_WUFC_EXT_FLX23 BIT(23) /* Flexible Filter 23 Enable */ +#define IGC_WUFC_EXT_FLX24 BIT(24) /* Flexible Filter 24 Enable */ +#define IGC_WUFC_EXT_FLX25 BIT(25) /* Flexible Filter 25 Enable */ +#define IGC_WUFC_EXT_FLX26 BIT(26) /* Flexible Filter 26 Enable */ +#define IGC_WUFC_EXT_FLX27 BIT(27) /* Flexible Filter 27 Enable */ +#define IGC_WUFC_EXT_FLX28 BIT(28) /* Flexible Filter 28 Enable */ +#define IGC_WUFC_EXT_FLX29 BIT(29) /* Flexible Filter 29 Enable */ +#define IGC_WUFC_EXT_FLX30 BIT(30) /* Flexible Filter 30 Enable */ +#define IGC_WUFC_EXT_FLX31 BIT(31) /* Flexible Filter 31 Enable */ + +#define IGC_WUFC_EXT_FILTER_MASK GENMASK(31, 8) /* Loop limit on how long we wait for auto-negotiation to complete */ #define COPPER_LINK_UP_LIMIT 10 #define PHY_AUTO_NEG_LIMIT 45 -#define PHY_FORCE_LIMIT 20 /* Number of 100 microseconds we wait for PCI Express master disable */ #define MASTER_DISABLE_TIMEOUT 800 @@ -63,8 +103,14 @@ * (RAR[15]) for our directed address used by controllers with * manageability enabled, allowing us room for 15 multicast addresses. */ +#define IGC_RAH_RAH_MASK 0x0000FFFF +#define IGC_RAH_ASEL_MASK 0x00030000 +#define IGC_RAH_ASEL_SRC_ADDR BIT(16) +#define IGC_RAH_QSEL_MASK 0x000C0000 +#define IGC_RAH_QSEL_SHIFT 18 +#define IGC_RAH_QSEL_ENABLE BIT(28) #define IGC_RAH_AV 0x80000000 /* Receive descriptor valid */ -#define IGC_RAH_POOL_1 0x00040000 + #define IGC_RAL_MAC_ADDR_LEN 4 #define IGC_RAH_MAC_ADDR_LEN 2 @@ -81,17 +127,19 @@ #define IGC_ERR_SWFW_SYNC 13 /* Device Control */ -#define IGC_CTRL_DEV_RST 0x20000000 /* Device reset */ +#define IGC_CTRL_RST 0x04000000 /* Global reset */ #define IGC_CTRL_PHY_RST 0x80000000 /* PHY Reset */ #define IGC_CTRL_SLU 0x00000040 /* Set link up (Force Link) */ #define IGC_CTRL_FRCSPD 0x00000800 /* Force Speed */ #define IGC_CTRL_FRCDPX 0x00001000 /* Force Duplex */ +#define IGC_CTRL_VME 0x40000000 /* IEEE VLAN mode enable */ #define IGC_CTRL_RFCE 0x08000000 /* Receive Flow Control enable */ #define IGC_CTRL_TFCE 0x10000000 /* Transmit flow control enable */ -#define IGC_CONNSW_AUTOSENSE_EN 0x1 +#define IGC_CTRL_SDP0_DIR 0x00400000 /* SDP0 Data direction */ +#define IGC_CTRL_SDP1_DIR 0x00800000 /* SDP1 Data direction */ /* As per the EAS the maximum supported size is 9.5KB (9728 bytes) */ #define MAX_JUMBO_FRAME_SIZE 0x2600 @@ -120,13 +168,11 @@ #define NWAY_LPAR_ASM_DIR 0x0800 /* LP Asymmetric Pause Direction bit */ /* 1000BASE-T Control Register */ -#define CR_1000T_ASYM_PAUSE 0x0080 /* Advertise asymmetric pause bit */ #define CR_1000T_HD_CAPS 0x0100 /* Advertise 1000T HD capability */ #define CR_1000T_FD_CAPS 0x0200 /* Advertise 1000T FD capability */ /* 1000BASE-T Status Register */ #define SR_1000T_REMOTE_RX_STATUS 0x1000 /* Remote receiver OK */ -#define SR_1000T_LOCAL_RX_STATUS 0x2000 /* Local receiver OK */ /* PHY GPY 211 registers */ #define STANDARD_AN_REG_MASK 0x0007 /* MMD */ @@ -157,17 +203,13 @@ #define IGC_NVM_RW_REG_START 1 /* Start operation */ #define IGC_NVM_RW_ADDR_SHIFT 2 /* Shift to the address bits */ #define IGC_NVM_POLL_READ 0 /* Flag for polling for read complete */ +#define IGC_NVM_DEV_STARTER 5 /* Dev_starter Version */ /* NVM Word Offsets */ #define NVM_CHECKSUM_REG 0x003F /* For checksumming, the sum of all words in the NVM should equal 0xBABA. */ #define NVM_SUM 0xBABA - -#define NVM_PBA_OFFSET_0 8 -#define NVM_PBA_OFFSET_1 9 -#define NVM_RESERVED_WORD 0xFFFF -#define NVM_PBA_PTR_GUARD 0xFAFA #define NVM_WORD_SIZE_BASE_SHIFT 6 /* Collision related configuration parameters */ @@ -181,7 +223,6 @@ #define IGC_STATUS_LU 0x00000002 /* Link up.0=no,1=link */ #define IGC_STATUS_FUNC_MASK 0x0000000C /* PCI Function Mask */ #define IGC_STATUS_FUNC_SHIFT 2 -#define IGC_STATUS_FUNC_1 0x00000004 /* Function 1 */ #define IGC_STATUS_TXOFF 0x00000010 /* transmission paused */ #define IGC_STATUS_SPEED_100 0x00000040 /* Speed 100Mb/s */ #define IGC_STATUS_SPEED_1000 0x00000080 /* Speed 1000Mb/s */ @@ -249,7 +290,6 @@ /* Interrupt Cause Set */ #define IGC_ICS_LSC IGC_ICR_LSC /* Link Status Change */ #define IGC_ICS_RXDMT0 IGC_ICR_RXDMT0 /* rx desc min. threshold */ -#define IGC_ICS_DRSTA IGC_ICR_DRSTA /* Device Reset Aserted */ #define IGC_ICR_DOUTSYNC 0x10000000 /* NIC DMA out of sync */ #define IGC_EITR_CNT_IGNR 0x80000000 /* Don't reset counters on write */ @@ -259,27 +299,22 @@ #define IGC_GPIE_EIAME 0x40000000 #define IGC_GPIE_PBA 0x80000000 +/* Receive Descriptor bit definitions */ +#define IGC_RXD_STAT_DD 0x01 /* Descriptor Done */ + /* Transmit Descriptor bit definitions */ #define IGC_TXD_DTYP_D 0x00100000 /* Data Descriptor */ #define IGC_TXD_DTYP_C 0x00000000 /* Context Descriptor */ #define IGC_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define IGC_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ #define IGC_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IGC_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ #define IGC_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IGC_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IGC_TXD_CMD_RPS 0x10000000 /* Report Packet Sent */ #define IGC_TXD_CMD_DEXT 0x20000000 /* Desc extension (0 = legacy) */ #define IGC_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IGC_TXD_CMD_IDE 0x80000000 /* Enable Tidv register */ #define IGC_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IGC_TXD_STAT_EC 0x00000002 /* Excess Collisions */ -#define IGC_TXD_STAT_LC 0x00000004 /* Late Collisions */ -#define IGC_TXD_STAT_TU 0x00000008 /* Transmit underrun */ #define IGC_TXD_CMD_TCP 0x01000000 /* TCP packet */ #define IGC_TXD_CMD_IP 0x02000000 /* IP packet */ #define IGC_TXD_CMD_TSE 0x04000000 /* TCP Seg enable */ -#define IGC_TXD_STAT_TC 0x00000004 /* Tx Underrun */ #define IGC_TXD_EXTCMD_TSTAMP 0x00000010 /* IEEE1588 Timestamp packet */ /* IPSec Encrypt Enable */ @@ -292,7 +327,6 @@ #define IGC_TCTL_CT 0x00000ff0 /* collision threshold */ #define IGC_TCTL_COLD 0x003ff000 /* collision distance */ #define IGC_TCTL_RTLC 0x01000000 /* Re-transmit on late collision */ -#define IGC_TCTL_MULR 0x10000000 /* Multiple request support */ /* Flow Control Constants */ #define FLOW_CONTROL_ADDRESS_LOW 0x00C28001 @@ -328,27 +362,17 @@ #define IGC_RXD_STAT_IXSM 0x04 /* Ignore checksum */ #define IGC_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ #define IGC_RXD_STAT_TCPCS 0x20 /* TCP xsum calculated */ +#define IGC_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ + +#define IGC_RXDEXT_STATERR_LB 0x00040000 /* Advanced Receive Descriptor bit definitions */ #define IGC_RXDADV_STAT_TSIP 0x08000 /* timestamp in packet */ -#define IGC_RXDADV_STAT_TS 0x10000 /* Pkt was time stamped */ -#define IGC_RXDEXT_STATERR_CE 0x01000000 -#define IGC_RXDEXT_STATERR_SE 0x02000000 -#define IGC_RXDEXT_STATERR_SEQ 0x04000000 -#define IGC_RXDEXT_STATERR_CXE 0x10000000 -#define IGC_RXDEXT_STATERR_TCPE 0x20000000 +#define IGC_RXDEXT_STATERR_L4E 0x20000000 #define IGC_RXDEXT_STATERR_IPE 0x40000000 #define IGC_RXDEXT_STATERR_RXE 0x80000000 -/* Same mask, but for extended and packet split descriptors */ -#define IGC_RXDEXT_ERR_FRAME_ERR_MASK ( \ - IGC_RXDEXT_STATERR_CE | \ - IGC_RXDEXT_STATERR_SE | \ - IGC_RXDEXT_STATERR_SEQ | \ - IGC_RXDEXT_STATERR_CXE | \ - IGC_RXDEXT_STATERR_RXE) - #define IGC_MRQC_RSS_FIELD_IPV4_TCP 0x00010000 #define IGC_MRQC_RSS_FIELD_IPV4 0x00020000 #define IGC_MRQC_RSS_FIELD_IPV6_TCP_EX 0x00040000 @@ -371,6 +395,11 @@ #define I225_TXPBSIZE_DEFAULT 0x04000014 /* TXPBSIZE default */ #define IGC_RXPBS_CFG_TS_EN 0x80000000 /* Timestamp in Rx buffer */ +#define IGC_TXPBSIZE_TSN 0x04145145 /* 5k bytes buffer for each queue */ + +#define IGC_DTXMXPKTSZ_TSN 0x19 /* 1600 bytes of max TX DMA packet size */ +#define IGC_DTXMXPKTSZ_DEFAULT 0x98 /* 9728-byte Jumbo frames */ + /* Time Sync Interrupt Causes */ #define IGC_TSICR_SYS_WRAP BIT(0) /* SYSTIM Wrap around. */ #define IGC_TSICR_TXTS BIT(1) /* Transmit Timestamp. */ @@ -381,16 +410,12 @@ #define IGC_TSICR_INTERRUPTS IGC_TSICR_TXTS -/* PTP Queue Filter */ -#define IGC_ETQF_1588 BIT(30) - #define IGC_FTQF_VF_BP 0x00008000 #define IGC_FTQF_1588_TIME_STAMP 0x08000000 #define IGC_FTQF_MASK 0xF0000000 #define IGC_FTQF_MASK_PROTO_BP 0x10000000 /* Time Sync Receive Control bit definitions */ -#define IGC_TSYNCRXCTL_VALID 0x00000001 /* Rx timestamp valid */ #define IGC_TSYNCRXCTL_TYPE_MASK 0x0000000E /* Rx type mask */ #define IGC_TSYNCRXCTL_TYPE_L2_V2 0x00 #define IGC_TSYNCRXCTL_TYPE_L4_V1 0x02 @@ -417,7 +442,7 @@ #define IGC_IMIREXT_SIZE_BP 0x00001000 /* Packet size bypass */ /* Time Sync Transmit Control bit definitions */ -#define IGC_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ +#define IGC_TSYNCTXCTL_TXTT_0 0x00000001 /* Tx timestamp reg 0 valid */ #define IGC_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ #define IGC_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */ #define IGC_TSYNCTXCTL_SYNC_COMP_ERR 0x20000000 /* sync err */ @@ -425,10 +450,115 @@ #define IGC_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */ #define IGC_TSYNCTXCTL_TXSYNSIG 0x00000020 /* Sample TX tstamp in PHY sop */ +/* Timer selection bits */ +#define IGC_AUX_IO_TIMER_SEL_SYSTIM0 (0u << 30) /* Select SYSTIM0 for auxiliary time stamp */ +#define IGC_AUX_IO_TIMER_SEL_SYSTIM1 (1u << 30) /* Select SYSTIM1 for auxiliary time stamp */ +#define IGC_AUX_IO_TIMER_SEL_SYSTIM2 (2u << 30) /* Select SYSTIM2 for auxiliary time stamp */ +#define IGC_AUX_IO_TIMER_SEL_SYSTIM3 (3u << 30) /* Select SYSTIM3 for auxiliary time stamp */ +#define IGC_TT_IO_TIMER_SEL_SYSTIM0 (0u << 30) /* Select SYSTIM0 for target time stamp */ +#define IGC_TT_IO_TIMER_SEL_SYSTIM1 (1u << 30) /* Select SYSTIM1 for target time stamp */ +#define IGC_TT_IO_TIMER_SEL_SYSTIM2 (2u << 30) /* Select SYSTIM2 for target time stamp */ +#define IGC_TT_IO_TIMER_SEL_SYSTIM3 (3u << 30) /* Select SYSTIM3 for target time stamp */ + +/* TSAUXC Configuration Bits */ +#define IGC_TSAUXC_EN_TT0 BIT(0) /* Enable target time 0. */ +#define IGC_TSAUXC_EN_TT1 BIT(1) /* Enable target time 1. */ +#define IGC_TSAUXC_EN_CLK0 BIT(2) /* Enable Configurable Frequency Clock 0. */ +#define IGC_TSAUXC_EN_CLK1 BIT(5) /* Enable Configurable Frequency Clock 1. */ +#define IGC_TSAUXC_EN_TS0 BIT(8) /* Enable hardware timestamp 0. */ +#define IGC_TSAUXC_AUTT0 BIT(9) /* Auxiliary Timestamp Taken. */ +#define IGC_TSAUXC_EN_TS1 BIT(10) /* Enable hardware timestamp 0. */ +#define IGC_TSAUXC_AUTT1 BIT(11) /* Auxiliary Timestamp Taken. */ +#define IGC_TSAUXC_PLSG BIT(17) /* Generate a pulse. */ +#define IGC_TSAUXC_DISABLE1 BIT(27) /* Disable SYSTIM0 Count Operation. */ +#define IGC_TSAUXC_DISABLE2 BIT(28) /* Disable SYSTIM1 Count Operation. */ +#define IGC_TSAUXC_DISABLE3 BIT(29) /* Disable SYSTIM2 Count Operation. */ +#define IGC_TSAUXC_DIS_TS_CLEAR BIT(30) /* Disable EN_TT0/1 auto clear. */ +#define IGC_TSAUXC_DISABLE0 BIT(31) /* Disable SYSTIM0 Count Operation. */ + +/* SDP Configuration Bits */ +#define IGC_AUX0_SEL_SDP0 (0u << 0) /* Assign SDP0 to auxiliary time stamp 0. */ +#define IGC_AUX0_SEL_SDP1 (1u << 0) /* Assign SDP1 to auxiliary time stamp 0. */ +#define IGC_AUX0_SEL_SDP2 (2u << 0) /* Assign SDP2 to auxiliary time stamp 0. */ +#define IGC_AUX0_SEL_SDP3 (3u << 0) /* Assign SDP3 to auxiliary time stamp 0. */ +#define IGC_AUX0_TS_SDP_EN (1u << 2) /* Enable auxiliary time stamp trigger 0. */ +#define IGC_AUX1_SEL_SDP0 (0u << 3) /* Assign SDP0 to auxiliary time stamp 1. */ +#define IGC_AUX1_SEL_SDP1 (1u << 3) /* Assign SDP1 to auxiliary time stamp 1. */ +#define IGC_AUX1_SEL_SDP2 (2u << 3) /* Assign SDP2 to auxiliary time stamp 1. */ +#define IGC_AUX1_SEL_SDP3 (3u << 3) /* Assign SDP3 to auxiliary time stamp 1. */ +#define IGC_AUX1_TS_SDP_EN (1u << 5) /* Enable auxiliary time stamp trigger 1. */ +#define IGC_TS_SDP0_SEL_TT0 (0u << 6) /* Target time 0 is output on SDP0. */ +#define IGC_TS_SDP0_SEL_TT1 (1u << 6) /* Target time 1 is output on SDP0. */ +#define IGC_TS_SDP0_SEL_FC0 (2u << 6) /* Freq clock 0 is output on SDP0. */ +#define IGC_TS_SDP0_SEL_FC1 (3u << 6) /* Freq clock 1 is output on SDP0. */ +#define IGC_TS_SDP0_EN (1u << 8) /* SDP0 is assigned to Tsync. */ +#define IGC_TS_SDP1_SEL_TT0 (0u << 9) /* Target time 0 is output on SDP1. */ +#define IGC_TS_SDP1_SEL_TT1 (1u << 9) /* Target time 1 is output on SDP1. */ +#define IGC_TS_SDP1_SEL_FC0 (2u << 9) /* Freq clock 0 is output on SDP1. */ +#define IGC_TS_SDP1_SEL_FC1 (3u << 9) /* Freq clock 1 is output on SDP1. */ +#define IGC_TS_SDP1_EN (1u << 11) /* SDP1 is assigned to Tsync. */ +#define IGC_TS_SDP2_SEL_TT0 (0u << 12) /* Target time 0 is output on SDP2. */ +#define IGC_TS_SDP2_SEL_TT1 (1u << 12) /* Target time 1 is output on SDP2. */ +#define IGC_TS_SDP2_SEL_FC0 (2u << 12) /* Freq clock 0 is output on SDP2. */ +#define IGC_TS_SDP2_SEL_FC1 (3u << 12) /* Freq clock 1 is output on SDP2. */ +#define IGC_TS_SDP2_EN (1u << 14) /* SDP2 is assigned to Tsync. */ +#define IGC_TS_SDP3_SEL_TT0 (0u << 15) /* Target time 0 is output on SDP3. */ +#define IGC_TS_SDP3_SEL_TT1 (1u << 15) /* Target time 1 is output on SDP3. */ +#define IGC_TS_SDP3_SEL_FC0 (2u << 15) /* Freq clock 0 is output on SDP3. */ +#define IGC_TS_SDP3_SEL_FC1 (3u << 15) /* Freq clock 1 is output on SDP3. */ +#define IGC_TS_SDP3_EN (1u << 17) /* SDP3 is assigned to Tsync. */ + +/* Transmit Scheduling */ +#define IGC_TQAVCTRL_TRANSMIT_MODE_TSN 0x00000001 +#define IGC_TQAVCTRL_ENHANCED_QAV 0x00000008 + +#define IGC_TXQCTL_QUEUE_MODE_LAUNCHT 0x00000001 +#define IGC_TXQCTL_STRICT_CYCLE 0x00000002 +#define IGC_TXQCTL_STRICT_END 0x00000004 +#define IGC_TXQCTL_QAV_SEL_MASK 0x000000C0 +#define IGC_TXQCTL_QAV_SEL_CBS0 0x00000080 +#define IGC_TXQCTL_QAV_SEL_CBS1 0x000000C0 + +#define IGC_TQAVCC_IDLESLOPE_MASK 0xFFFF +#define IGC_TQAVCC_KEEP_CREDITS BIT(30) + +#define IGC_MAX_SR_QUEUES 2 + /* Receive Checksum Control */ #define IGC_RXCSUM_CRCOFL 0x00000800 /* CRC32 offload enable */ #define IGC_RXCSUM_PCSD 0x00002000 /* packet checksum disabled */ +/* PCIe PTM Control */ +#define IGC_PTM_CTRL_START_NOW BIT(29) /* Start PTM Now */ +#define IGC_PTM_CTRL_EN BIT(30) /* Enable PTM */ +#define IGC_PTM_CTRL_TRIG BIT(31) /* PTM Cycle trigger */ +#define IGC_PTM_CTRL_SHRT_CYC(usec) (((usec) & 0x2f) << 2) +#define IGC_PTM_CTRL_PTM_TO(usec) (((usec) & 0xff) << 8) + +#define IGC_PTM_SHORT_CYC_DEFAULT 10 /* Default Short/interrupted cycle interval */ +#define IGC_PTM_CYC_TIME_DEFAULT 5 /* Default PTM cycle time */ +#define IGC_PTM_TIMEOUT_DEFAULT 255 /* Default timeout for PTM errors */ + +/* PCIe Digital Delay */ +#define IGC_PCIE_DIG_DELAY_DEFAULT 0x01440000 + +/* PCIe PHY Delay */ +#define IGC_PCIE_PHY_DELAY_DEFAULT 0x40900000 + +#define IGC_TIMADJ_ADJUST_METH 0x40000000 + +/* PCIe PTM Status */ +#define IGC_PTM_STAT_VALID BIT(0) /* PTM Status */ +#define IGC_PTM_STAT_RET_ERR BIT(1) /* Root port timeout */ +#define IGC_PTM_STAT_BAD_PTM_RES BIT(2) /* PTM Response msg instead of PTM Response Data */ +#define IGC_PTM_STAT_T4M1_OVFL BIT(3) /* T4 minus T1 overflow */ +#define IGC_PTM_STAT_ADJUST_1ST BIT(4) /* 1588 timer adjusted during 1st PTM cycle */ +#define IGC_PTM_STAT_ADJUST_CYC BIT(5) /* 1588 timer adjusted during non-1st PTM cycle */ + +/* PCIe PTM Cycle Control */ +#define IGC_PTM_CYCLE_CTRL_CYC_TIME(msec) ((msec) & 0x3ff) /* PTM Cycle Time (msec) */ +#define IGC_PTM_CYCLE_CTRL_AUTO_CYC_EN BIT(31) /* PTM Cycle Control */ + /* GPY211 - I225 defines */ #define GPY_MMD_MASK 0xFFFF0000 #define GPY_MMD_SHIFT 16 @@ -451,15 +581,9 @@ #define IGC_GEN_POLL_TIMEOUT 1920 /* PHY Control Register */ -#define MII_CR_FULL_DUPLEX 0x0100 /* FDX =1, half duplex =0 */ #define MII_CR_RESTART_AUTO_NEG 0x0200 /* Restart auto negotiation */ #define MII_CR_POWER_DOWN 0x0800 /* Power down */ #define MII_CR_AUTO_NEG_EN 0x1000 /* Auto Neg Enable */ -#define MII_CR_LOOPBACK 0x4000 /* 0 = normal, 1 = loopback */ -#define MII_CR_RESET 0x8000 /* 0 = normal, 1 = PHY reset */ -#define MII_CR_SPEED_1000 0x0040 -#define MII_CR_SPEED_100 0x2000 -#define MII_CR_SPEED_10 0x0000 /* PHY Status Register */ #define MII_SR_LINK_STATUS 0x0004 /* Link Status 1 = link */ @@ -477,9 +601,6 @@ #define PHY_1000T_CTRL 0x09 /* 1000Base-T Control Reg */ #define PHY_1000T_STATUS 0x0A /* 1000Base-T Status Reg */ -/* Bit definitions for valid PHY IDs. I = Integrated E = External */ -#define I225_I_PHY_ID 0x67C9DC00 - /* MDI Control */ #define IGC_MDIC_DATA_MASK 0x0000FFFF #define IGC_MDIC_REG_MASK 0x001F0000 @@ -489,18 +610,16 @@ #define IGC_MDIC_OP_WRITE 0x04000000 #define IGC_MDIC_OP_READ 0x08000000 #define IGC_MDIC_READY 0x10000000 -#define IGC_MDIC_INT_EN 0x20000000 #define IGC_MDIC_ERROR 0x40000000 -#define IGC_MDIC_DEST 0x80000000 #define IGC_N0_QUEUE -1 #define IGC_MAX_MAC_HDR_LEN 127 #define IGC_MAX_NETWORK_HDR_LEN 511 -#define IGC_VLAPQF_QUEUE_SEL(_n, q_idx) ((q_idx) << ((_n) * 4)) -#define IGC_VLAPQF_P_VALID(_n) (0x1 << (3 + (_n) * 4)) -#define IGC_VLAPQF_QUEUE_MASK 0x03 +#define IGC_VLANPQF_QSEL(_n, q_idx) ((q_idx) << ((_n) * 4)) +#define IGC_VLANPQF_VALID(_n) (0x1 << (3 + (_n) * 4)) +#define IGC_VLANPQF_QUEUE_MASK 0x03 #define IGC_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ #define IGC_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type:1=IPv4 */ @@ -510,4 +629,41 @@ /* Maximum size of the MTA register table in all supported adapters */ #define MAX_MTA_REG 128 +/* EEE defines */ +#define IGC_IPCNFG_EEE_2_5G_AN 0x00000010 /* IPCNFG EEE Ena 2.5G AN */ +#define IGC_IPCNFG_EEE_1G_AN 0x00000008 /* IPCNFG EEE Ena 1G AN */ +#define IGC_IPCNFG_EEE_100M_AN 0x00000004 /* IPCNFG EEE Ena 100M AN */ +#define IGC_EEER_EEE_NEG 0x20000000 /* EEE capability nego */ +#define IGC_EEER_TX_LPI_EN 0x00010000 /* EEER Tx LPI Enable */ +#define IGC_EEER_RX_LPI_EN 0x00020000 /* EEER Rx LPI Enable */ +#define IGC_EEER_LPI_FC 0x00040000 /* EEER Ena on Flow Cntrl */ +#define IGC_EEE_SU_LPI_CLK_STP 0x00800000 /* EEE LPI Clock Stop */ + +/* LTR defines */ +#define IGC_LTRC_EEEMS_EN 0x00000020 /* Enable EEE LTR max send */ +#define IGC_RXPBS_SIZE_I225_MASK 0x0000003F /* Rx packet buffer size */ +#define IGC_TW_SYSTEM_1000_MASK 0x000000FF +/* Minimum time for 100BASE-T where no data will be transmit following move out + * of EEE LPI Tx state + */ +#define IGC_TW_SYSTEM_100_MASK 0x0000FF00 +#define IGC_TW_SYSTEM_100_SHIFT 8 +#define IGC_DMACR_DMAC_EN 0x80000000 /* Enable DMA Coalescing */ +#define IGC_DMACR_DMACTHR_MASK 0x00FF0000 +#define IGC_DMACR_DMACTHR_SHIFT 16 +/* Reg val to set scale to 1024 nsec */ +#define IGC_LTRMINV_SCALE_1024 2 +/* Reg val to set scale to 32768 nsec */ +#define IGC_LTRMINV_SCALE_32768 3 +/* Reg val to set scale to 1024 nsec */ +#define IGC_LTRMAXV_SCALE_1024 2 +/* Reg val to set scale to 32768 nsec */ +#define IGC_LTRMAXV_SCALE_32768 3 +#define IGC_LTRMINV_LTRV_MASK 0x000003FF /* LTR minimum value */ +#define IGC_LTRMAXV_LTRV_MASK 0x000003FF /* LTR maximum value */ +#define IGC_LTRMINV_LSNP_REQ 0x00008000 /* LTR Snoop Requirement */ +#define IGC_LTRMINV_SCALE_SHIFT 10 +#define IGC_LTRMAXV_LSNP_REQ 0x00008000 /* LTR Snoop Requirement */ +#define IGC_LTRMAXV_SCALE_SHIFT 10 + #endif /* _IGC_DEFINES_H_ */ diff --git a/drivers/net/ethernet/intel/igc/igc_diag.c b/drivers/net/ethernet/intel/igc/igc_diag.c new file mode 100644 index 000000000000..cc621970c0cd --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_diag.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Intel Corporation */ + +#include "igc.h" +#include "igc_diag.h" + +static struct igc_reg_test reg_test[] = { + { IGC_FCAL, 1, PATTERN_TEST, 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_FCAH, 1, PATTERN_TEST, 0x0000FFFF, 0xFFFFFFFF }, + { IGC_FCT, 1, PATTERN_TEST, 0x0000FFFF, 0xFFFFFFFF }, + { IGC_RDBAH(0), 4, PATTERN_TEST, 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_RDBAL(0), 4, PATTERN_TEST, 0xFFFFFF80, 0xFFFFFF80 }, + { IGC_RDLEN(0), 4, PATTERN_TEST, 0x000FFF80, 0x000FFFFF }, + { IGC_RDT(0), 4, PATTERN_TEST, 0x0000FFFF, 0x0000FFFF }, + { IGC_FCRTH, 1, PATTERN_TEST, 0x0003FFF0, 0x0003FFF0 }, + { IGC_FCTTV, 1, PATTERN_TEST, 0x0000FFFF, 0x0000FFFF }, + { IGC_TIPG, 1, PATTERN_TEST, 0x3FFFFFFF, 0x3FFFFFFF }, + { IGC_TDBAH(0), 4, PATTERN_TEST, 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_TDBAL(0), 4, PATTERN_TEST, 0xFFFFFF80, 0xFFFFFF80 }, + { IGC_TDLEN(0), 4, PATTERN_TEST, 0x000FFF80, 0x000FFFFF }, + { IGC_TDT(0), 4, PATTERN_TEST, 0x0000FFFF, 0x0000FFFF }, + { IGC_RCTL, 1, SET_READ_TEST, 0xFFFFFFFF, 0x00000000 }, + { IGC_RCTL, 1, SET_READ_TEST, 0x04CFB2FE, 0x003FFFFB }, + { IGC_RCTL, 1, SET_READ_TEST, 0x04CFB2FE, 0xFFFFFFFF }, + { IGC_TCTL, 1, SET_READ_TEST, 0xFFFFFFFF, 0x00000000 }, + { IGC_RA, 16, TABLE64_TEST_LO, + 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_RA, 16, TABLE64_TEST_HI, + 0x900FFFFF, 0xFFFFFFFF }, + { IGC_MTA, 128, TABLE32_TEST, + 0xFFFFFFFF, 0xFFFFFFFF }, + { 0, 0, 0, 0} +}; + +static bool reg_pattern_test(struct igc_adapter *adapter, u64 *data, int reg, + u32 mask, u32 write) +{ + struct igc_hw *hw = &adapter->hw; + u32 pat, val, before; + static const u32 test_pattern[] = { + 0x5A5A5A5A, 0xA5A5A5A5, 0x00000000, 0xFFFFFFFF + }; + + for (pat = 0; pat < ARRAY_SIZE(test_pattern); pat++) { + before = rd32(reg); + wr32(reg, test_pattern[pat] & write); + val = rd32(reg); + if (val != (test_pattern[pat] & write & mask)) { + netdev_err(adapter->netdev, + "pattern test reg %04X failed: got 0x%08X expected 0x%08X", + reg, val, test_pattern[pat] & write & mask); + *data = reg; + wr32(reg, before); + return false; + } + wr32(reg, before); + } + return true; +} + +static bool reg_set_and_check(struct igc_adapter *adapter, u64 *data, int reg, + u32 mask, u32 write) +{ + struct igc_hw *hw = &adapter->hw; + u32 val, before; + + before = rd32(reg); + wr32(reg, write & mask); + val = rd32(reg); + if ((write & mask) != (val & mask)) { + netdev_err(adapter->netdev, + "set/check reg %04X test failed: got 0x%08X expected 0x%08X", + reg, (val & mask), (write & mask)); + *data = reg; + wr32(reg, before); + return false; + } + wr32(reg, before); + return true; +} + +bool igc_reg_test(struct igc_adapter *adapter, u64 *data) +{ + struct igc_reg_test *test = reg_test; + struct igc_hw *hw = &adapter->hw; + u32 value, before, after; + u32 i, toggle, b = false; + + /* Because the status register is such a special case, + * we handle it separately from the rest of the register + * tests. Some bits are read-only, some toggle, and some + * are writeable. + */ + toggle = 0x6800D3; + before = rd32(IGC_STATUS); + value = before & toggle; + wr32(IGC_STATUS, toggle); + after = rd32(IGC_STATUS) & toggle; + if (value != after) { + netdev_err(adapter->netdev, + "failed STATUS register test got: 0x%08X expected: 0x%08X", + after, value); + *data = 1; + return false; + } + /* restore previous status */ + wr32(IGC_STATUS, before); + + /* Perform the remainder of the register test, looping through + * the test table until we either fail or reach the null entry. + */ + while (test->reg) { + for (i = 0; i < test->array_len; i++) { + switch (test->test_type) { + case PATTERN_TEST: + b = reg_pattern_test(adapter, data, + test->reg + (i * 0x40), + test->mask, + test->write); + break; + case SET_READ_TEST: + b = reg_set_and_check(adapter, data, + test->reg + (i * 0x40), + test->mask, + test->write); + break; + case TABLE64_TEST_LO: + b = reg_pattern_test(adapter, data, + test->reg + (i * 8), + test->mask, + test->write); + break; + case TABLE64_TEST_HI: + b = reg_pattern_test(adapter, data, + test->reg + 4 + (i * 8), + test->mask, + test->write); + break; + case TABLE32_TEST: + b = reg_pattern_test(adapter, data, + test->reg + (i * 4), + test->mask, + test->write); + break; + } + if (!b) + return false; + } + test++; + } + *data = 0; + return true; +} + +bool igc_eeprom_test(struct igc_adapter *adapter, u64 *data) +{ + struct igc_hw *hw = &adapter->hw; + + *data = 0; + + if (hw->nvm.ops.validate(hw) != IGC_SUCCESS) { + *data = 1; + return false; + } + + return true; +} + +bool igc_link_test(struct igc_adapter *adapter, u64 *data) +{ + bool link_up; + + *data = 0; + + /* add delay to give enough time for autonegotioation to finish */ + if (adapter->hw.mac.autoneg) + ssleep(5); + + link_up = igc_has_link(adapter); + if (!link_up) { + *data = 1; + return false; + } + + return true; +} diff --git a/drivers/net/ethernet/intel/igc/igc_diag.h b/drivers/net/ethernet/intel/igc/igc_diag.h new file mode 100644 index 000000000000..600658e33bec --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_diag.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020 Intel Corporation */ + +bool igc_reg_test(struct igc_adapter *adapter, u64 *data); +bool igc_eeprom_test(struct igc_adapter *adapter, u64 *data); +bool igc_link_test(struct igc_adapter *adapter, u64 *data); + +struct igc_reg_test { + u16 reg; + u8 array_len; + u8 test_type; + u32 mask; + u32 write; +}; + +/* In the hardware, registers are laid out either singly, in arrays + * spaced 0x40 bytes apart, or in contiguous tables. We assume + * most tests take place on arrays or single registers (handled + * as a single-element array) and special-case the tables. + * Table tests are always pattern tests. + * + * We also make provision for some required setup steps by specifying + * registers to be written without any read-back testing. + */ + +#define PATTERN_TEST 1 +#define SET_READ_TEST 2 +#define TABLE32_TEST 3 +#define TABLE64_TEST_LO 4 +#define TABLE64_TEST_HI 5 diff --git a/drivers/net/ethernet/intel/igc/igc_dump.c b/drivers/net/ethernet/intel/igc/igc_dump.c new file mode 100644 index 000000000000..c09c95cc5f70 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_dump.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2018 Intel Corporation */ + +#include "igc.h" + +struct igc_reg_info { + u32 ofs; + char *name; +}; + +static const struct igc_reg_info igc_reg_info_tbl[] = { + /* General Registers */ + {IGC_CTRL, "CTRL"}, + {IGC_STATUS, "STATUS"}, + {IGC_CTRL_EXT, "CTRL_EXT"}, + {IGC_MDIC, "MDIC"}, + + /* Interrupt Registers */ + {IGC_ICR, "ICR"}, + + /* RX Registers */ + {IGC_RCTL, "RCTL"}, + {IGC_RDLEN(0), "RDLEN"}, + {IGC_RDH(0), "RDH"}, + {IGC_RDT(0), "RDT"}, + {IGC_RXDCTL(0), "RXDCTL"}, + {IGC_RDBAL(0), "RDBAL"}, + {IGC_RDBAH(0), "RDBAH"}, + + /* TX Registers */ + {IGC_TCTL, "TCTL"}, + {IGC_TDBAL(0), "TDBAL"}, + {IGC_TDBAH(0), "TDBAH"}, + {IGC_TDLEN(0), "TDLEN"}, + {IGC_TDH(0), "TDH"}, + {IGC_TDT(0), "TDT"}, + {IGC_TXDCTL(0), "TXDCTL"}, + + /* List Terminator */ + {} +}; + +/* igc_regdump - register printout routine */ +static void igc_regdump(struct igc_hw *hw, struct igc_reg_info *reginfo) +{ + struct net_device *dev = igc_get_hw_dev(hw); + int n = 0; + char rname[16]; + u32 regs[8]; + + switch (reginfo->ofs) { + case IGC_RDLEN(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_RDLEN(n)); + break; + case IGC_RDH(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_RDH(n)); + break; + case IGC_RDT(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_RDT(n)); + break; + case IGC_RXDCTL(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_RXDCTL(n)); + break; + case IGC_RDBAL(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_RDBAL(n)); + break; + case IGC_RDBAH(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_RDBAH(n)); + break; + case IGC_TDBAL(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_TDBAL(n)); + break; + case IGC_TDBAH(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_TDBAH(n)); + break; + case IGC_TDLEN(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_TDLEN(n)); + break; + case IGC_TDH(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_TDH(n)); + break; + case IGC_TDT(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_TDT(n)); + break; + case IGC_TXDCTL(0): + for (n = 0; n < 4; n++) + regs[n] = rd32(IGC_TXDCTL(n)); + break; + default: + netdev_info(dev, "%-15s %08x\n", reginfo->name, + rd32(reginfo->ofs)); + return; + } + + snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]"); + netdev_info(dev, "%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], + regs[2], regs[3]); +} + +/* igc_rings_dump - Tx-rings and Rx-rings */ +void igc_rings_dump(struct igc_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + struct my_u0 { __le64 a; __le64 b; } *u0; + union igc_adv_tx_desc *tx_desc; + union igc_adv_rx_desc *rx_desc; + struct igc_ring *tx_ring; + struct igc_ring *rx_ring; + u32 staterr; + u16 i, n; + + if (!netif_msg_hw(adapter)) + return; + + netdev_info(netdev, "Device info: state %016lX trans_start %016lX\n", + netdev->state, dev_trans_start(netdev)); + + /* Print TX Ring Summary */ + if (!netif_running(netdev)) + goto exit; + + netdev_info(netdev, "TX Rings Summary\n"); + netdev_info(netdev, "Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); + for (n = 0; n < adapter->num_tx_queues; n++) { + struct igc_tx_buffer *buffer_info; + + tx_ring = adapter->tx_ring[n]; + buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; + + netdev_info(netdev, "%5d %5X %5X %016llX %04X %p %016llX\n", + n, tx_ring->next_to_use, tx_ring->next_to_clean, + (u64)dma_unmap_addr(buffer_info, dma), + dma_unmap_len(buffer_info, len), + buffer_info->next_to_watch, + (u64)buffer_info->time_stamp); + } + + /* Print TX Rings */ + if (!netif_msg_tx_done(adapter)) + goto rx_ring_summary; + + netdev_info(netdev, "TX Rings Dump\n"); + + /* Transmit Descriptor Formats + * + * Advanced Transmit Descriptor + * +--------------------------------------------------------------+ + * 0 | Buffer Address [63:0] | + * +--------------------------------------------------------------+ + * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN | + * +--------------------------------------------------------------+ + * 63 46 45 40 39 38 36 35 32 31 24 15 0 + */ + + for (n = 0; n < adapter->num_tx_queues; n++) { + tx_ring = adapter->tx_ring[n]; + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "TX QUEUE INDEX = %d\n", + tx_ring->queue_index); + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n"); + + for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { + const char *next_desc; + struct igc_tx_buffer *buffer_info; + + tx_desc = IGC_TX_DESC(tx_ring, i); + buffer_info = &tx_ring->tx_buffer_info[i]; + u0 = (struct my_u0 *)tx_desc; + if (i == tx_ring->next_to_use && + i == tx_ring->next_to_clean) + next_desc = " NTC/U"; + else if (i == tx_ring->next_to_use) + next_desc = " NTU"; + else if (i == tx_ring->next_to_clean) + next_desc = " NTC"; + else + next_desc = ""; + + netdev_info(netdev, "T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n", + i, le64_to_cpu(u0->a), + le64_to_cpu(u0->b), + (u64)dma_unmap_addr(buffer_info, dma), + dma_unmap_len(buffer_info, len), + buffer_info->next_to_watch, + (u64)buffer_info->time_stamp, + buffer_info->skb, next_desc); + + if (netif_msg_pktdata(adapter) && buffer_info->skb) + print_hex_dump(KERN_INFO, "", + DUMP_PREFIX_ADDRESS, + 16, 1, buffer_info->skb->data, + dma_unmap_len(buffer_info, len), + true); + } + } + + /* Print RX Rings Summary */ +rx_ring_summary: + netdev_info(netdev, "RX Rings Summary\n"); + netdev_info(netdev, "Queue [NTU] [NTC]\n"); + for (n = 0; n < adapter->num_rx_queues; n++) { + rx_ring = adapter->rx_ring[n]; + netdev_info(netdev, "%5d %5X %5X\n", n, rx_ring->next_to_use, + rx_ring->next_to_clean); + } + + /* Print RX Rings */ + if (!netif_msg_rx_status(adapter)) + goto exit; + + netdev_info(netdev, "RX Rings Dump\n"); + + /* Advanced Receive Descriptor (Read) Format + * 63 1 0 + * +-----------------------------------------------------+ + * 0 | Packet Buffer Address [63:1] |A0/NSE| + * +----------------------------------------------+------+ + * 8 | Header Buffer Address [63:1] | DD | + * +-----------------------------------------------------+ + * + * + * Advanced Receive Descriptor (Write-Back) Format + * + * 63 48 47 32 31 30 21 20 17 16 4 3 0 + * +------------------------------------------------------+ + * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | + * | Checksum Ident | | | | Type | Type | + * +------------------------------------------------------+ + * 8 | VLAN Tag | Length | Extended Error | Extended Status | + * +------------------------------------------------------+ + * 63 48 47 32 31 20 19 0 + */ + + for (n = 0; n < adapter->num_rx_queues; n++) { + rx_ring = adapter->rx_ring[n]; + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "RX QUEUE INDEX = %d\n", + rx_ring->queue_index); + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); + netdev_info(netdev, "RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n"); + + for (i = 0; i < rx_ring->count; i++) { + const char *next_desc; + struct igc_rx_buffer *buffer_info; + + buffer_info = &rx_ring->rx_buffer_info[i]; + rx_desc = IGC_RX_DESC(rx_ring, i); + u0 = (struct my_u0 *)rx_desc; + staterr = le32_to_cpu(rx_desc->wb.upper.status_error); + + if (i == rx_ring->next_to_use) + next_desc = " NTU"; + else if (i == rx_ring->next_to_clean) + next_desc = " NTC"; + else + next_desc = ""; + + if (staterr & IGC_RXD_STAT_DD) { + /* Descriptor Done */ + netdev_info(netdev, "%s[0x%03X] %016llX %016llX ---------------- %s\n", + "RWB", i, + le64_to_cpu(u0->a), + le64_to_cpu(u0->b), + next_desc); + } else { + netdev_info(netdev, "%s[0x%03X] %016llX %016llX %016llX %s\n", + "R ", i, + le64_to_cpu(u0->a), + le64_to_cpu(u0->b), + (u64)buffer_info->dma, + next_desc); + + if (netif_msg_pktdata(adapter) && + buffer_info->dma && buffer_info->page) { + print_hex_dump(KERN_INFO, "", + DUMP_PREFIX_ADDRESS, + 16, 1, + page_address + (buffer_info->page) + + buffer_info->page_offset, + igc_rx_bufsz(rx_ring), + true); + } + } + } + } + +exit: + return; +} + +/* igc_regs_dump - registers dump */ +void igc_regs_dump(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + struct igc_reg_info *reginfo; + + /* Print Registers */ + netdev_info(adapter->netdev, "Register Dump\n"); + netdev_info(adapter->netdev, "Register Name Value\n"); + for (reginfo = (struct igc_reg_info *)igc_reg_info_tbl; + reginfo->name; reginfo++) { + igc_regdump(hw, reginfo); + } +} diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index ee07011e13e9..8cc077b712ad 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -4,8 +4,10 @@ /* ethtool support for igc */ #include <linux/if_vlan.h> #include <linux/pm_runtime.h> +#include <linux/mdio.h> #include "igc.h" +#include "igc_diag.h" /* forward declaration */ struct igc_stats { @@ -63,6 +65,8 @@ static const struct igc_stats igc_gstrings_stats[] = { IGC_STAT("tx_hwtstamp_timeouts", tx_hwtstamp_timeouts), IGC_STAT("tx_hwtstamp_skipped", tx_hwtstamp_skipped), IGC_STAT("rx_hwtstamp_cleared", rx_hwtstamp_cleared), + IGC_STAT("tx_lpi_counter", stats.tlpic), + IGC_STAT("rx_lpi_counter", stats.rlpic), }; #define IGC_NETDEV_STAT(_net_stat) { \ @@ -123,28 +127,44 @@ static const char igc_priv_flags_strings[][ETH_GSTRING_LEN] = { #define IGC_PRIV_FLAGS_STR_LEN ARRAY_SIZE(igc_priv_flags_strings) -static void igc_get_drvinfo(struct net_device *netdev, - struct ethtool_drvinfo *drvinfo) +static void igc_ethtool_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *drvinfo) { struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + u16 nvm_version = 0; + u16 gphy_version; + + strscpy(drvinfo->driver, igc_driver_name, sizeof(drvinfo->driver)); + + /* NVM image version is reported as firmware version for i225 device */ + hw->nvm.ops.read(hw, IGC_NVM_DEV_STARTER, 1, &nvm_version); + + /* gPHY firmware version is reported as PHY FW version */ + gphy_version = igc_read_phy_fw_version(hw); - strlcpy(drvinfo->driver, igc_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, igc_driver_version, sizeof(drvinfo->version)); + scnprintf(adapter->fw_version, + sizeof(adapter->fw_version), + "%x:%x", + nvm_version, + gphy_version); - /* add fw_version here */ - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->fw_version, adapter->fw_version, + sizeof(drvinfo->fw_version)); + + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); drvinfo->n_priv_flags = IGC_PRIV_FLAGS_STR_LEN; } -static int igc_get_regs_len(struct net_device *netdev) +static int igc_ethtool_get_regs_len(struct net_device *netdev) { return IGC_REGS_LEN * sizeof(u32); } -static void igc_get_regs(struct net_device *netdev, - struct ethtool_regs *regs, void *p) +static void igc_ethtool_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, void *p) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -153,7 +173,7 @@ static void igc_get_regs(struct net_device *netdev, memset(p, 0, IGC_REGS_LEN * sizeof(u32)); - regs->version = (1u << 24) | (hw->revision_id << 16) | hw->device_id; + regs->version = (2u << 24) | (hw->revision_id << 16) | hw->device_id; /* General Registers */ regs_buff[0] = rd32(IGC_CTRL); @@ -306,23 +326,101 @@ static void igc_get_regs(struct net_device *netdev, regs_buff[164 + i] = rd32(IGC_TDT(i)); for (i = 0; i < 4; i++) regs_buff[168 + i] = rd32(IGC_TXDCTL(i)); + + /* XXX: Due to a bug few lines above, RAL and RAH registers are + * overwritten. To preserve the ABI, we write these registers again in + * regs_buff. + */ + for (i = 0; i < 16; i++) + regs_buff[172 + i] = rd32(IGC_RAL(i)); + for (i = 0; i < 16; i++) + regs_buff[188 + i] = rd32(IGC_RAH(i)); + + regs_buff[204] = rd32(IGC_VLANPQF); + + for (i = 0; i < 8; i++) + regs_buff[205 + i] = rd32(IGC_ETQF(i)); + + regs_buff[213] = adapter->stats.tlpic; + regs_buff[214] = adapter->stats.rlpic; +} + +static void igc_ethtool_get_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + + wol->wolopts = 0; + + if (!(adapter->flags & IGC_FLAG_WOL_SUPPORTED)) + return; + + wol->supported = WAKE_UCAST | WAKE_MCAST | + WAKE_BCAST | WAKE_MAGIC | + WAKE_PHY; + + /* apply any specific unsupported masks here */ + switch (adapter->hw.device_id) { + default: + break; + } + + if (adapter->wol & IGC_WUFC_EX) + wol->wolopts |= WAKE_UCAST; + if (adapter->wol & IGC_WUFC_MC) + wol->wolopts |= WAKE_MCAST; + if (adapter->wol & IGC_WUFC_BC) + wol->wolopts |= WAKE_BCAST; + if (adapter->wol & IGC_WUFC_MAG) + wol->wolopts |= WAKE_MAGIC; + if (adapter->wol & IGC_WUFC_LNKC) + wol->wolopts |= WAKE_PHY; +} + +static int igc_ethtool_set_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + + if (wol->wolopts & (WAKE_ARP | WAKE_MAGICSECURE | WAKE_FILTER)) + return -EOPNOTSUPP; + + if (!(adapter->flags & IGC_FLAG_WOL_SUPPORTED)) + return wol->wolopts ? -EOPNOTSUPP : 0; + + /* these settings will always override what we currently have */ + adapter->wol = 0; + + if (wol->wolopts & WAKE_UCAST) + adapter->wol |= IGC_WUFC_EX; + if (wol->wolopts & WAKE_MCAST) + adapter->wol |= IGC_WUFC_MC; + if (wol->wolopts & WAKE_BCAST) + adapter->wol |= IGC_WUFC_BC; + if (wol->wolopts & WAKE_MAGIC) + adapter->wol |= IGC_WUFC_MAG; + if (wol->wolopts & WAKE_PHY) + adapter->wol |= IGC_WUFC_LNKC; + device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); + + return 0; } -static u32 igc_get_msglevel(struct net_device *netdev) +static u32 igc_ethtool_get_msglevel(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); return adapter->msg_enable; } -static void igc_set_msglevel(struct net_device *netdev, u32 data) +static void igc_ethtool_set_msglevel(struct net_device *netdev, u32 data) { struct igc_adapter *adapter = netdev_priv(netdev); adapter->msg_enable = data; } -static int igc_nway_reset(struct net_device *netdev) +static int igc_ethtool_nway_reset(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -331,7 +429,7 @@ static int igc_nway_reset(struct net_device *netdev) return 0; } -static u32 igc_get_link(struct net_device *netdev) +static u32 igc_ethtool_get_link(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_mac_info *mac = &adapter->hw.mac; @@ -348,15 +446,15 @@ static u32 igc_get_link(struct net_device *netdev) return igc_has_link(adapter); } -static int igc_get_eeprom_len(struct net_device *netdev) +static int igc_ethtool_get_eeprom_len(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); return adapter->hw.nvm.word_size * 2; } -static int igc_get_eeprom(struct net_device *netdev, - struct ethtool_eeprom *eeprom, u8 *bytes) +static int igc_ethtool_get_eeprom(struct net_device *netdev, + struct ethtool_eeprom *eeprom, u8 *bytes) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -402,8 +500,8 @@ static int igc_get_eeprom(struct net_device *netdev, return ret_val; } -static int igc_set_eeprom(struct net_device *netdev, - struct ethtool_eeprom *eeprom, u8 *bytes) +static int igc_ethtool_set_eeprom(struct net_device *netdev, + struct ethtool_eeprom *eeprom, u8 *bytes) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -456,7 +554,7 @@ static int igc_set_eeprom(struct net_device *netdev, memcpy(ptr, bytes, eeprom->len); for (i = 0; i < last_word - first_word + 1; i++) - eeprom_buff[i] = cpu_to_le16(eeprom_buff[i]); + cpu_to_le16s(&eeprom_buff[i]); ret_val = hw->nvm.ops.write(hw, first_word, last_word - first_word + 1, eeprom_buff); @@ -465,13 +563,15 @@ static int igc_set_eeprom(struct net_device *netdev, if (ret_val == 0) hw->nvm.ops.update(hw); - /* check if need: igc_set_fw_version(adapter); */ kfree(eeprom_buff); return ret_val; } -static void igc_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void +igc_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -481,8 +581,11 @@ static void igc_get_ringparam(struct net_device *netdev, ring->tx_pending = adapter->tx_ring_count; } -static int igc_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static int +igc_ethtool_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_ring *temp_ring; @@ -596,8 +699,8 @@ clear_reset: return err; } -static void igc_get_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *pause) +static void igc_ethtool_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -615,8 +718,8 @@ static void igc_get_pauseparam(struct net_device *netdev, } } -static int igc_set_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *pause) +static int igc_ethtool_set_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -655,7 +758,8 @@ static int igc_set_pauseparam(struct net_device *netdev, return retval; } -static void igc_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +static void igc_ethtool_get_strings(struct net_device *netdev, u32 stringset, + u8 *data) { struct igc_adapter *adapter = netdev_priv(netdev); u8 *p = data; @@ -667,35 +771,22 @@ static void igc_get_strings(struct net_device *netdev, u32 stringset, u8 *data) IGC_TEST_LEN * ETH_GSTRING_LEN); break; case ETH_SS_STATS: - for (i = 0; i < IGC_GLOBAL_STATS_LEN; i++) { - memcpy(p, igc_gstrings_stats[i].stat_string, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - for (i = 0; i < IGC_NETDEV_STATS_LEN; i++) { - memcpy(p, igc_gstrings_net_stats[i].stat_string, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } + for (i = 0; i < IGC_GLOBAL_STATS_LEN; i++) + ethtool_sprintf(&p, igc_gstrings_stats[i].stat_string); + for (i = 0; i < IGC_NETDEV_STATS_LEN; i++) + ethtool_sprintf(&p, + igc_gstrings_net_stats[i].stat_string); for (i = 0; i < adapter->num_tx_queues; i++) { - sprintf(p, "tx_queue_%u_packets", i); - p += ETH_GSTRING_LEN; - sprintf(p, "tx_queue_%u_bytes", i); - p += ETH_GSTRING_LEN; - sprintf(p, "tx_queue_%u_restart", i); - p += ETH_GSTRING_LEN; + ethtool_sprintf(&p, "tx_queue_%u_packets", i); + ethtool_sprintf(&p, "tx_queue_%u_bytes", i); + ethtool_sprintf(&p, "tx_queue_%u_restart", i); } for (i = 0; i < adapter->num_rx_queues; i++) { - sprintf(p, "rx_queue_%u_packets", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_bytes", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_drops", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_csum_err", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_alloc_failed", i); - p += ETH_GSTRING_LEN; + ethtool_sprintf(&p, "rx_queue_%u_packets", i); + ethtool_sprintf(&p, "rx_queue_%u_bytes", i); + ethtool_sprintf(&p, "rx_queue_%u_drops", i); + ethtool_sprintf(&p, "rx_queue_%u_csum_err", i); + ethtool_sprintf(&p, "rx_queue_%u_alloc_failed", i); } /* BUG_ON(p - data != IGC_STATS_LEN * ETH_GSTRING_LEN); */ break; @@ -706,7 +797,7 @@ static void igc_get_strings(struct net_device *netdev, u32 stringset, u8 *data) } } -static int igc_get_sset_count(struct net_device *netdev, int sset) +static int igc_ethtool_get_sset_count(struct net_device *netdev, int sset) { switch (sset) { case ETH_SS_STATS: @@ -720,7 +811,7 @@ static int igc_get_sset_count(struct net_device *netdev, int sset) } } -static void igc_get_ethtool_stats(struct net_device *netdev, +static void igc_ethtool_get_stats(struct net_device *netdev, struct ethtool_stats *stats, u64 *data) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -776,8 +867,10 @@ static void igc_get_ethtool_stats(struct net_device *netdev, spin_unlock(&adapter->stats64_lock); } -static int igc_get_coalesce(struct net_device *netdev, - struct ethtool_coalesce *ec) +static int igc_ethtool_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -796,33 +889,14 @@ static int igc_get_coalesce(struct net_device *netdev, return 0; } -static int igc_set_coalesce(struct net_device *netdev, - struct ethtool_coalesce *ec) +static int igc_ethtool_set_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); int i; - if (ec->rx_max_coalesced_frames || - ec->rx_coalesce_usecs_irq || - ec->rx_max_coalesced_frames_irq || - ec->tx_max_coalesced_frames || - ec->tx_coalesce_usecs_irq || - ec->stats_block_coalesce_usecs || - ec->use_adaptive_rx_coalesce || - ec->use_adaptive_tx_coalesce || - ec->pkt_rate_low || - ec->rx_coalesce_usecs_low || - ec->rx_max_coalesced_frames_low || - ec->tx_coalesce_usecs_low || - ec->tx_max_coalesced_frames_low || - ec->pkt_rate_high || - ec->rx_coalesce_usecs_high || - ec->rx_max_coalesced_frames_high || - ec->tx_coalesce_usecs_high || - ec->tx_max_coalesced_frames_high || - ec->rate_sample_interval) - return -ENOTSUPP; - if (ec->rx_coalesce_usecs > IGC_MAX_ITR_USECS || (ec->rx_coalesce_usecs > 3 && ec->rx_coalesce_usecs < IGC_MIN_ITR_USECS) || @@ -875,81 +949,89 @@ static int igc_set_coalesce(struct net_device *netdev, } #define ETHER_TYPE_FULL_MASK ((__force __be16)~0) -static int igc_get_ethtool_nfc_entry(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_get_nfc_rule(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = &cmd->fs; - struct igc_nfc_filter *rule = NULL; + struct igc_nfc_rule *rule = NULL; - /* report total rule count */ - cmd->data = IGC_MAX_RXNFC_FILTERS; + cmd->data = IGC_MAX_RXNFC_RULES; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - if (fsp->location <= rule->sw_idx) - break; + mutex_lock(&adapter->nfc_rule_lock); + + rule = igc_get_nfc_rule(adapter, fsp->location); + if (!rule) + goto out; + + fsp->flow_type = ETHER_FLOW; + fsp->ring_cookie = rule->action; + + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { + fsp->h_u.ether_spec.h_proto = htons(rule->filter.etype); + fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK; } - if (!rule || fsp->location != rule->sw_idx) - return -EINVAL; + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { + fsp->flow_type |= FLOW_EXT; + fsp->h_ext.vlan_tci = htons(rule->filter.vlan_tci); + fsp->m_ext.vlan_tci = htons(VLAN_PRIO_MASK); + } - if (rule->filter.match_flags) { - fsp->flow_type = ETHER_FLOW; - fsp->ring_cookie = rule->action; - if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { - fsp->h_u.ether_spec.h_proto = rule->filter.etype; - fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK; - } - if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { - fsp->flow_type |= FLOW_EXT; - fsp->h_ext.vlan_tci = rule->filter.vlan_tci; - fsp->m_ext.vlan_tci = htons(VLAN_PRIO_MASK); - } - if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { - ether_addr_copy(fsp->h_u.ether_spec.h_dest, - rule->filter.dst_addr); - /* As we only support matching by the full - * mask, return the mask to userspace - */ - eth_broadcast_addr(fsp->m_u.ether_spec.h_dest); - } - if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { - ether_addr_copy(fsp->h_u.ether_spec.h_source, - rule->filter.src_addr); - /* As we only support matching by the full - * mask, return the mask to userspace - */ - eth_broadcast_addr(fsp->m_u.ether_spec.h_source); - } + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { + ether_addr_copy(fsp->h_u.ether_spec.h_dest, + rule->filter.dst_addr); + eth_broadcast_addr(fsp->m_u.ether_spec.h_dest); + } - return 0; + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { + ether_addr_copy(fsp->h_u.ether_spec.h_source, + rule->filter.src_addr); + eth_broadcast_addr(fsp->m_u.ether_spec.h_source); + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA) { + fsp->flow_type |= FLOW_EXT; + memcpy(fsp->h_ext.data, rule->filter.user_data, sizeof(fsp->h_ext.data)); + memcpy(fsp->m_ext.data, rule->filter.user_mask, sizeof(fsp->m_ext.data)); } + + mutex_unlock(&adapter->nfc_rule_lock); + return 0; + +out: + mutex_unlock(&adapter->nfc_rule_lock); return -EINVAL; } -static int igc_get_ethtool_nfc_all(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd, - u32 *rule_locs) +static int igc_ethtool_get_nfc_rules(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd, + u32 *rule_locs) { - struct igc_nfc_filter *rule; + struct igc_nfc_rule *rule; int cnt = 0; - /* report total rule count */ - cmd->data = IGC_MAX_RXNFC_FILTERS; + cmd->data = IGC_MAX_RXNFC_RULES; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - if (cnt == cmd->rule_cnt) + mutex_lock(&adapter->nfc_rule_lock); + + list_for_each_entry(rule, &adapter->nfc_rule_list, list) { + if (cnt == cmd->rule_cnt) { + mutex_unlock(&adapter->nfc_rule_lock); return -EMSGSIZE; - rule_locs[cnt] = rule->sw_idx; + } + rule_locs[cnt] = rule->location; cnt++; } + mutex_unlock(&adapter->nfc_rule_lock); + cmd->rule_cnt = cnt; return 0; } -static int igc_get_rss_hash_opts(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_get_rss_hash_opts(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { cmd->data = 0; @@ -957,37 +1039,29 @@ static int igc_get_rss_hash_opts(struct igc_adapter *adapter, switch (cmd->flow_type) { case TCP_V4_FLOW: cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - /* Fall through */ + fallthrough; case UDP_V4_FLOW: if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - /* Fall through */ + fallthrough; case SCTP_V4_FLOW: - /* Fall through */ case AH_ESP_V4_FLOW: - /* Fall through */ case AH_V4_FLOW: - /* Fall through */ case ESP_V4_FLOW: - /* Fall through */ case IPV4_FLOW: cmd->data |= RXH_IP_SRC | RXH_IP_DST; break; case TCP_V6_FLOW: cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - /* Fall through */ + fallthrough; case UDP_V6_FLOW: if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - /* Fall through */ + fallthrough; case SCTP_V6_FLOW: - /* Fall through */ case AH_ESP_V6_FLOW: - /* Fall through */ case AH_V6_FLOW: - /* Fall through */ case ESP_V6_FLOW: - /* Fall through */ case IPV6_FLOW: cmd->data |= RXH_IP_SRC | RXH_IP_DST; break; @@ -998,41 +1072,33 @@ static int igc_get_rss_hash_opts(struct igc_adapter *adapter, return 0; } -static int igc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, - u32 *rule_locs) +static int igc_ethtool_get_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *cmd, u32 *rule_locs) { struct igc_adapter *adapter = netdev_priv(dev); - int ret = -EOPNOTSUPP; switch (cmd->cmd) { case ETHTOOL_GRXRINGS: cmd->data = adapter->num_rx_queues; - ret = 0; - break; + return 0; case ETHTOOL_GRXCLSRLCNT: - cmd->rule_cnt = adapter->nfc_filter_count; - ret = 0; - break; + cmd->rule_cnt = adapter->nfc_rule_count; + return 0; case ETHTOOL_GRXCLSRULE: - ret = igc_get_ethtool_nfc_entry(adapter, cmd); - break; + return igc_ethtool_get_nfc_rule(adapter, cmd); case ETHTOOL_GRXCLSRLALL: - ret = igc_get_ethtool_nfc_all(adapter, cmd, rule_locs); - break; + return igc_ethtool_get_nfc_rules(adapter, cmd, rule_locs); case ETHTOOL_GRXFH: - ret = igc_get_rss_hash_opts(adapter, cmd); - break; + return igc_ethtool_get_rss_hash_opts(adapter, cmd); default: - break; + return -EOPNOTSUPP; } - - return ret; } #define UDP_RSS_FLAGS (IGC_FLAG_RSS_FIELD_IPV4_UDP | \ IGC_FLAG_RSS_FIELD_IPV6_UDP) -static int igc_set_rss_hash_opt(struct igc_adapter *adapter, - struct ethtool_rxnfc *nfc) +static int igc_ethtool_set_rss_hash_opt(struct igc_adapter *adapter, + struct ethtool_rxnfc *nfc) { u32 flags = adapter->flags; @@ -1107,8 +1173,8 @@ static int igc_set_rss_hash_opt(struct igc_adapter *adapter, if ((flags & UDP_RSS_FLAGS) && !(adapter->flags & UDP_RSS_FLAGS)) - dev_err(&adapter->pdev->dev, - "enabling UDP RSS: fragmented packets may arrive out of order to the stack above\n"); + netdev_err(adapter->netdev, + "Enabling UDP RSS: fragmented packets may arrive out of order to the stack above\n"); adapter->flags = flags; @@ -1133,344 +1199,197 @@ static int igc_set_rss_hash_opt(struct igc_adapter *adapter, return 0; } -static int igc_rxnfc_write_etype_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input) +static void igc_ethtool_init_nfc_rule(struct igc_nfc_rule *rule, + const struct ethtool_rx_flow_spec *fsp) { - struct igc_hw *hw = &adapter->hw; - u8 i; - u32 etqf; - u16 etype; - - /* find an empty etype filter register */ - for (i = 0; i < MAX_ETYPE_FILTER; ++i) { - if (!adapter->etype_bitmap[i]) - break; - } - if (i == MAX_ETYPE_FILTER) { - dev_err(&adapter->pdev->dev, "ethtool -N: etype filters are all used.\n"); - return -EINVAL; - } - - adapter->etype_bitmap[i] = true; + INIT_LIST_HEAD(&rule->list); - etqf = rd32(IGC_ETQF(i)); - etype = ntohs(input->filter.etype & ETHER_TYPE_FULL_MASK); + rule->action = fsp->ring_cookie; + rule->location = fsp->location; - etqf |= IGC_ETQF_FILTER_ENABLE; - etqf &= ~IGC_ETQF_ETYPE_MASK; - etqf |= (etype & IGC_ETQF_ETYPE_MASK); - - etqf &= ~IGC_ETQF_QUEUE_MASK; - etqf |= ((input->action << IGC_ETQF_QUEUE_SHIFT) - & IGC_ETQF_QUEUE_MASK); - etqf |= IGC_ETQF_QUEUE_ENABLE; - - wr32(IGC_ETQF(i), etqf); - - input->etype_reg_index = i; - - return 0; -} - -static int igc_rxnfc_write_vlan_prio_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input) -{ - struct igc_hw *hw = &adapter->hw; - u8 vlan_priority; - u16 queue_index; - u32 vlapqf; - - vlapqf = rd32(IGC_VLAPQF); - vlan_priority = (ntohs(input->filter.vlan_tci) & VLAN_PRIO_MASK) - >> VLAN_PRIO_SHIFT; - queue_index = (vlapqf >> (vlan_priority * 4)) & IGC_VLAPQF_QUEUE_MASK; - - /* check whether this vlan prio is already set */ - if (vlapqf & IGC_VLAPQF_P_VALID(vlan_priority) && - queue_index != input->action) { - dev_err(&adapter->pdev->dev, "ethtool rxnfc set vlan prio filter failed.\n"); - return -EEXIST; + if ((fsp->flow_type & FLOW_EXT) && fsp->m_ext.vlan_tci) { + rule->filter.vlan_tci = ntohs(fsp->h_ext.vlan_tci); + rule->filter.match_flags |= IGC_FILTER_FLAG_VLAN_TCI; } - vlapqf |= IGC_VLAPQF_P_VALID(vlan_priority); - vlapqf |= IGC_VLAPQF_QUEUE_SEL(vlan_priority, input->action); - - wr32(IGC_VLAPQF, vlapqf); - - return 0; -} - -int igc_add_filter(struct igc_adapter *adapter, struct igc_nfc_filter *input) -{ - struct igc_hw *hw = &adapter->hw; - int err = -EINVAL; - - if (hw->mac.type == igc_i225 && - !(input->filter.match_flags & ~IGC_FILTER_FLAG_SRC_MAC_ADDR)) { - dev_err(&adapter->pdev->dev, - "i225 doesn't support flow classification rules specifying only source addresses.\n"); - return -EOPNOTSUPP; + if (fsp->m_u.ether_spec.h_proto == ETHER_TYPE_FULL_MASK) { + rule->filter.etype = ntohs(fsp->h_u.ether_spec.h_proto); + rule->filter.match_flags = IGC_FILTER_FLAG_ETHER_TYPE; } - if (input->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { - err = igc_rxnfc_write_etype_filter(adapter, input); - if (err) - return err; + /* Both source and destination address filters only support the full + * mask. + */ + if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_source)) { + rule->filter.match_flags |= IGC_FILTER_FLAG_SRC_MAC_ADDR; + ether_addr_copy(rule->filter.src_addr, + fsp->h_u.ether_spec.h_source); } - if (input->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { - err = igc_add_mac_steering_filter(adapter, - input->filter.dst_addr, - input->action, 0); - err = min_t(int, err, 0); - if (err) - return err; + if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_dest)) { + rule->filter.match_flags |= IGC_FILTER_FLAG_DST_MAC_ADDR; + ether_addr_copy(rule->filter.dst_addr, + fsp->h_u.ether_spec.h_dest); } - if (input->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { - err = igc_add_mac_steering_filter(adapter, - input->filter.src_addr, - input->action, - IGC_MAC_STATE_SRC_ADDR); - err = min_t(int, err, 0); - if (err) - return err; + /* VLAN etype matching */ + if ((fsp->flow_type & FLOW_EXT) && fsp->h_ext.vlan_etype) { + rule->filter.vlan_etype = fsp->h_ext.vlan_etype; + rule->filter.match_flags |= IGC_FILTER_FLAG_VLAN_ETYPE; } - if (input->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) - err = igc_rxnfc_write_vlan_prio_filter(adapter, input); - - return err; -} - -static void igc_clear_etype_filter_regs(struct igc_adapter *adapter, - u16 reg_index) -{ - struct igc_hw *hw = &adapter->hw; - u32 etqf = rd32(IGC_ETQF(reg_index)); - - etqf &= ~IGC_ETQF_QUEUE_ENABLE; - etqf &= ~IGC_ETQF_QUEUE_MASK; - etqf &= ~IGC_ETQF_FILTER_ENABLE; - - wr32(IGC_ETQF(reg_index), etqf); - - adapter->etype_bitmap[reg_index] = false; -} - -static void igc_clear_vlan_prio_filter(struct igc_adapter *adapter, - u16 vlan_tci) -{ - struct igc_hw *hw = &adapter->hw; - u8 vlan_priority; - u32 vlapqf; - - vlan_priority = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; - - vlapqf = rd32(IGC_VLAPQF); - vlapqf &= ~IGC_VLAPQF_P_VALID(vlan_priority); - vlapqf &= ~IGC_VLAPQF_QUEUE_SEL(vlan_priority, - IGC_VLAPQF_QUEUE_MASK); - - wr32(IGC_VLAPQF, vlapqf); -} - -int igc_erase_filter(struct igc_adapter *adapter, struct igc_nfc_filter *input) -{ - if (input->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) - igc_clear_etype_filter_regs(adapter, - input->etype_reg_index); - - if (input->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) - igc_clear_vlan_prio_filter(adapter, - ntohs(input->filter.vlan_tci)); - - if (input->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) - igc_del_mac_steering_filter(adapter, input->filter.src_addr, - input->action, - IGC_MAC_STATE_SRC_ADDR); - - if (input->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) - igc_del_mac_steering_filter(adapter, input->filter.dst_addr, - input->action, 0); + /* Check for user defined data */ + if ((fsp->flow_type & FLOW_EXT) && + (fsp->h_ext.data[0] || fsp->h_ext.data[1])) { + rule->filter.match_flags |= IGC_FILTER_FLAG_USER_DATA; + memcpy(rule->filter.user_data, fsp->h_ext.data, sizeof(fsp->h_ext.data)); + memcpy(rule->filter.user_mask, fsp->m_ext.data, sizeof(fsp->m_ext.data)); + } - return 0; + /* When multiple filter options or user data or vlan etype is set, use a + * flex filter. + */ + if ((rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA) || + (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE) || + (rule->filter.match_flags & (rule->filter.match_flags - 1))) + rule->flex = true; + else + rule->flex = false; } -static int igc_update_ethtool_nfc_entry(struct igc_adapter *adapter, - struct igc_nfc_filter *input, - u16 sw_idx) +/** + * igc_ethtool_check_nfc_rule() - Check if NFC rule is valid + * @adapter: Pointer to adapter + * @rule: Rule under evaluation + * + * The driver doesn't support rules with multiple matches so if more than + * one bit in filter flags is set, @rule is considered invalid. + * + * Also, if there is already another rule with the same filter in a different + * location, @rule is considered invalid. + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + * + * Return: 0 in case of success, negative errno code otherwise. + */ +static int igc_ethtool_check_nfc_rule(struct igc_adapter *adapter, + struct igc_nfc_rule *rule) { - struct igc_nfc_filter *rule, *parent; - int err = -EINVAL; - - parent = NULL; - rule = NULL; + struct net_device *dev = adapter->netdev; + u8 flags = rule->filter.match_flags; + struct igc_nfc_rule *tmp; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - /* hash found, or no matching entry */ - if (rule->sw_idx >= sw_idx) - break; - parent = rule; + if (!flags) { + netdev_dbg(dev, "Rule with no match\n"); + return -EINVAL; } - /* if there is an old rule occupying our place remove it */ - if (rule && rule->sw_idx == sw_idx) { - if (!input) - err = igc_erase_filter(adapter, rule); - - hlist_del(&rule->nfc_node); - kfree(rule); - adapter->nfc_filter_count--; + list_for_each_entry(tmp, &adapter->nfc_rule_list, list) { + if (!memcmp(&rule->filter, &tmp->filter, + sizeof(rule->filter)) && + tmp->location != rule->location) { + netdev_dbg(dev, "Rule already exists\n"); + return -EEXIST; + } } - /* If no input this was a delete, err should be 0 if a rule was - * successfully found and removed from the list else -EINVAL - */ - if (!input) - return err; - - /* initialize node */ - INIT_HLIST_NODE(&input->nfc_node); - - /* add filter to the list */ - if (parent) - hlist_add_behind(&input->nfc_node, &parent->nfc_node); - else - hlist_add_head(&input->nfc_node, &adapter->nfc_filter_list); - - /* update counts */ - adapter->nfc_filter_count++; - return 0; } -static int igc_add_ethtool_nfc_entry(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_add_nfc_rule(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { struct net_device *netdev = adapter->netdev; struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; - struct igc_nfc_filter *input, *rule; - int err = 0; + struct igc_nfc_rule *rule, *old_rule; + int err; - if (!(netdev->hw_features & NETIF_F_NTUPLE)) + if (!(netdev->hw_features & NETIF_F_NTUPLE)) { + netdev_dbg(netdev, "N-tuple filters disabled\n"); return -EOPNOTSUPP; - - /* Don't allow programming if the action is a queue greater than - * the number of online Rx queues. - */ - if (fsp->ring_cookie == RX_CLS_FLOW_DISC || - fsp->ring_cookie >= adapter->num_rx_queues) { - dev_err(&adapter->pdev->dev, "ethtool -N: The specified action is invalid\n"); - return -EINVAL; } - /* Don't allow indexes to exist outside of available space */ - if (fsp->location >= IGC_MAX_RXNFC_FILTERS) { - dev_err(&adapter->pdev->dev, "Location out of range\n"); - return -EINVAL; + if ((fsp->flow_type & ~FLOW_EXT) != ETHER_FLOW) { + netdev_dbg(netdev, "Only ethernet flow type is supported\n"); + return -EOPNOTSUPP; } - if ((fsp->flow_type & ~FLOW_EXT) != ETHER_FLOW) + if (fsp->ring_cookie >= adapter->num_rx_queues) { + netdev_dbg(netdev, "Invalid action\n"); return -EINVAL; - - input = kzalloc(sizeof(*input), GFP_KERNEL); - if (!input) - return -ENOMEM; - - if (fsp->m_u.ether_spec.h_proto == ETHER_TYPE_FULL_MASK) { - input->filter.etype = fsp->h_u.ether_spec.h_proto; - input->filter.match_flags = IGC_FILTER_FLAG_ETHER_TYPE; } - /* Only support matching addresses by the full mask */ - if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_source)) { - input->filter.match_flags |= IGC_FILTER_FLAG_SRC_MAC_ADDR; - ether_addr_copy(input->filter.src_addr, - fsp->h_u.ether_spec.h_source); + if (fsp->location >= IGC_MAX_RXNFC_RULES) { + netdev_dbg(netdev, "Invalid location\n"); + return -EINVAL; } - /* Only support matching addresses by the full mask */ - if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_dest)) { - input->filter.match_flags |= IGC_FILTER_FLAG_DST_MAC_ADDR; - ether_addr_copy(input->filter.dst_addr, - fsp->h_u.ether_spec.h_dest); - } + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return -ENOMEM; - if ((fsp->flow_type & FLOW_EXT) && fsp->m_ext.vlan_tci) { - if (fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK)) { - err = -EINVAL; - goto err_out; - } - input->filter.vlan_tci = fsp->h_ext.vlan_tci; - input->filter.match_flags |= IGC_FILTER_FLAG_VLAN_TCI; - } + igc_ethtool_init_nfc_rule(rule, fsp); - input->action = fsp->ring_cookie; - input->sw_idx = fsp->location; + mutex_lock(&adapter->nfc_rule_lock); - spin_lock(&adapter->nfc_lock); + err = igc_ethtool_check_nfc_rule(adapter, rule); + if (err) + goto err; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - if (!memcmp(&input->filter, &rule->filter, - sizeof(input->filter))) { - err = -EEXIST; - dev_err(&adapter->pdev->dev, - "ethtool: this filter is already set\n"); - goto err_out_w_lock; - } - } + old_rule = igc_get_nfc_rule(adapter, fsp->location); + if (old_rule) + igc_del_nfc_rule(adapter, old_rule); - err = igc_add_filter(adapter, input); + err = igc_add_nfc_rule(adapter, rule); if (err) - goto err_out_w_lock; - - igc_update_ethtool_nfc_entry(adapter, input, input->sw_idx); + goto err; - spin_unlock(&adapter->nfc_lock); + mutex_unlock(&adapter->nfc_rule_lock); return 0; -err_out_w_lock: - spin_unlock(&adapter->nfc_lock); -err_out: - kfree(input); +err: + mutex_unlock(&adapter->nfc_rule_lock); + kfree(rule); return err; } -static int igc_del_ethtool_nfc_entry(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_del_nfc_rule(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; - int err; + struct igc_nfc_rule *rule; - spin_lock(&adapter->nfc_lock); - err = igc_update_ethtool_nfc_entry(adapter, NULL, fsp->location); - spin_unlock(&adapter->nfc_lock); + mutex_lock(&adapter->nfc_rule_lock); - return err; + rule = igc_get_nfc_rule(adapter, fsp->location); + if (!rule) { + mutex_unlock(&adapter->nfc_rule_lock); + return -EINVAL; + } + + igc_del_nfc_rule(adapter, rule); + + mutex_unlock(&adapter->nfc_rule_lock); + return 0; } -static int igc_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +static int igc_ethtool_set_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *cmd) { struct igc_adapter *adapter = netdev_priv(dev); - int ret = -EOPNOTSUPP; switch (cmd->cmd) { case ETHTOOL_SRXFH: - ret = igc_set_rss_hash_opt(adapter, cmd); - break; + return igc_ethtool_set_rss_hash_opt(adapter, cmd); case ETHTOOL_SRXCLSRLINS: - ret = igc_add_ethtool_nfc_entry(adapter, cmd); - break; + return igc_ethtool_add_nfc_rule(adapter, cmd); case ETHTOOL_SRXCLSRLDEL: - ret = igc_del_ethtool_nfc_entry(adapter, cmd); + return igc_ethtool_del_nfc_rule(adapter, cmd); default: - break; + return -EOPNOTSUPP; } - - return ret; } void igc_write_rss_indir_tbl(struct igc_adapter *adapter) @@ -1495,13 +1414,13 @@ void igc_write_rss_indir_tbl(struct igc_adapter *adapter) } } -static u32 igc_get_rxfh_indir_size(struct net_device *netdev) +static u32 igc_ethtool_get_rxfh_indir_size(struct net_device *netdev) { return IGC_RETA_SIZE; } -static int igc_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int igc_ethtool_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, + u8 *hfunc) { struct igc_adapter *adapter = netdev_priv(netdev); int i; @@ -1516,8 +1435,8 @@ static int igc_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, return 0; } -static int igc_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int igc_ethtool_set_rxfh(struct net_device *netdev, const u32 *indir, + const u8 *key, const u8 hfunc) { struct igc_adapter *adapter = netdev_priv(netdev); u32 num_queues; @@ -1545,18 +1464,13 @@ static int igc_set_rxfh(struct net_device *netdev, const u32 *indir, return 0; } -static unsigned int igc_max_channels(struct igc_adapter *adapter) -{ - return igc_get_max_rss_queues(adapter); -} - -static void igc_get_channels(struct net_device *netdev, - struct ethtool_channels *ch) +static void igc_ethtool_get_channels(struct net_device *netdev, + struct ethtool_channels *ch) { struct igc_adapter *adapter = netdev_priv(netdev); /* Report maximum channels */ - ch->max_combined = igc_max_channels(adapter); + ch->max_combined = igc_get_max_rss_queues(adapter); /* Report info for other vector */ if (adapter->flags & IGC_FLAG_HAS_MSIX) { @@ -1567,8 +1481,8 @@ static void igc_get_channels(struct net_device *netdev, ch->combined_count = adapter->rss_queues; } -static int igc_set_channels(struct net_device *netdev, - struct ethtool_channels *ch) +static int igc_ethtool_set_channels(struct net_device *netdev, + struct ethtool_channels *ch) { struct igc_adapter *adapter = netdev_priv(netdev); unsigned int count = ch->combined_count; @@ -1583,7 +1497,7 @@ static int igc_set_channels(struct net_device *netdev, return -EINVAL; /* Verify the number of channels doesn't exceed hw limits */ - max_combined = igc_max_channels(adapter); + max_combined = igc_get_max_rss_queues(adapter); if (count > max_combined) return -EINVAL; @@ -1600,8 +1514,8 @@ static int igc_set_channels(struct net_device *netdev, return 0; } -static int igc_get_ts_info(struct net_device *dev, - struct ethtool_ts_info *info) +static int igc_ethtool_get_ts_info(struct net_device *dev, + struct ethtool_ts_info *info) { struct igc_adapter *adapter = netdev_priv(dev); @@ -1633,7 +1547,7 @@ static int igc_get_ts_info(struct net_device *dev, } } -static u32 igc_get_priv_flags(struct net_device *netdev) +static u32 igc_ethtool_get_priv_flags(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); u32 priv_flags = 0; @@ -1644,7 +1558,7 @@ static u32 igc_get_priv_flags(struct net_device *netdev) return priv_flags; } -static int igc_set_priv_flags(struct net_device *netdev, u32 priv_flags) +static int igc_ethtool_set_priv_flags(struct net_device *netdev, u32 priv_flags) { struct igc_adapter *adapter = netdev_priv(netdev); unsigned int flags = adapter->flags; @@ -1664,6 +1578,98 @@ static int igc_set_priv_flags(struct net_device *netdev, u32 priv_flags) return 0; } +static int igc_ethtool_get_eee(struct net_device *netdev, + struct ethtool_eee *edata) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + u32 eeer; + + if (hw->dev_spec._base.eee_enable) + edata->advertised = + mmd_eee_adv_to_ethtool_adv_t(adapter->eee_advert); + + *edata = adapter->eee; + edata->supported = SUPPORTED_Autoneg; + + eeer = rd32(IGC_EEER); + + /* EEE status on negotiated link */ + if (eeer & IGC_EEER_EEE_NEG) + edata->eee_active = true; + + if (eeer & IGC_EEER_TX_LPI_EN) + edata->tx_lpi_enabled = true; + + edata->eee_enabled = hw->dev_spec._base.eee_enable; + + edata->advertised = SUPPORTED_Autoneg; + edata->lp_advertised = SUPPORTED_Autoneg; + + /* Report correct negotiated EEE status for devices that + * wrongly report EEE at half-duplex + */ + if (adapter->link_duplex == HALF_DUPLEX) { + edata->eee_enabled = false; + edata->eee_active = false; + edata->tx_lpi_enabled = false; + edata->advertised &= ~edata->advertised; + } + + return 0; +} + +static int igc_ethtool_set_eee(struct net_device *netdev, + struct ethtool_eee *edata) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + struct ethtool_eee eee_curr; + s32 ret_val; + + memset(&eee_curr, 0, sizeof(struct ethtool_eee)); + + ret_val = igc_ethtool_get_eee(netdev, &eee_curr); + if (ret_val) { + netdev_err(netdev, + "Problem setting EEE advertisement options\n"); + return -EINVAL; + } + + if (eee_curr.eee_enabled) { + if (eee_curr.tx_lpi_enabled != edata->tx_lpi_enabled) { + netdev_err(netdev, + "Setting EEE tx-lpi is not supported\n"); + return -EINVAL; + } + + /* Tx LPI timer is not implemented currently */ + if (edata->tx_lpi_timer) { + netdev_err(netdev, + "Setting EEE Tx LPI timer is not supported\n"); + return -EINVAL; + } + } else if (!edata->eee_enabled) { + netdev_err(netdev, + "Setting EEE options are not supported with EEE disabled\n"); + return -EINVAL; + } + + adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised); + if (hw->dev_spec._base.eee_enable != edata->eee_enabled) { + hw->dev_spec._base.eee_enable = edata->eee_enabled; + adapter->flags |= IGC_FLAG_EEE; + + /* reset link */ + if (netif_running(netdev)) + igc_reinit_locked(adapter); + else + igc_reset(adapter); + } + + return 0; +} + static int igc_ethtool_begin(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -1679,8 +1685,8 @@ static void igc_ethtool_complete(struct net_device *netdev) pm_runtime_put(&adapter->pdev->dev); } -static int igc_get_link_ksettings(struct net_device *netdev, - struct ethtool_link_ksettings *cmd) +static int igc_ethtool_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -1703,12 +1709,18 @@ static int igc_get_link_ksettings(struct net_device *netdev, cmd->base.phy_address = hw->phy.addr; /* advertising link modes */ - ethtool_link_ksettings_add_link_mode(cmd, advertising, 10baseT_Half); - ethtool_link_ksettings_add_link_mode(cmd, advertising, 10baseT_Full); - ethtool_link_ksettings_add_link_mode(cmd, advertising, 100baseT_Half); - ethtool_link_ksettings_add_link_mode(cmd, advertising, 100baseT_Full); - ethtool_link_ksettings_add_link_mode(cmd, advertising, 1000baseT_Full); - ethtool_link_ksettings_add_link_mode(cmd, advertising, 2500baseT_Full); + if (hw->phy.autoneg_advertised & ADVERTISE_10_HALF) + ethtool_link_ksettings_add_link_mode(cmd, advertising, 10baseT_Half); + if (hw->phy.autoneg_advertised & ADVERTISE_10_FULL) + ethtool_link_ksettings_add_link_mode(cmd, advertising, 10baseT_Full); + if (hw->phy.autoneg_advertised & ADVERTISE_100_HALF) + ethtool_link_ksettings_add_link_mode(cmd, advertising, 100baseT_Half); + if (hw->phy.autoneg_advertised & ADVERTISE_100_FULL) + ethtool_link_ksettings_add_link_mode(cmd, advertising, 100baseT_Full); + if (hw->phy.autoneg_advertised & ADVERTISE_1000_FULL) + ethtool_link_ksettings_add_link_mode(cmd, advertising, 1000baseT_Full); + if (hw->phy.autoneg_advertised & ADVERTISE_2500_FULL) + ethtool_link_ksettings_add_link_mode(cmd, advertising, 2500baseT_Full); /* set autoneg settings */ if (hw->mac.autoneg == 1) { @@ -1717,6 +1729,9 @@ static int igc_get_link_ksettings(struct net_device *netdev, Autoneg); } + /* Set pause flow control settings */ + ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); + switch (hw->fc.requested_mode) { case igc_fc_full: ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause); @@ -1731,12 +1746,11 @@ static int igc_get_link_ksettings(struct net_device *netdev, Asym_Pause); break; default: - ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause); - ethtool_link_ksettings_add_link_mode(cmd, advertising, - Asym_Pause); + break; } - status = rd32(IGC_STATUS); + status = pm_runtime_suspended(&adapter->pdev->dev) ? + 0 : rd32(IGC_STATUS); if (status & IGC_STATUS_LU) { if (status & IGC_STATUS_SPEED_1000) { @@ -1786,10 +1800,12 @@ static int igc_get_link_ksettings(struct net_device *netdev, return 0; } -static int igc_set_link_ksettings(struct net_device *netdev, - const struct ethtool_link_ksettings *cmd) +static int +igc_ethtool_set_link_ksettings(struct net_device *netdev, + const struct ethtool_link_ksettings *cmd) { struct igc_adapter *adapter = netdev_priv(netdev); + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; u32 advertising; @@ -1797,8 +1813,7 @@ static int igc_set_link_ksettings(struct net_device *netdev, * cannot be changed */ if (igc_check_reset_block(hw)) { - dev_err(&adapter->pdev->dev, - "Cannot change link characteristics when reset is active.\n"); + netdev_err(dev, "Cannot change link characteristics when reset is active\n"); return -EINVAL; } @@ -1809,7 +1824,7 @@ static int igc_set_link_ksettings(struct net_device *netdev, if (cmd->base.eth_tp_mdix_ctrl) { if (cmd->base.eth_tp_mdix_ctrl != ETH_TP_MDI_AUTO && cmd->base.autoneg != AUTONEG_ENABLE) { - dev_err(&adapter->pdev->dev, "forcing MDI/MDI-X state is not supported when link speed and/or duplex are forced\n"); + netdev_err(dev, "Forcing MDI/MDI-X state is not supported when link speed and/or duplex are forced\n"); return -EINVAL; } } @@ -1819,6 +1834,12 @@ static int igc_set_link_ksettings(struct net_device *netdev, ethtool_convert_link_mode_to_legacy_u32(&advertising, cmd->link_modes.advertising); + /* Converting to legacy u32 drops ETHTOOL_LINK_MODE_2500baseT_Full_BIT. + * We have to check this and convert it to ADVERTISE_2500_FULL + * (aka ETHTOOL_LINK_MODE_2500baseX_Full_BIT) explicitly. + */ + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, 2500baseT_Full)) + advertising |= ADVERTISE_2500_FULL; if (cmd->base.autoneg == AUTONEG_ENABLE) { hw->mac.autoneg = 1; @@ -1826,9 +1847,7 @@ static int igc_set_link_ksettings(struct net_device *netdev, if (adapter->fc_autoneg) hw->fc.requested_mode = igc_fc_default; } else { - /* calling this overrides forced MDI setting */ - dev_info(&adapter->pdev->dev, - "Force mode currently not supported\n"); + netdev_info(dev, "Force mode currently not supported\n"); } /* MDI-X => 2; MDI => 1; Auto => 3 */ @@ -1855,43 +1874,107 @@ static int igc_set_link_ksettings(struct net_device *netdev, return 0; } +static void igc_ethtool_diag_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + bool if_running = netif_running(netdev); + + if (eth_test->flags == ETH_TEST_FL_OFFLINE) { + netdev_info(adapter->netdev, "Offline testing starting"); + set_bit(__IGC_TESTING, &adapter->state); + + /* Link test performed before hardware reset so autoneg doesn't + * interfere with test result + */ + if (!igc_link_test(adapter, &data[TEST_LINK])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (if_running) + igc_close(netdev); + else + igc_reset(adapter); + + netdev_info(adapter->netdev, "Register testing starting"); + if (!igc_reg_test(adapter, &data[TEST_REG])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + igc_reset(adapter); + + netdev_info(adapter->netdev, "EEPROM testing starting"); + if (!igc_eeprom_test(adapter, &data[TEST_EEP])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + igc_reset(adapter); + + /* loopback and interrupt tests + * will be implemented in the future + */ + data[TEST_LOOP] = 0; + data[TEST_IRQ] = 0; + + clear_bit(__IGC_TESTING, &adapter->state); + if (if_running) + igc_open(netdev); + } else { + netdev_info(adapter->netdev, "Online testing starting"); + + /* register, eeprom, intr and loopback tests not run online */ + data[TEST_REG] = 0; + data[TEST_EEP] = 0; + data[TEST_IRQ] = 0; + data[TEST_LOOP] = 0; + + if (!igc_link_test(adapter, &data[TEST_LINK])) + eth_test->flags |= ETH_TEST_FL_FAILED; + } + + msleep_interruptible(4 * 1000); +} + static const struct ethtool_ops igc_ethtool_ops = { - .get_drvinfo = igc_get_drvinfo, - .get_regs_len = igc_get_regs_len, - .get_regs = igc_get_regs, - .get_msglevel = igc_get_msglevel, - .set_msglevel = igc_set_msglevel, - .nway_reset = igc_nway_reset, - .get_link = igc_get_link, - .get_eeprom_len = igc_get_eeprom_len, - .get_eeprom = igc_get_eeprom, - .set_eeprom = igc_set_eeprom, - .get_ringparam = igc_get_ringparam, - .set_ringparam = igc_set_ringparam, - .get_pauseparam = igc_get_pauseparam, - .set_pauseparam = igc_set_pauseparam, - .get_strings = igc_get_strings, - .get_sset_count = igc_get_sset_count, - .get_ethtool_stats = igc_get_ethtool_stats, - .get_coalesce = igc_get_coalesce, - .set_coalesce = igc_set_coalesce, - .get_rxnfc = igc_get_rxnfc, - .set_rxnfc = igc_set_rxnfc, - .get_rxfh_indir_size = igc_get_rxfh_indir_size, - .get_rxfh = igc_get_rxfh, - .set_rxfh = igc_set_rxfh, - .get_ts_info = igc_get_ts_info, - .get_channels = igc_get_channels, - .set_channels = igc_set_channels, - .get_priv_flags = igc_get_priv_flags, - .set_priv_flags = igc_set_priv_flags, + .supported_coalesce_params = ETHTOOL_COALESCE_USECS, + .get_drvinfo = igc_ethtool_get_drvinfo, + .get_regs_len = igc_ethtool_get_regs_len, + .get_regs = igc_ethtool_get_regs, + .get_wol = igc_ethtool_get_wol, + .set_wol = igc_ethtool_set_wol, + .get_msglevel = igc_ethtool_get_msglevel, + .set_msglevel = igc_ethtool_set_msglevel, + .nway_reset = igc_ethtool_nway_reset, + .get_link = igc_ethtool_get_link, + .get_eeprom_len = igc_ethtool_get_eeprom_len, + .get_eeprom = igc_ethtool_get_eeprom, + .set_eeprom = igc_ethtool_set_eeprom, + .get_ringparam = igc_ethtool_get_ringparam, + .set_ringparam = igc_ethtool_set_ringparam, + .get_pauseparam = igc_ethtool_get_pauseparam, + .set_pauseparam = igc_ethtool_set_pauseparam, + .get_strings = igc_ethtool_get_strings, + .get_sset_count = igc_ethtool_get_sset_count, + .get_ethtool_stats = igc_ethtool_get_stats, + .get_coalesce = igc_ethtool_get_coalesce, + .set_coalesce = igc_ethtool_set_coalesce, + .get_rxnfc = igc_ethtool_get_rxnfc, + .set_rxnfc = igc_ethtool_set_rxnfc, + .get_rxfh_indir_size = igc_ethtool_get_rxfh_indir_size, + .get_rxfh = igc_ethtool_get_rxfh, + .set_rxfh = igc_ethtool_set_rxfh, + .get_ts_info = igc_ethtool_get_ts_info, + .get_channels = igc_ethtool_get_channels, + .set_channels = igc_ethtool_set_channels, + .get_priv_flags = igc_ethtool_get_priv_flags, + .set_priv_flags = igc_ethtool_set_priv_flags, + .get_eee = igc_ethtool_get_eee, + .set_eee = igc_ethtool_set_eee, .begin = igc_ethtool_begin, .complete = igc_ethtool_complete, - .get_link_ksettings = igc_get_link_ksettings, - .set_link_ksettings = igc_set_link_ksettings, + .get_link_ksettings = igc_ethtool_get_link_ksettings, + .set_link_ksettings = igc_ethtool_set_link_ksettings, + .self_test = igc_ethtool_diag_test, }; -void igc_set_ethtool_ops(struct net_device *netdev) +void igc_ethtool_set_ops(struct net_device *netdev) { netdev->ethtool_ops = &igc_ethtool_ops; } diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h index 90ac0e0144d8..88680e3d613d 100644 --- a/drivers/net/ethernet/intel/igc/igc_hw.h +++ b/drivers/net/ethernet/intel/igc/igc_hw.h @@ -21,6 +21,16 @@ #define IGC_DEV_ID_I225_I 0x15F8 #define IGC_DEV_ID_I220_V 0x15F7 #define IGC_DEV_ID_I225_K 0x3100 +#define IGC_DEV_ID_I225_K2 0x3101 +#define IGC_DEV_ID_I226_K 0x3102 +#define IGC_DEV_ID_I225_LMVP 0x5502 +#define IGC_DEV_ID_I226_LMVP 0x5503 +#define IGC_DEV_ID_I225_IT 0x0D9F +#define IGC_DEV_ID_I226_LM 0x125B +#define IGC_DEV_ID_I226_V 0x125C +#define IGC_DEV_ID_I226_IT 0x125D +#define IGC_DEV_ID_I221_V 0x125E +#define IGC_DEV_ID_I226_BLANK_NVM 0x125F #define IGC_DEV_ID_I225_BLANK_NVM 0x15FD /* Function pointers for the MAC. */ @@ -43,12 +53,6 @@ enum igc_mac_type { igc_num_macs /* List is 1-based, so subtract 1 for true count. */ }; -enum igc_phy_type { - igc_phy_unknown = 0, - igc_phy_none, - igc_phy_i225, -}; - enum igc_media_type { igc_media_type_unknown = 0, igc_media_type_copper = 1, @@ -58,8 +62,6 @@ enum igc_media_type { enum igc_nvm_type { igc_nvm_unknown = 0, igc_nvm_eeprom_spi, - igc_nvm_flash_hw, - igc_nvm_invm, }; struct igc_info { @@ -79,13 +81,7 @@ struct igc_mac_info { enum igc_mac_type type; - u32 collision_delta; - u32 ledctl_default; - u32 ledctl_mode1; - u32 ledctl_mode2; u32 mc_filter_type; - u32 tx_packet_delta; - u32 txcw; u16 mta_reg_count; u16 uta_reg_count; @@ -93,10 +89,6 @@ struct igc_mac_info { u32 mta_shadow[MAX_MTA_REG]; u16 rar_entry_count; - u8 forced_speed_duplex; - - bool adaptive_ifs; - bool has_fwsm; bool asf_firmware_present; bool arc_subsystem_valid; @@ -112,7 +104,6 @@ struct igc_nvm_operations { s32 (*write)(struct igc_hw *hw, u16 offset, u16 i, u16 *data); s32 (*update)(struct igc_hw *hw); s32 (*validate)(struct igc_hw *hw); - s32 (*valid_led_default)(struct igc_hw *hw, u16 *data); }; struct igc_phy_operations { @@ -130,9 +121,6 @@ struct igc_nvm_info { struct igc_nvm_operations ops; enum igc_nvm_type type; - u32 flash_bank_size; - u32 flash_base_addr; - u16 word_size; u16 delay_usec; u16 address_bits; @@ -143,8 +131,6 @@ struct igc_nvm_info { struct igc_phy_info { struct igc_phy_operations ops; - enum igc_phy_type type; - u32 addr; u32 id; u32 reset_delay_us; /* in usec */ @@ -158,7 +144,6 @@ struct igc_phy_info { u8 mdix; bool is_mdix; - bool reset_disable; bool speed_downgraded; bool autoneg_wait_to_complete; }; @@ -188,6 +173,7 @@ struct igc_fc_info { struct igc_dev_spec_base { bool clear_semaphore_once; + bool eee_enable; }; struct igc_hw { @@ -243,6 +229,8 @@ struct igc_hw_stats { u64 prc511; u64 prc1023; u64 prc1522; + u64 tlpic; + u64 rlpic; u64 gprc; u64 bprc; u64 mprc; @@ -272,21 +260,9 @@ struct igc_hw_stats { u64 tsctc; u64 tsctfc; u64 iac; - u64 icrxptc; - u64 icrxatc; - u64 ictxptc; - u64 ictxatc; - u64 ictxqec; - u64 ictxqmtc; - u64 icrxdmtc; - u64 icrxoc; - u64 cbtmpc; u64 htdpmc; - u64 cbrdpc; - u64 cbrmpc; u64 rpthc; u64 hgptc; - u64 htcbdpc; u64 hgorc; u64 hgotc; u64 lenerrs; diff --git a/drivers/net/ethernet/intel/igc/igc_i225.c b/drivers/net/ethernet/intel/igc/igc_i225.c index c25f555aaf82..59d5c467ea6e 100644 --- a/drivers/net/ethernet/intel/igc/igc_i225.c +++ b/drivers/net/ethernet/intel/igc/igc_i225.c @@ -6,7 +6,7 @@ #include "igc_hw.h" /** - * igc_get_hw_semaphore_i225 - Acquire hardware semaphore + * igc_acquire_nvm_i225 - Acquire exclusive access to EEPROM * @hw: pointer to the HW structure * * Acquire the necessary semaphores for exclusive access to the EEPROM. @@ -156,8 +156,15 @@ void igc_release_swfw_sync_i225(struct igc_hw *hw, u16 mask) { u32 swfw_sync; - while (igc_get_hw_semaphore_i225(hw)) - ; /* Empty */ + /* Releasing the resource requires first getting the HW semaphore. + * If we fail to get the semaphore, there is nothing we can do, + * except log an error and quit. We are not allowed to hang here + * indefinitely, as it may cause denial of service or system crash. + */ + if (igc_get_hw_semaphore_i225(hw)) { + hw_dbg("Failed to release SW_FW_SYNC.\n"); + return; + } swfw_sync = rd32(IGC_SW_FW_SYNC); swfw_sync &= ~mask; @@ -219,9 +226,9 @@ static s32 igc_write_nvm_srwr(struct igc_hw *hw, u16 offset, u16 words, u16 *data) { struct igc_nvm_info *nvm = &hw->nvm; + s32 ret_val = -IGC_ERR_NVM; u32 attempts = 100000; u32 i, k, eewr = 0; - s32 ret_val = 0; /* A check for invalid values: offset too large, too many words, * too many words for the offset, and not enough words. @@ -229,11 +236,11 @@ static s32 igc_write_nvm_srwr(struct igc_hw *hw, u16 offset, u16 words, if (offset >= nvm->word_size || (words > (nvm->word_size - offset)) || words == 0) { hw_dbg("nvm parameter(s) out of bounds\n"); - ret_val = -IGC_ERR_NVM; - goto out; + return ret_val; } for (i = 0; i < words; i++) { + ret_val = -IGC_ERR_NVM; eewr = ((offset + i) << IGC_NVM_RW_ADDR_SHIFT) | (data[i] << IGC_NVM_RW_REG_DATA) | IGC_NVM_RW_REG_START; @@ -255,7 +262,6 @@ static s32 igc_write_nvm_srwr(struct igc_hw *hw, u16 offset, u16 words, } } -out: return ret_val; } @@ -474,13 +480,11 @@ s32 igc_init_nvm_params_i225(struct igc_hw *hw) /* NVM Function Pointers */ if (igc_get_flash_presence_i225(hw)) { - hw->nvm.type = igc_nvm_flash_hw; nvm->ops.read = igc_read_nvm_srrd_i225; nvm->ops.write = igc_write_nvm_srwr_i225; nvm->ops.validate = igc_validate_nvm_checksum_i225; nvm->ops.update = igc_update_nvm_checksum_i225; } else { - hw->nvm.type = igc_nvm_invm; nvm->ops.read = igc_read_nvm_eerd; nvm->ops.write = NULL; nvm->ops.validate = NULL; @@ -488,3 +492,159 @@ s32 igc_init_nvm_params_i225(struct igc_hw *hw) } return 0; } + +/** + * igc_set_eee_i225 - Enable/disable EEE support + * @hw: pointer to the HW structure + * @adv2p5G: boolean flag enabling 2.5G EEE advertisement + * @adv1G: boolean flag enabling 1G EEE advertisement + * @adv100M: boolean flag enabling 100M EEE advertisement + * + * Enable/disable EEE based on setting in dev_spec structure. + **/ +s32 igc_set_eee_i225(struct igc_hw *hw, bool adv2p5G, bool adv1G, + bool adv100M) +{ + u32 ipcnfg, eeer; + + ipcnfg = rd32(IGC_IPCNFG); + eeer = rd32(IGC_EEER); + + /* enable or disable per user setting */ + if (hw->dev_spec._base.eee_enable) { + u32 eee_su = rd32(IGC_EEE_SU); + + if (adv100M) + ipcnfg |= IGC_IPCNFG_EEE_100M_AN; + else + ipcnfg &= ~IGC_IPCNFG_EEE_100M_AN; + + if (adv1G) + ipcnfg |= IGC_IPCNFG_EEE_1G_AN; + else + ipcnfg &= ~IGC_IPCNFG_EEE_1G_AN; + + if (adv2p5G) + ipcnfg |= IGC_IPCNFG_EEE_2_5G_AN; + else + ipcnfg &= ~IGC_IPCNFG_EEE_2_5G_AN; + + eeer |= (IGC_EEER_TX_LPI_EN | IGC_EEER_RX_LPI_EN | + IGC_EEER_LPI_FC); + + /* This bit should not be set in normal operation. */ + if (eee_su & IGC_EEE_SU_LPI_CLK_STP) + hw_dbg("LPI Clock Stop Bit should not be set!\n"); + } else { + ipcnfg &= ~(IGC_IPCNFG_EEE_2_5G_AN | IGC_IPCNFG_EEE_1G_AN | + IGC_IPCNFG_EEE_100M_AN); + eeer &= ~(IGC_EEER_TX_LPI_EN | IGC_EEER_RX_LPI_EN | + IGC_EEER_LPI_FC); + } + wr32(IGC_IPCNFG, ipcnfg); + wr32(IGC_EEER, eeer); + rd32(IGC_IPCNFG); + rd32(IGC_EEER); + + return IGC_SUCCESS; +} + +/* igc_set_ltr_i225 - Set Latency Tolerance Reporting thresholds + * @hw: pointer to the HW structure + * @link: bool indicating link status + * + * Set the LTR thresholds based on the link speed (Mbps), EEE, and DMAC + * settings, otherwise specify that there is no LTR requirement. + */ +s32 igc_set_ltr_i225(struct igc_hw *hw, bool link) +{ + u32 tw_system, ltrc, ltrv, ltr_min, ltr_max, scale_min, scale_max; + u16 speed, duplex; + s32 size; + + /* If we do not have link, LTR thresholds are zero. */ + if (link) { + hw->mac.ops.get_speed_and_duplex(hw, &speed, &duplex); + + /* Check if using copper interface with EEE enabled or if the + * link speed is 10 Mbps. + */ + if (hw->dev_spec._base.eee_enable && + speed != SPEED_10) { + /* EEE enabled, so send LTRMAX threshold. */ + ltrc = rd32(IGC_LTRC) | + IGC_LTRC_EEEMS_EN; + wr32(IGC_LTRC, ltrc); + + /* Calculate tw_system (nsec). */ + if (speed == SPEED_100) { + tw_system = ((rd32(IGC_EEE_SU) & + IGC_TW_SYSTEM_100_MASK) >> + IGC_TW_SYSTEM_100_SHIFT) * 500; + } else { + tw_system = (rd32(IGC_EEE_SU) & + IGC_TW_SYSTEM_1000_MASK) * 500; + } + } else { + tw_system = 0; + } + + /* Get the Rx packet buffer size. */ + size = rd32(IGC_RXPBS) & + IGC_RXPBS_SIZE_I225_MASK; + + /* Calculations vary based on DMAC settings. */ + if (rd32(IGC_DMACR) & IGC_DMACR_DMAC_EN) { + size -= (rd32(IGC_DMACR) & + IGC_DMACR_DMACTHR_MASK) >> + IGC_DMACR_DMACTHR_SHIFT; + /* Convert size to bits. */ + size *= 1024 * 8; + } else { + /* Convert size to bytes, subtract the MTU, and then + * convert the size to bits. + */ + size *= 1024; + size *= 8; + } + + if (size < 0) { + hw_dbg("Invalid effective Rx buffer size %d\n", + size); + return -IGC_ERR_CONFIG; + } + + /* Calculate the thresholds. Since speed is in Mbps, simplify + * the calculation by multiplying size/speed by 1000 for result + * to be in nsec before dividing by the scale in nsec. Set the + * scale such that the LTR threshold fits in the register. + */ + ltr_min = (1000 * size) / speed; + ltr_max = ltr_min + tw_system; + scale_min = (ltr_min / 1024) < 1024 ? IGC_LTRMINV_SCALE_1024 : + IGC_LTRMINV_SCALE_32768; + scale_max = (ltr_max / 1024) < 1024 ? IGC_LTRMAXV_SCALE_1024 : + IGC_LTRMAXV_SCALE_32768; + ltr_min /= scale_min == IGC_LTRMINV_SCALE_1024 ? 1024 : 32768; + ltr_min -= 1; + ltr_max /= scale_max == IGC_LTRMAXV_SCALE_1024 ? 1024 : 32768; + ltr_max -= 1; + + /* Only write the LTR thresholds if they differ from before. */ + ltrv = rd32(IGC_LTRMINV); + if (ltr_min != (ltrv & IGC_LTRMINV_LTRV_MASK)) { + ltrv = IGC_LTRMINV_LSNP_REQ | ltr_min | + (scale_min << IGC_LTRMINV_SCALE_SHIFT); + wr32(IGC_LTRMINV, ltrv); + } + + ltrv = rd32(IGC_LTRMAXV); + if (ltr_max != (ltrv & IGC_LTRMAXV_LTRV_MASK)) { + ltrv = IGC_LTRMAXV_LSNP_REQ | ltr_max | + (scale_max << IGC_LTRMAXV_SCALE_SHIFT); + wr32(IGC_LTRMAXV, ltrv); + } + } + + return IGC_SUCCESS; +} diff --git a/drivers/net/ethernet/intel/igc/igc_i225.h b/drivers/net/ethernet/intel/igc/igc_i225.h index 7b66e1f9c0e6..dae47e4f16b0 100644 --- a/drivers/net/ethernet/intel/igc/igc_i225.h +++ b/drivers/net/ethernet/intel/igc/igc_i225.h @@ -9,5 +9,8 @@ void igc_release_swfw_sync_i225(struct igc_hw *hw, u16 mask); s32 igc_init_nvm_params_i225(struct igc_hw *hw); bool igc_get_flash_presence_i225(struct igc_hw *hw); +s32 igc_set_eee_i225(struct igc_hw *hw, bool adv2p5G, bool adv1G, + bool adv100M); +s32 igc_set_ltr_i225(struct igc_hw *hw, bool link); #endif diff --git a/drivers/net/ethernet/intel/igc/igc_mac.c b/drivers/net/ethernet/intel/igc/igc_mac.c index 12aa6b5fcb5d..a5c4b19d71a2 100644 --- a/drivers/net/ethernet/intel/igc/igc_mac.c +++ b/drivers/net/ethernet/intel/igc/igc_mac.c @@ -193,7 +193,7 @@ s32 igc_force_mac_fc(struct igc_hw *hw) * 1: Rx flow control is enabled (we can receive pause * frames but not send pause frames). * 2: Tx flow control is enabled (we can send pause frames - * frames but we do not receive pause frames). + * but we do not receive pause frames). * 3: Both Rx and TX flow control (symmetric) is enabled. * other: No other values should be possible at this point. */ @@ -235,15 +235,14 @@ out: void igc_clear_hw_cntrs_base(struct igc_hw *hw) { rd32(IGC_CRCERRS); - rd32(IGC_SYMERRS); rd32(IGC_MPC); rd32(IGC_SCC); rd32(IGC_ECOL); rd32(IGC_MCC); rd32(IGC_LATECOL); rd32(IGC_COLC); + rd32(IGC_RERC); rd32(IGC_DC); - rd32(IGC_SEC); rd32(IGC_RLEC); rd32(IGC_XONRXC); rd32(IGC_XONTXC); @@ -288,31 +287,20 @@ void igc_clear_hw_cntrs_base(struct igc_hw *hw) rd32(IGC_ALGNERRC); rd32(IGC_RXERRC); rd32(IGC_TNCRS); - rd32(IGC_CEXTERR); + rd32(IGC_HTDPMC); rd32(IGC_TSCTC); - rd32(IGC_TSCTFC); rd32(IGC_MGTPRC); rd32(IGC_MGTPDC); rd32(IGC_MGTPTC); rd32(IGC_IAC); - rd32(IGC_ICRXOC); - - rd32(IGC_ICRXPTC); - rd32(IGC_ICRXATC); - rd32(IGC_ICTXPTC); - rd32(IGC_ICTXATC); - rd32(IGC_ICTXQEC); - rd32(IGC_ICTXQMTC); - rd32(IGC_ICRXDMTC); - rd32(IGC_CBTMPC); - rd32(IGC_HTDPMC); - rd32(IGC_CBRMPC); rd32(IGC_RPTHC); + rd32(IGC_TLPIC); + rd32(IGC_RLPIC); rd32(IGC_HGPTC); - rd32(IGC_HTCBDPC); + rd32(IGC_RXDMTC); rd32(IGC_HGORCL); rd32(IGC_HGORCH); rd32(IGC_HGOTCL); @@ -367,8 +355,8 @@ void igc_rar_set(struct igc_hw *hw, u8 *addr, u32 index) s32 igc_check_for_copper_link(struct igc_hw *hw) { struct igc_mac_info *mac = &hw->mac; + bool link = false; s32 ret_val; - bool link; /* We only want to go out to the PHY registers to see if Auto-Neg * has completed and/or if our link status has changed. The @@ -422,6 +410,11 @@ s32 igc_check_for_copper_link(struct igc_hw *hw) hw_dbg("Error configuring flow control\n"); out: + /* Now that we are aware of our link settings, we can set the LTR + * thresholds. + */ + ret_val = igc_set_ltr_i225(hw, link); + return ret_val; } @@ -467,10 +460,8 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw) * so we had to force link. In this case, we need to force the * configuration of the MAC to match the "fc" parameter. */ - if (mac->autoneg_failed) { - if (hw->phy.media_type == igc_media_type_copper) - ret_val = igc_force_mac_fc(hw); - } + if (mac->autoneg_failed) + ret_val = igc_force_mac_fc(hw); if (ret_val) { hw_dbg("Error forcing flow control settings\n"); @@ -482,7 +473,7 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw) * has completed, and if so, how the PHY and link partner has * flow control configured. */ - if (hw->phy.media_type == igc_media_type_copper && mac->autoneg) { + if (mac->autoneg) { /* Read the MII Status Register and check to see if AutoNeg * has completed. We read this twice because this reg has * some "sticky" (latched) bits. @@ -647,7 +638,7 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw) } out: - return 0; + return ret_val; } /** diff --git a/drivers/net/ethernet/intel/igc/igc_mac.h b/drivers/net/ethernet/intel/igc/igc_mac.h index 832cccec87cd..b5963f86defb 100644 --- a/drivers/net/ethernet/intel/igc/igc_mac.h +++ b/drivers/net/ethernet/intel/igc/igc_mac.h @@ -8,10 +8,6 @@ #include "igc_phy.h" #include "igc_defines.h" -#ifndef IGC_REMOVED -#define IGC_REMOVED(a) (0) -#endif /* IGC_REMOVED */ - /* forward declaration */ s32 igc_disable_pcie_master(struct igc_hw *hw); s32 igc_check_for_copper_link(struct igc_hw *hw); diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index d9d5425fe8d9..34889be63e78 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -9,28 +9,36 @@ #include <linux/udp.h> #include <linux/ip.h> #include <linux/pm_runtime.h> +#include <net/pkt_sched.h> +#include <linux/bpf_trace.h> +#include <net/xdp_sock_drv.h> +#include <linux/pci.h> #include <net/ipv6.h> #include "igc.h" #include "igc_hw.h" +#include "igc_tsn.h" +#include "igc_xdp.h" -#define DRV_VERSION "0.0.1-k" #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) +#define IGC_XDP_PASS 0 +#define IGC_XDP_CONSUMED BIT(0) +#define IGC_XDP_TX BIT(1) +#define IGC_XDP_REDIRECT BIT(2) + static int debug = -1; MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); MODULE_DESCRIPTION(DRV_SUMMARY); MODULE_LICENSE("GPL v2"); -MODULE_VERSION(DRV_VERSION); module_param(debug, int, 0); MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); char igc_driver_name[] = "igc"; -char igc_driver_version[] = DRV_VERSION; static const char igc_driver_string[] = DRV_SUMMARY; static const char igc_copyright[] = "Copyright(c) 2018 Intel Corporation."; @@ -45,6 +53,16 @@ static const struct pci_device_id igc_pci_tbl[] = { { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, /* required last entry */ {0, } @@ -59,19 +77,9 @@ enum latency_range { latency_invalid = 255 }; -/** - * igc_power_down_link - Power down the phy/serdes link - * @adapter: address of board private structure - */ -static void igc_power_down_link(struct igc_adapter *adapter) -{ - if (adapter->hw.phy.media_type == igc_media_type_copper) - igc_power_down_phy_copper_base(&adapter->hw); -} - void igc_reset(struct igc_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; struct igc_fc_info *fc = &hw->fc; u32 pba, hwm; @@ -98,14 +106,23 @@ void igc_reset(struct igc_adapter *adapter) hw->mac.ops.reset_hw(hw); if (hw->mac.ops.init_hw(hw)) - dev_err(&pdev->dev, "Hardware Error\n"); + netdev_err(dev, "Error on hardware initialization\n"); + + /* Re-establish EEE setting */ + igc_set_eee_i225(hw, true, true, true); if (!netif_running(adapter->netdev)) - igc_power_down_link(adapter); + igc_power_down_phy_copper_base(&adapter->hw); + + /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */ + wr32(IGC_VET, ETH_P_8021Q); /* Re-enable PTP, where applicable. */ igc_ptp_reset(adapter); + /* Re-enable TSN offloading, where applicable. */ + igc_tsn_reset(adapter); + igc_get_phy_info(hw); } @@ -117,8 +134,7 @@ static void igc_power_up_link(struct igc_adapter *adapter) { igc_reset_phy(&adapter->hw); - if (adapter->hw.phy.media_type == igc_media_type_copper) - igc_power_up_phy_copper(&adapter->hw); + igc_power_up_phy_copper(&adapter->hw); igc_setup_link(&adapter->hw); } @@ -136,6 +152,9 @@ static void igc_release_hw_control(struct igc_adapter *adapter) struct igc_hw *hw = &adapter->hw; u32 ctrl_ext; + if (!pci_device_is_present(adapter->pdev)) + return; + /* Let firmware take over control of h/w */ ctrl_ext = rd32(IGC_CTRL_EXT); wr32(IGC_CTRL_EXT, @@ -161,6 +180,14 @@ static void igc_get_hw_control(struct igc_adapter *adapter) ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); } +static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf) +{ + dma_unmap_single(dev, dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), DMA_TO_DEVICE); + + dma_unmap_len_set(buf, len, 0); +} + /** * igc_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned @@ -169,18 +196,27 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) { u16 i = tx_ring->next_to_clean; struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; + u32 xsk_frames = 0; while (i != tx_ring->next_to_use) { union igc_adv_tx_desc *eop_desc, *tx_desc; - /* Free all the Tx ring sk_buffs */ - dev_kfree_skb_any(tx_buffer->skb); - - /* unmap skb header data */ - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + switch (tx_buffer->type) { + case IGC_TX_BUFFER_TYPE_XSK: + xsk_frames++; + break; + case IGC_TX_BUFFER_TYPE_XDP: + xdp_return_frame(tx_buffer->xdpf); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + case IGC_TX_BUFFER_TYPE_SKB: + dev_kfree_skb_any(tx_buffer->skb); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + default: + netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); + break; + } /* check for eop_desc to determine the end of the packet */ eop_desc = tx_buffer->next_to_watch; @@ -199,12 +235,11 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); } + tx_buffer->next_to_watch = NULL; + /* move us one more past the eop_desc for start of next pkt */ tx_buffer++; i++; @@ -214,6 +249,9 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) } } + if (tx_ring->xsk_pool && xsk_frames) + xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); + /* reset BQL for queue */ netdev_tx_reset_queue(txring_txq(tx_ring)); @@ -280,6 +318,7 @@ static void igc_clean_all_tx_rings(struct igc_adapter *adapter) */ int igc_setup_tx_resources(struct igc_ring *tx_ring) { + struct net_device *ndev = tx_ring->netdev; struct device *dev = tx_ring->dev; int size = 0; @@ -305,8 +344,7 @@ int igc_setup_tx_resources(struct igc_ring *tx_ring) err: vfree(tx_ring->tx_buffer_info); - dev_err(dev, - "Unable to allocate memory for the transmit descriptor ring\n"); + netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); return -ENOMEM; } @@ -318,14 +356,13 @@ err: */ static int igc_setup_all_tx_resources(struct igc_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; int i, err = 0; for (i = 0; i < adapter->num_tx_queues; i++) { err = igc_setup_tx_resources(adapter->tx_ring[i]); if (err) { - dev_err(&pdev->dev, - "Allocation for Tx Queue %u failed\n", i); + netdev_err(dev, "Error on Tx queue %u setup\n", i); for (i--; i >= 0; i--) igc_free_tx_resources(adapter->tx_ring[i]); break; @@ -335,11 +372,7 @@ static int igc_setup_all_tx_resources(struct igc_adapter *adapter) return err; } -/** - * igc_clean_rx_ring - Free Rx Buffers per Queue - * @rx_ring: ring to free buffers from - */ -static void igc_clean_rx_ring(struct igc_ring *rx_ring) +static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring) { u16 i = rx_ring->next_to_clean; @@ -372,10 +405,39 @@ static void igc_clean_rx_ring(struct igc_ring *rx_ring) if (i == rx_ring->count) i = 0; } +} - rx_ring->next_to_alloc = 0; - rx_ring->next_to_clean = 0; - rx_ring->next_to_use = 0; +static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring) +{ + struct igc_rx_buffer *bi; + u16 i; + + for (i = 0; i < ring->count; i++) { + bi = &ring->rx_buffer_info[i]; + if (!bi->xdp) + continue; + + xsk_buff_free(bi->xdp); + bi->xdp = NULL; + } +} + +/** + * igc_clean_rx_ring - Free Rx Buffers per Queue + * @ring: ring to free buffers from + */ +static void igc_clean_rx_ring(struct igc_ring *ring) +{ + if (ring->xsk_pool) + igc_clean_rx_ring_xsk_pool(ring); + else + igc_clean_rx_ring_page_shared(ring); + + clear_ring_uses_large_buffer(ring); + + ring->next_to_alloc = 0; + ring->next_to_clean = 0; + ring->next_to_use = 0; } /** @@ -401,6 +463,8 @@ void igc_free_rx_resources(struct igc_ring *rx_ring) { igc_clean_rx_ring(rx_ring); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; @@ -436,8 +500,21 @@ static void igc_free_all_rx_resources(struct igc_adapter *adapter) */ int igc_setup_rx_resources(struct igc_ring *rx_ring) { + struct net_device *ndev = rx_ring->netdev; struct device *dev = rx_ring->dev; - int size, desc_len; + u8 index = rx_ring->queue_index; + int size, desc_len, res; + + /* XDP RX-queue info */ + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index, + rx_ring->q_vector->napi.napi_id); + if (res < 0) { + netdev_err(ndev, "Failed to register xdp_rxq index %u\n", + index); + return res; + } size = sizeof(struct igc_rx_buffer) * rx_ring->count; rx_ring->rx_buffer_info = vzalloc(size); @@ -463,10 +540,10 @@ int igc_setup_rx_resources(struct igc_ring *rx_ring) return 0; err: + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; - dev_err(dev, - "Unable to allocate memory for the receive descriptor ring\n"); + netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); return -ENOMEM; } @@ -479,14 +556,13 @@ err: */ static int igc_setup_all_rx_resources(struct igc_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; int i, err = 0; for (i = 0; i < adapter->num_rx_queues; i++) { err = igc_setup_rx_resources(adapter->rx_ring[i]); if (err) { - dev_err(&pdev->dev, - "Allocation for Rx Queue %u failed\n", i); + netdev_err(dev, "Error on Rx queue %u setup\n", i); for (i--; i >= 0; i--) igc_free_rx_resources(adapter->rx_ring[i]); break; @@ -496,6 +572,16 @@ static int igc_setup_all_rx_resources(struct igc_adapter *adapter) return err; } +static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter, + struct igc_ring *ring) +{ + if (!igc_xdp_is_enabled(adapter) || + !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags)) + return NULL; + + return xsk_get_pool_from_qid(ring->netdev, ring->queue_index); +} + /** * igc_configure_rx_ring - Configure a receive ring after Reset * @adapter: board private structure @@ -511,6 +597,23 @@ static void igc_configure_rx_ring(struct igc_adapter *adapter, int reg_idx = ring->reg_idx; u32 srrctl = 0, rxdctl = 0; u64 rdba = ring->dma; + u32 buf_size; + + xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + ring->xsk_pool = igc_get_xsk_pool(adapter, ring); + if (ring->xsk_pool) { + WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL)); + xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); + } else { + WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_PAGE_SHARED, + NULL)); + } + + if (igc_xdp_is_enabled(adapter)) + set_ring_uses_large_buffer(ring); /* disable the queue */ wr32(IGC_RXDCTL(reg_idx), 0); @@ -531,12 +634,15 @@ static void igc_configure_rx_ring(struct igc_adapter *adapter, ring->next_to_clean = 0; ring->next_to_use = 0; - /* set descriptor configuration */ - srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring_uses_large_buffer(ring)) - srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; + if (ring->xsk_pool) + buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool); + else if (ring_uses_large_buffer(ring)) + buf_size = IGC_RXBUFFER_3072; else - srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; + buf_size = IGC_RXBUFFER_2048; + + srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; + srrctl |= buf_size >> IGC_SRRCTL_BSIZEPKT_SHIFT; srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; wr32(IGC_SRRCTL(reg_idx), srrctl); @@ -591,6 +697,8 @@ static void igc_configure_tx_ring(struct igc_adapter *adapter, u64 tdba = ring->dma; u32 txdctl = 0; + ring->xsk_pool = igc_get_xsk_pool(adapter, ring); + /* disable the queue */ wr32(IGC_TXDCTL(reg_idx), 0); wrfl(); @@ -757,48 +865,76 @@ static void igc_setup_tctl(struct igc_adapter *adapter) } /** - * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table - * @adapter: address of board private structure - * @index: Index of the RAR entry which need to be synced with MAC table + * igc_set_mac_filter_hw() - Set MAC address filter in hardware + * @adapter: Pointer to adapter where the filter should be set + * @index: Filter index + * @type: MAC address filter type (source or destination) + * @addr: MAC address + * @queue: If non-negative, queue assignment feature is enabled and frames + * matching the filter are enqueued onto 'queue'. Otherwise, queue + * assignment is disabled. */ -static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) +static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, + enum igc_mac_filter_type type, + const u8 *addr, int queue) { - u8 *addr = adapter->mac_table[index].addr; + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; - u32 rar_low, rar_high; + u32 ral, rah; - /* HW expects these to be in network order when they are plugged - * into the registers which are little endian. In order to guarantee - * that ordering we need to do an leXX_to_cpup here in order to be - * ready for the byteswap that occurs with writel - */ - rar_low = le32_to_cpup((__le32 *)(addr)); - rar_high = le16_to_cpup((__le16 *)(addr + 4)); + if (WARN_ON(index >= hw->mac.rar_entry_count)) + return; - /* Indicate to hardware the Address is Valid. */ - if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { - if (is_valid_ether_addr(addr)) - rar_high |= IGC_RAH_AV; + ral = le32_to_cpup((__le32 *)(addr)); + rah = le16_to_cpup((__le16 *)(addr + 4)); - rar_high |= IGC_RAH_POOL_1 << - adapter->mac_table[index].queue; + if (type == IGC_MAC_FILTER_TYPE_SRC) { + rah &= ~IGC_RAH_ASEL_MASK; + rah |= IGC_RAH_ASEL_SRC_ADDR; } - wr32(IGC_RAL(index), rar_low); - wrfl(); - wr32(IGC_RAH(index), rar_high); - wrfl(); + if (queue >= 0) { + rah &= ~IGC_RAH_QSEL_MASK; + rah |= (queue << IGC_RAH_QSEL_SHIFT); + rah |= IGC_RAH_QSEL_ENABLE; + } + + rah |= IGC_RAH_AV; + + wr32(IGC_RAL(index), ral); + wr32(IGC_RAH(index), rah); + + netdev_dbg(dev, "MAC address filter set in HW: index %d", index); +} + +/** + * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware + * @adapter: Pointer to adapter where the filter should be cleared + * @index: Filter index + */ +static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) +{ + struct net_device *dev = adapter->netdev; + struct igc_hw *hw = &adapter->hw; + + if (WARN_ON(index >= hw->mac.rar_entry_count)) + return; + + wr32(IGC_RAL(index), 0); + wr32(IGC_RAH(index), 0); + + netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); } /* Set default MAC address for the PF in the first RAR entry */ static void igc_set_default_mac_filter(struct igc_adapter *adapter) { - struct igc_mac_addr *mac_table = &adapter->mac_table[0]; + struct net_device *dev = adapter->netdev; + u8 *addr = adapter->hw.mac.addr; - ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); - mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; + netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); - igc_rar_set_index(adapter, 0); + igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); } /** @@ -817,7 +953,7 @@ static int igc_set_mac(struct net_device *netdev, void *p) if (!is_valid_ether_addr(addr->sa_data)) return -EADDRNOTAVAIL; - memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); + eth_hw_addr_set(netdev, addr->sa_data); memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); /* set the correct pool for the new PF MAC address in entry 0 */ @@ -864,6 +1000,23 @@ static int igc_write_mc_addr_list(struct net_device *netdev) return netdev_mc_count(netdev); } +static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) +{ + ktime_t cycle_time = adapter->cycle_time; + ktime_t base_time = adapter->base_time; + u32 launchtime; + + /* FIXME: when using ETF together with taprio, we may have a + * case where 'delta' is larger than the cycle_time, this may + * cause problems if we don't read the current value of + * IGC_BASET, as the value writen into the launchtime + * descriptor field may be misinterpreted. + */ + div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); + + return cpu_to_le32(launchtime); +} + static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, struct igc_tx_buffer *first, u32 vlan_macip_lens, u32 type_tucmd, @@ -871,7 +1024,6 @@ static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, { struct igc_adv_tx_context_desc *context_desc; u16 i = tx_ring->next_to_use; - struct timespec64 ts; context_desc = IGC_TX_CTXTDESC(tx_ring, i); @@ -893,23 +1045,17 @@ static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, * should have been handled by the upper layers. */ if (tx_ring->launchtime_enable) { - ts = ktime_to_timespec64(first->skb->tstamp); - first->skb->tstamp = ktime_set(0, 0); - context_desc->launch_time = cpu_to_le32(ts.tv_nsec / 32); + struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); + ktime_t txtime = first->skb->tstamp; + + skb_txtime_consumed(first->skb); + context_desc->launch_time = igc_tx_launchtime(adapter, + txtime); } else { context_desc->launch_time = 0; } } -static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) -{ - unsigned int offset = 0; - - ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); - - return offset == skb_checksum_start_offset(skb); -} - static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) { struct sk_buff *skb = first->skb; @@ -927,19 +1073,16 @@ csum_failed: switch (skb->csum_offset) { case offsetof(struct tcphdr, check): type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; - /* fall through */ + fallthrough; case offsetof(struct udphdr, check): break; case offsetof(struct sctphdr, checksum): /* validate that this is actually an SCTP request */ - if ((first->protocol == htons(ETH_P_IP) && - (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || - (first->protocol == htons(ETH_P_IPV6) && - igc_ipv6_csum_is_sctp(skb))) { + if (skb_csum_is_sctp(skb)) { type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; break; } - /* fall through */ + fallthrough; default: skb_checksum_help(skb); goto csum_failed; @@ -1000,6 +1143,10 @@ static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DCMD_IFCS; + /* set HW vlan bit if vlan is present */ + cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN, + IGC_ADVTXD_DCMD_VLE); + /* set segmentation bits for TSO */ cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, (IGC_ADVTXD_DCMD_TSE)); @@ -1008,6 +1155,9 @@ static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, (IGC_ADVTXD_MAC_TSTAMP)); + /* insert frame checksum */ + cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS); + return cmd_type; } @@ -1042,8 +1192,9 @@ static int igc_tx_map(struct igc_ring *tx_ring, u16 i = tx_ring->next_to_use; unsigned int data_len, size; dma_addr_t dma; - u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); + u32 cmd_type; + cmd_type = igc_tx_cmd_type(skb, tx_flags); tx_desc = IGC_TX_DESC(tx_ring, i); igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); @@ -1143,17 +1294,13 @@ static int igc_tx_map(struct igc_ring *tx_ring, return 0; dma_error: - dev_err(tx_ring->dev, "TX DMA map failed\n"); + netdev_err(tx_ring->netdev, "TX DMA map failed\n"); tx_buffer = &tx_ring->tx_buffer_info[i]; /* clear dma mappings for failed tx_buffer_info map */ while (tx_buffer != first) { if (dma_unmap_len(tx_buffer, len)) - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); if (i-- == 0) i += tx_ring->count; @@ -1161,11 +1308,7 @@ dma_error: } if (dma_unmap_len(tx_buffer, len)) - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); dev_kfree_skb_any(tx_buffer->skb); tx_buffer->skb = NULL; @@ -1297,6 +1440,7 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, /* record the location of the first descriptor for this packet */ first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first->type = IGC_TX_BUFFER_TYPE_SKB; first->skb = skb; first->bytecount = skb->len; first->gso_segs = 1; @@ -1321,6 +1465,11 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, } } + if (skb_vlan_tag_present(skb)) { + tx_flags |= IGC_TX_FLAGS_VLAN; + tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT); + } + /* record initial flags and protocol */ first->tx_flags = tx_flags; first->protocol = protocol; @@ -1386,7 +1535,7 @@ static void igc_rx_checksum(struct igc_ring *ring, /* TCP/UDP checksum error bit is set */ if (igc_test_staterr(rx_desc, - IGC_RXDEXT_STATERR_TCPE | + IGC_RXDEXT_STATERR_L4E | IGC_RXDEXT_STATERR_IPE)) { /* work around errata with sctp packets where the TCPE aka * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) @@ -1406,8 +1555,8 @@ static void igc_rx_checksum(struct igc_ring *ring, IGC_RXD_STAT_UDPCS)) skb->ip_summed = CHECKSUM_UNNECESSARY; - dev_dbg(ring->dev, "cksum success: bits %08X\n", - le32_to_cpu(rx_desc->wb.upper.status_error)); + netdev_dbg(ring->netdev, "cksum success: bits %08X\n", + le32_to_cpu(rx_desc->wb.upper.status_error)); } static inline void igc_rx_hash(struct igc_ring *ring, @@ -1420,15 +1569,34 @@ static inline void igc_rx_hash(struct igc_ring *ring, PKT_HASH_TYPE_L3); } +static void igc_rx_vlan(struct igc_ring *rx_ring, + union igc_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + struct net_device *dev = rx_ring->netdev; + u16 vid; + + if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && + igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) { + if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) && + test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) + vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); + else + vid = le16_to_cpu(rx_desc->wb.upper.vlan); + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + } +} + /** * igc_process_skb_fields - Populate skb header fields from Rx descriptor * @rx_ring: rx descriptor ring packet is being transacted on * @rx_desc: pointer to the EOP Rx descriptor * @skb: pointer to current skb being populated * - * This function checks the ring, descriptor, and packet information in - * order to populate the hash, checksum, VLAN, timestamp, protocol, and - * other fields within the skb. + * This function checks the ring, descriptor, and packet information in order + * to populate the hash, checksum, VLAN, protocol, and other fields within the + * skb. */ static void igc_process_skb_fields(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc, @@ -1438,21 +1606,50 @@ static void igc_process_skb_fields(struct igc_ring *rx_ring, igc_rx_checksum(rx_ring, rx_desc, skb); - if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TS) && - !igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) - igc_ptp_rx_rgtstamp(rx_ring->q_vector, skb); + igc_rx_vlan(rx_ring, rx_desc, skb); skb_record_rx_queue(skb, rx_ring->queue_index); skb->protocol = eth_type_trans(skb, rx_ring->netdev); } +static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features) +{ + bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + u32 ctrl; + + ctrl = rd32(IGC_CTRL); + + if (enable) { + /* enable VLAN tag insert/strip */ + ctrl |= IGC_CTRL_VME; + } else { + /* disable VLAN tag insert/strip */ + ctrl &= ~IGC_CTRL_VME; + } + wr32(IGC_CTRL, ctrl); +} + +static void igc_restore_vlan(struct igc_adapter *adapter) +{ + igc_vlan_mode(adapter->netdev, adapter->netdev->features); +} + static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, - const unsigned int size) + const unsigned int size, + int *rx_buffer_pgcnt) { struct igc_rx_buffer *rx_buffer; rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + *rx_buffer_pgcnt = +#if (PAGE_SIZE < 8192) + page_count(rx_buffer->page); +#else + 0; +#endif prefetchw(rx_buffer->page); /* we are reusing so sync this buffer for CPU use */ @@ -1467,6 +1664,32 @@ static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, return rx_buffer; } +static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer, + unsigned int truesize) +{ +#if (PAGE_SIZE < 8192) + buffer->page_offset ^= truesize; +#else + buffer->page_offset += truesize; +#endif +} + +static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring, + unsigned int size) +{ + unsigned int truesize; + +#if (PAGE_SIZE < 8192) + truesize = igc_rx_pg_size(ring) / 2; +#else + truesize = ring_uses_build_skb(ring) ? + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(IGC_SKB_PAD + size) : + SKB_DATA_ALIGN(size); +#endif + return truesize; +} + /** * igc_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on @@ -1481,91 +1704,71 @@ static void igc_add_rx_frag(struct igc_ring *rx_ring, struct sk_buff *skb, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; + unsigned int truesize; - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); - rx_buffer->page_offset ^= truesize; +#if (PAGE_SIZE < 8192) + truesize = igc_rx_pg_size(rx_ring) / 2; #else - unsigned int truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IGC_SKB_PAD + size) : - SKB_DATA_ALIGN(size); + truesize = ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(IGC_SKB_PAD + size) : + SKB_DATA_ALIGN(size); +#endif skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); - rx_buffer->page_offset += truesize; -#endif + + igc_rx_buffer_flip(rx_buffer, truesize); } static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, - union igc_adv_rx_desc *rx_desc, - unsigned int size) + struct xdp_buff *xdp) { - void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; -#if (PAGE_SIZE < 8192) - unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(IGC_SKB_PAD + size); -#endif + unsigned int size = xdp->data_end - xdp->data; + unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); + unsigned int metasize = xdp->data - xdp->data_meta; struct sk_buff *skb; /* prefetch first cache line of first page */ - prefetch(va); -#if L1_CACHE_BYTES < 128 - prefetch(va + L1_CACHE_BYTES); -#endif + net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(va - IGC_SKB_PAD, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; /* update pointers within the skb to store the data */ - skb_reserve(skb, IGC_SKB_PAD); + skb_reserve(skb, xdp->data - xdp->data_hard_start); __skb_put(skb, size); + if (metasize) + skb_metadata_set(skb, metasize); - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - + igc_rx_buffer_flip(rx_buffer, truesize); return skb; } static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, - union igc_adv_rx_desc *rx_desc, - unsigned int size) + struct xdp_buff *xdp, + ktime_t timestamp) { - void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; -#if (PAGE_SIZE < 8192) - unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(size); -#endif + unsigned int metasize = xdp->data - xdp->data_meta; + unsigned int size = xdp->data_end - xdp->data; + unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); + void *va = xdp->data; unsigned int headlen; struct sk_buff *skb; /* prefetch first cache line of first page */ - prefetch(va); -#if L1_CACHE_BYTES < 128 - prefetch(va + L1_CACHE_BYTES); -#endif + net_prefetch(xdp->data_meta); /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); + skb = napi_alloc_skb(&rx_ring->q_vector->napi, + IGC_RX_HDR_LEN + metasize); if (unlikely(!skb)) return NULL; - if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { - igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); - va += IGC_TS_HDR_LEN; - size -= IGC_TS_HDR_LEN; - } + if (timestamp) + skb_hwtstamps(skb)->hwtstamp = timestamp; /* Determine available headroom for copy */ headlen = size; @@ -1573,7 +1776,13 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); /* align pull length to size of long to optimize memcpy performance */ - memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); + memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta, + ALIGN(headlen + metasize, sizeof(long))); + + if (metasize) { + skb_metadata_set(skb, metasize); + __skb_pull(skb, metasize); + } /* update all of the pointers */ size -= headlen; @@ -1581,11 +1790,7 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, skb_add_rx_frag(skb, 0, rx_buffer->page, (va + headlen) - page_address(rx_buffer->page), size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif + igc_rx_buffer_flip(rx_buffer, truesize); } else { rx_buffer->pagecnt_bias++; } @@ -1622,23 +1827,19 @@ static void igc_reuse_rx_page(struct igc_ring *rx_ring, new_buff->pagecnt_bias = old_buff->pagecnt_bias; } -static inline bool igc_page_is_reserved(struct page *page) -{ - return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); -} - -static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) +static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer, + int rx_buffer_pgcnt) { unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; struct page *page = rx_buffer->page; - /* avoid re-using remote pages */ - if (unlikely(igc_page_is_reserved(page))) + /* avoid re-using remote and pfmemalloc pages */ + if (!dev_page_is_reusable(page)) return false; #if (PAGE_SIZE < 8192) /* if we are only owner of page we can reuse it */ - if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) + if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) return false; #else #define IGC_LAST_OFFSET \ @@ -1652,8 +1853,8 @@ static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) * the pagecnt_bias and page count so that we fully restock the * number of references the driver holds. */ - if (unlikely(!pagecnt_bias)) { - page_ref_add(page, USHRT_MAX); + if (unlikely(pagecnt_bias == 1)) { + page_ref_add(page, USHRT_MAX - 1); rx_buffer->pagecnt_bias = USHRT_MAX; } @@ -1705,8 +1906,11 @@ static bool igc_cleanup_headers(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { - if (unlikely((igc_test_staterr(rx_desc, - IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { + /* XDP packets use error pointer so abort at this point */ + if (IS_ERR(skb)) + return true; + + if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { struct net_device *netdev = rx_ring->netdev; if (!(netdev->features & NETIF_F_RXALL)) { @@ -1723,9 +1927,10 @@ static bool igc_cleanup_headers(struct igc_ring *rx_ring, } static void igc_put_rx_buffer(struct igc_ring *rx_ring, - struct igc_rx_buffer *rx_buffer) + struct igc_rx_buffer *rx_buffer, + int rx_buffer_pgcnt) { - if (igc_can_reuse_rx_page(rx_buffer)) { + if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { /* hand second half of page back to the ring */ igc_reuse_rx_page(rx_ring, rx_buffer); } else { @@ -1745,7 +1950,14 @@ static void igc_put_rx_buffer(struct igc_ring *rx_ring, static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) { - return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; + struct igc_adapter *adapter = rx_ring->q_vector->adapter; + + if (ring_uses_build_skb(rx_ring)) + return IGC_SKB_PAD; + if (igc_xdp_is_enabled(adapter)) + return XDP_PACKET_HEADROOM; + + return 0; } static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, @@ -1784,7 +1996,8 @@ static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, bi->dma = dma; bi->page = page; bi->page_offset = igc_rx_offset(rx_ring); - bi->pagecnt_bias = 1; + page_ref_add(page, USHRT_MAX - 1); + bi->pagecnt_bias = USHRT_MAX; return true; } @@ -1859,17 +2072,305 @@ static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) } } +static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) +{ + union igc_adv_rx_desc *desc; + u16 i = ring->next_to_use; + struct igc_rx_buffer *bi; + dma_addr_t dma; + bool ok = true; + + if (!count) + return ok; + + desc = IGC_RX_DESC(ring, i); + bi = &ring->rx_buffer_info[i]; + i -= ring->count; + + do { + bi->xdp = xsk_buff_alloc(ring->xsk_pool); + if (!bi->xdp) { + ok = false; + break; + } + + dma = xsk_buff_xdp_get_dma(bi->xdp); + desc->read.pkt_addr = cpu_to_le64(dma); + + desc++; + bi++; + i++; + if (unlikely(!i)) { + desc = IGC_RX_DESC(ring, 0); + bi = ring->rx_buffer_info; + i -= ring->count; + } + + /* Clear the length for the next_to_use descriptor. */ + desc->wb.upper.length = 0; + + count--; + } while (count); + + i += ring->count; + + if (ring->next_to_use != i) { + ring->next_to_use = i; + + /* Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. (Only + * applicable for weak-ordered memory model archs, + * such as IA-64). + */ + wmb(); + writel(i, ring->tail); + } + + return ok; +} + +/* This function requires __netif_tx_lock is held by the caller. */ +static int igc_xdp_init_tx_descriptor(struct igc_ring *ring, + struct xdp_frame *xdpf) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); + u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; + u16 count, index = ring->next_to_use; + struct igc_tx_buffer *head = &ring->tx_buffer_info[index]; + struct igc_tx_buffer *buffer = head; + union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index); + u32 olinfo_status, len = xdpf->len, cmd_type; + void *data = xdpf->data; + u16 i; + + count = TXD_USE_COUNT(len); + for (i = 0; i < nr_frags; i++) + count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); + + if (igc_maybe_stop_tx(ring, count + 3)) { + /* this is a hard error */ + return -EBUSY; + } + + i = 0; + head->bytecount = xdp_get_frame_len(xdpf); + head->type = IGC_TX_BUFFER_TYPE_XDP; + head->gso_segs = 1; + head->xdpf = xdpf; + + olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; + desc->read.olinfo_status = cpu_to_le32(olinfo_status); + + for (;;) { + dma_addr_t dma; + + dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); + if (dma_mapping_error(ring->dev, dma)) { + netdev_err_once(ring->netdev, + "Failed to map DMA for TX\n"); + goto unmap; + } + + dma_unmap_len_set(buffer, len, len); + dma_unmap_addr_set(buffer, dma, dma); + + cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | + IGC_ADVTXD_DCMD_IFCS | len; + + desc->read.cmd_type_len = cpu_to_le32(cmd_type); + desc->read.buffer_addr = cpu_to_le64(dma); + + buffer->protocol = 0; + + if (++index == ring->count) + index = 0; + + if (i == nr_frags) + break; + + buffer = &ring->tx_buffer_info[index]; + desc = IGC_TX_DESC(ring, index); + desc->read.olinfo_status = 0; + + data = skb_frag_address(&sinfo->frags[i]); + len = skb_frag_size(&sinfo->frags[i]); + i++; + } + desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD); + + netdev_tx_sent_queue(txring_txq(ring), head->bytecount); + /* set the timestamp */ + head->time_stamp = jiffies; + /* set next_to_watch value indicating a packet is present */ + head->next_to_watch = desc; + ring->next_to_use = index; + + return 0; + +unmap: + for (;;) { + buffer = &ring->tx_buffer_info[index]; + if (dma_unmap_len(buffer, len)) + dma_unmap_page(ring->dev, + dma_unmap_addr(buffer, dma), + dma_unmap_len(buffer, len), + DMA_TO_DEVICE); + dma_unmap_len_set(buffer, len, 0); + if (buffer == head) + break; + + if (!index) + index += ring->count; + index--; + } + + return -ENOMEM; +} + +static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter, + int cpu) +{ + int index = cpu; + + if (unlikely(index < 0)) + index = 0; + + while (index >= adapter->num_tx_queues) + index -= adapter->num_tx_queues; + + return adapter->tx_ring[index]; +} + +static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp) +{ + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); + int cpu = smp_processor_id(); + struct netdev_queue *nq; + struct igc_ring *ring; + int res; + + if (unlikely(!xdpf)) + return -EFAULT; + + ring = igc_xdp_get_tx_ring(adapter, cpu); + nq = txring_txq(ring); + + __netif_tx_lock(nq, cpu); + res = igc_xdp_init_tx_descriptor(ring, xdpf); + __netif_tx_unlock(nq); + return res; +} + +/* This function assumes rcu_read_lock() is held by the caller. */ +static int __igc_xdp_run_prog(struct igc_adapter *adapter, + struct bpf_prog *prog, + struct xdp_buff *xdp) +{ + u32 act = bpf_prog_run_xdp(prog, xdp); + + switch (act) { + case XDP_PASS: + return IGC_XDP_PASS; + case XDP_TX: + if (igc_xdp_xmit_back(adapter, xdp) < 0) + goto out_failure; + return IGC_XDP_TX; + case XDP_REDIRECT: + if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0) + goto out_failure; + return IGC_XDP_REDIRECT; + break; + default: + bpf_warn_invalid_xdp_action(adapter->netdev, prog, act); + fallthrough; + case XDP_ABORTED: +out_failure: + trace_xdp_exception(adapter->netdev, prog, act); + fallthrough; + case XDP_DROP: + return IGC_XDP_CONSUMED; + } +} + +static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter, + struct xdp_buff *xdp) +{ + struct bpf_prog *prog; + int res; + + prog = READ_ONCE(adapter->xdp_prog); + if (!prog) { + res = IGC_XDP_PASS; + goto out; + } + + res = __igc_xdp_run_prog(adapter, prog, xdp); + +out: + return ERR_PTR(-res); +} + +/* This function assumes __netif_tx_lock is held by the caller. */ +static void igc_flush_tx_descriptors(struct igc_ring *ring) +{ + /* Once tail pointer is updated, hardware can fetch the descriptors + * any time so we issue a write membar here to ensure all memory + * writes are complete before the tail pointer is updated. + */ + wmb(); + writel(ring->next_to_use, ring->tail); +} + +static void igc_finalize_xdp(struct igc_adapter *adapter, int status) +{ + int cpu = smp_processor_id(); + struct netdev_queue *nq; + struct igc_ring *ring; + + if (status & IGC_XDP_TX) { + ring = igc_xdp_get_tx_ring(adapter, cpu); + nq = txring_txq(ring); + + __netif_tx_lock(nq, cpu); + igc_flush_tx_descriptors(ring); + __netif_tx_unlock(nq); + } + + if (status & IGC_XDP_REDIRECT) + xdp_do_flush(); +} + +static void igc_update_rx_stats(struct igc_q_vector *q_vector, + unsigned int packets, unsigned int bytes) +{ + struct igc_ring *ring = q_vector->rx.ring; + + u64_stats_update_begin(&ring->rx_syncp); + ring->rx_stats.packets += packets; + ring->rx_stats.bytes += bytes; + u64_stats_update_end(&ring->rx_syncp); + + q_vector->rx.total_packets += packets; + q_vector->rx.total_bytes += bytes; +} + static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) { unsigned int total_bytes = 0, total_packets = 0; + struct igc_adapter *adapter = q_vector->adapter; struct igc_ring *rx_ring = q_vector->rx.ring; struct sk_buff *skb = rx_ring->skb; u16 cleaned_count = igc_desc_unused(rx_ring); + int xdp_status = 0, rx_buffer_pgcnt; while (likely(total_packets < budget)) { union igc_adv_rx_desc *rx_desc; struct igc_rx_buffer *rx_buffer; - unsigned int size; + unsigned int size, truesize; + ktime_t timestamp = 0; + struct xdp_buff xdp; + int pkt_offset = 0; + void *pktbuf; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IGC_RX_BUFFER_WRITE) { @@ -1888,16 +2389,51 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) */ dma_rmb(); - rx_buffer = igc_get_rx_buffer(rx_ring, size); + rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt); + truesize = igc_get_rx_frame_truesize(rx_ring, size); + + pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; + + if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) { + timestamp = igc_ptp_rx_pktstamp(q_vector->adapter, + pktbuf); + pkt_offset = IGC_TS_HDR_LEN; + size -= IGC_TS_HDR_LEN; + } + + if (!skb) { + xdp_init_buff(&xdp, truesize, &rx_ring->xdp_rxq); + xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring), + igc_rx_offset(rx_ring) + pkt_offset, + size, true); + xdp_buff_clear_frags_flag(&xdp); + + skb = igc_xdp_run_prog(adapter, &xdp); + } + + if (IS_ERR(skb)) { + unsigned int xdp_res = -PTR_ERR(skb); - /* retrieve a buffer from the ring */ - if (skb) + switch (xdp_res) { + case IGC_XDP_CONSUMED: + rx_buffer->pagecnt_bias++; + break; + case IGC_XDP_TX: + case IGC_XDP_REDIRECT: + igc_rx_buffer_flip(rx_buffer, truesize); + xdp_status |= xdp_res; + break; + } + + total_packets++; + total_bytes += size; + } else if (skb) igc_add_rx_frag(rx_ring, rx_buffer, skb, size); else if (ring_uses_build_skb(rx_ring)) - skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); + skb = igc_build_skb(rx_ring, rx_buffer, &xdp); else - skb = igc_construct_skb(rx_ring, rx_buffer, - rx_desc, size); + skb = igc_construct_skb(rx_ring, rx_buffer, &xdp, + timestamp); /* exit if we failed to retrieve a buffer */ if (!skb) { @@ -1906,7 +2442,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) break; } - igc_put_rx_buffer(rx_ring, rx_buffer); + igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt); cleaned_count++; /* fetch next buffer in frame if non-eop */ @@ -1922,7 +2458,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) /* probably a little skewed due to removing CRC */ total_bytes += skb->len; - /* populate checksum, timestamp, VLAN, and protocol */ + /* populate checksum, VLAN, and protocol */ igc_process_skb_fields(rx_ring, rx_desc, skb); napi_gro_receive(&q_vector->napi, skb); @@ -1934,15 +2470,13 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) total_packets++; } + if (xdp_status) + igc_finalize_xdp(adapter, xdp_status); + /* place incomplete frames back on ring for completion */ rx_ring->skb = skb; - u64_stats_update_begin(&rx_ring->rx_syncp); - rx_ring->rx_stats.packets += total_packets; - rx_ring->rx_stats.bytes += total_bytes; - u64_stats_update_end(&rx_ring->rx_syncp); - q_vector->rx.total_packets += total_packets; - q_vector->rx.total_bytes += total_bytes; + igc_update_rx_stats(q_vector, total_packets, total_bytes); if (cleaned_count) igc_alloc_rx_buffers(rx_ring, cleaned_count); @@ -1950,6 +2484,224 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) return total_packets; } +static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, + struct xdp_buff *xdp) +{ + unsigned int totalsize = xdp->data_end - xdp->data_meta; + unsigned int metasize = xdp->data - xdp->data_meta; + struct sk_buff *skb; + + net_prefetch(xdp->data_meta); + + skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize, + GFP_ATOMIC | __GFP_NOWARN); + if (unlikely(!skb)) + return NULL; + + memcpy(__skb_put(skb, totalsize), xdp->data_meta, + ALIGN(totalsize, sizeof(long))); + + if (metasize) { + skb_metadata_set(skb, metasize); + __skb_pull(skb, metasize); + } + + return skb; +} + +static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, + union igc_adv_rx_desc *desc, + struct xdp_buff *xdp, + ktime_t timestamp) +{ + struct igc_ring *ring = q_vector->rx.ring; + struct sk_buff *skb; + + skb = igc_construct_skb_zc(ring, xdp); + if (!skb) { + ring->rx_stats.alloc_failed++; + return; + } + + if (timestamp) + skb_hwtstamps(skb)->hwtstamp = timestamp; + + if (igc_cleanup_headers(ring, desc, skb)) + return; + + igc_process_skb_fields(ring, desc, skb); + napi_gro_receive(&q_vector->napi, skb); +} + +static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) +{ + struct igc_adapter *adapter = q_vector->adapter; + struct igc_ring *ring = q_vector->rx.ring; + u16 cleaned_count = igc_desc_unused(ring); + int total_bytes = 0, total_packets = 0; + u16 ntc = ring->next_to_clean; + struct bpf_prog *prog; + bool failure = false; + int xdp_status = 0; + + rcu_read_lock(); + + prog = READ_ONCE(adapter->xdp_prog); + + while (likely(total_packets < budget)) { + union igc_adv_rx_desc *desc; + struct igc_rx_buffer *bi; + ktime_t timestamp = 0; + unsigned int size; + int res; + + desc = IGC_RX_DESC(ring, ntc); + size = le16_to_cpu(desc->wb.upper.length); + if (!size) + break; + + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * descriptor has been written back + */ + dma_rmb(); + + bi = &ring->rx_buffer_info[ntc]; + + if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) { + timestamp = igc_ptp_rx_pktstamp(q_vector->adapter, + bi->xdp->data); + + bi->xdp->data += IGC_TS_HDR_LEN; + + /* HW timestamp has been copied into local variable. Metadata + * length when XDP program is called should be 0. + */ + bi->xdp->data_meta += IGC_TS_HDR_LEN; + size -= IGC_TS_HDR_LEN; + } + + bi->xdp->data_end = bi->xdp->data + size; + xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool); + + res = __igc_xdp_run_prog(adapter, prog, bi->xdp); + switch (res) { + case IGC_XDP_PASS: + igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp); + fallthrough; + case IGC_XDP_CONSUMED: + xsk_buff_free(bi->xdp); + break; + case IGC_XDP_TX: + case IGC_XDP_REDIRECT: + xdp_status |= res; + break; + } + + bi->xdp = NULL; + total_bytes += size; + total_packets++; + cleaned_count++; + ntc++; + if (ntc == ring->count) + ntc = 0; + } + + ring->next_to_clean = ntc; + rcu_read_unlock(); + + if (cleaned_count >= IGC_RX_BUFFER_WRITE) + failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count); + + if (xdp_status) + igc_finalize_xdp(adapter, xdp_status); + + igc_update_rx_stats(q_vector, total_packets, total_bytes); + + if (xsk_uses_need_wakeup(ring->xsk_pool)) { + if (failure || ring->next_to_clean == ring->next_to_use) + xsk_set_rx_need_wakeup(ring->xsk_pool); + else + xsk_clear_rx_need_wakeup(ring->xsk_pool); + return total_packets; + } + + return failure ? budget : total_packets; +} + +static void igc_update_tx_stats(struct igc_q_vector *q_vector, + unsigned int packets, unsigned int bytes) +{ + struct igc_ring *ring = q_vector->tx.ring; + + u64_stats_update_begin(&ring->tx_syncp); + ring->tx_stats.bytes += bytes; + ring->tx_stats.packets += packets; + u64_stats_update_end(&ring->tx_syncp); + + q_vector->tx.total_bytes += bytes; + q_vector->tx.total_packets += packets; +} + +static void igc_xdp_xmit_zc(struct igc_ring *ring) +{ + struct xsk_buff_pool *pool = ring->xsk_pool; + struct netdev_queue *nq = txring_txq(ring); + union igc_adv_tx_desc *tx_desc = NULL; + int cpu = smp_processor_id(); + u16 ntu = ring->next_to_use; + struct xdp_desc xdp_desc; + u16 budget; + + if (!netif_carrier_ok(ring->netdev)) + return; + + __netif_tx_lock(nq, cpu); + + budget = igc_desc_unused(ring); + + while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) { + u32 cmd_type, olinfo_status; + struct igc_tx_buffer *bi; + dma_addr_t dma; + + cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | + IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | + xdp_desc.len; + olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; + + dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); + xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); + + tx_desc = IGC_TX_DESC(ring, ntu); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); + tx_desc->read.buffer_addr = cpu_to_le64(dma); + + bi = &ring->tx_buffer_info[ntu]; + bi->type = IGC_TX_BUFFER_TYPE_XSK; + bi->protocol = 0; + bi->bytecount = xdp_desc.len; + bi->gso_segs = 1; + bi->time_stamp = jiffies; + bi->next_to_watch = tx_desc; + + netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len); + + ntu++; + if (ntu == ring->count) + ntu = 0; + } + + ring->next_to_use = ntu; + if (tx_desc) { + igc_flush_tx_descriptors(ring); + xsk_tx_release(pool); + } + + __netif_tx_unlock(nq); +} + /** * igc_clean_tx_irq - Reclaim resources after transmit completes * @q_vector: pointer to q_vector containing needed info @@ -1966,6 +2718,7 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) unsigned int i = tx_ring->next_to_clean; struct igc_tx_buffer *tx_buffer; union igc_adv_tx_desc *tx_desc; + u32 xsk_frames = 0; if (test_bit(__IGC_DOWN, &adapter->state)) return true; @@ -1995,17 +2748,22 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) total_bytes += tx_buffer->bytecount; total_packets += tx_buffer->gso_segs; - /* free the skb */ - napi_consume_skb(tx_buffer->skb, napi_budget); - - /* unmap skb header data */ - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - - /* clear tx_buffer data */ - dma_unmap_len_set(tx_buffer, len, 0); + switch (tx_buffer->type) { + case IGC_TX_BUFFER_TYPE_XSK: + xsk_frames++; + break; + case IGC_TX_BUFFER_TYPE_XDP: + xdp_return_frame(tx_buffer->xdpf); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + case IGC_TX_BUFFER_TYPE_SKB: + napi_consume_skb(tx_buffer->skb, napi_budget); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + default: + netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); + break; + } /* clear last DMA location and unmap remaining buffers */ while (tx_desc != eop_desc) { @@ -2019,13 +2777,8 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) } /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buffer, len)) { - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); - } + if (dma_unmap_len(tx_buffer, len)) + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); } /* move us one more past the eop_desc for start of next pkt */ @@ -2050,12 +2803,16 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) i += tx_ring->count; tx_ring->next_to_clean = i; - u64_stats_update_begin(&tx_ring->tx_syncp); - tx_ring->tx_stats.bytes += total_bytes; - tx_ring->tx_stats.packets += total_packets; - u64_stats_update_end(&tx_ring->tx_syncp); - q_vector->tx.total_bytes += total_bytes; - q_vector->tx.total_packets += total_packets; + + igc_update_tx_stats(q_vector, total_packets, total_bytes); + + if (tx_ring->xsk_pool) { + if (xsk_frames) + xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); + if (xsk_uses_need_wakeup(tx_ring->xsk_pool)) + xsk_set_tx_need_wakeup(tx_ring->xsk_pool); + igc_xdp_xmit_zc(tx_ring); + } if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { struct igc_hw *hw = &adapter->hw; @@ -2069,27 +2826,27 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) (adapter->tx_timeout_factor * HZ)) && !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { /* detected Tx unit hang */ - dev_err(tx_ring->dev, - "Detected Tx Unit Hang\n" - " Tx Queue <%d>\n" - " TDH <%x>\n" - " TDT <%x>\n" - " next_to_use <%x>\n" - " next_to_clean <%x>\n" - "buffer_info[next_to_clean]\n" - " time_stamp <%lx>\n" - " next_to_watch <%p>\n" - " jiffies <%lx>\n" - " desc.status <%x>\n", - tx_ring->queue_index, - rd32(IGC_TDH(tx_ring->reg_idx)), - readl(tx_ring->tail), - tx_ring->next_to_use, - tx_ring->next_to_clean, - tx_buffer->time_stamp, - tx_buffer->next_to_watch, - jiffies, - tx_buffer->next_to_watch->wb.status); + netdev_err(tx_ring->netdev, + "Detected Tx Unit Hang\n" + " Tx Queue <%d>\n" + " TDH <%x>\n" + " TDT <%x>\n" + " next_to_use <%x>\n" + " next_to_clean <%x>\n" + "buffer_info[next_to_clean]\n" + " time_stamp <%lx>\n" + " next_to_watch <%p>\n" + " jiffies <%lx>\n" + " desc.status <%x>\n", + tx_ring->queue_index, + rd32(IGC_TDH(tx_ring->reg_idx)), + readl(tx_ring->tail), + tx_ring->next_to_use, + tx_ring->next_to_clean, + tx_buffer->time_stamp, + tx_buffer->next_to_watch, + jiffies, + tx_buffer->next_to_watch->wb.status); netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); @@ -2121,140 +2878,749 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) return !!budget; } -static void igc_nfc_filter_restore(struct igc_adapter *adapter) +static int igc_find_mac_filter(struct igc_adapter *adapter, + enum igc_mac_filter_type type, const u8 *addr) { - struct igc_nfc_filter *rule; + struct igc_hw *hw = &adapter->hw; + int max_entries = hw->mac.rar_entry_count; + u32 ral, rah; + int i; - spin_lock(&adapter->nfc_lock); + for (i = 0; i < max_entries; i++) { + ral = rd32(IGC_RAL(i)); + rah = rd32(IGC_RAH(i)); - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) - igc_add_filter(adapter, rule); + if (!(rah & IGC_RAH_AV)) + continue; + if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) + continue; + if ((rah & IGC_RAH_RAH_MASK) != + le16_to_cpup((__le16 *)(addr + 4))) + continue; + if (ral != le32_to_cpup((__le32 *)(addr))) + continue; - spin_unlock(&adapter->nfc_lock); + return i; + } + + return -1; } -/* If the filter to be added and an already existing filter express - * the same address and address type, it should be possible to only - * override the other configurations, for example the queue to steer - * traffic. - */ -static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, - const u8 *addr, const u8 flags) +static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) { - if (!(entry->state & IGC_MAC_STATE_IN_USE)) - return true; + struct igc_hw *hw = &adapter->hw; + int max_entries = hw->mac.rar_entry_count; + u32 rah; + int i; - if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != - (flags & IGC_MAC_STATE_SRC_ADDR)) - return false; + for (i = 0; i < max_entries; i++) { + rah = rd32(IGC_RAH(i)); - if (!ether_addr_equal(addr, entry->addr)) - return false; + if (!(rah & IGC_RAH_AV)) + return i; + } - return true; + return -1; } -/* Add a MAC filter for 'addr' directing matching traffic to 'queue', - * 'flags' is used to indicate what kind of match is made, match is by - * default for the destination address, if matching by source address - * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. +/** + * igc_add_mac_filter() - Add MAC address filter + * @adapter: Pointer to adapter where the filter should be added + * @type: MAC address filter type (source or destination) + * @addr: MAC address + * @queue: If non-negative, queue assignment feature is enabled and frames + * matching the filter are enqueued onto 'queue'. Otherwise, queue + * assignment is disabled. + * + * Return: 0 in case of success, negative errno code otherwise. */ static int igc_add_mac_filter(struct igc_adapter *adapter, - const u8 *addr, const u8 queue) + enum igc_mac_filter_type type, const u8 *addr, + int queue) +{ + struct net_device *dev = adapter->netdev; + int index; + + index = igc_find_mac_filter(adapter, type, addr); + if (index >= 0) + goto update_filter; + + index = igc_get_avail_mac_filter_slot(adapter); + if (index < 0) + return -ENOSPC; + + netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", + index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", + addr, queue); + +update_filter: + igc_set_mac_filter_hw(adapter, index, type, addr, queue); + return 0; +} + +/** + * igc_del_mac_filter() - Delete MAC address filter + * @adapter: Pointer to adapter where the filter should be deleted from + * @type: MAC address filter type (source or destination) + * @addr: MAC address + */ +static void igc_del_mac_filter(struct igc_adapter *adapter, + enum igc_mac_filter_type type, const u8 *addr) +{ + struct net_device *dev = adapter->netdev; + int index; + + index = igc_find_mac_filter(adapter, type, addr); + if (index < 0) + return; + + if (index == 0) { + /* If this is the default filter, we don't actually delete it. + * We just reset to its default value i.e. disable queue + * assignment. + */ + netdev_dbg(dev, "Disable default MAC filter queue assignment"); + + igc_set_mac_filter_hw(adapter, 0, type, addr, -1); + } else { + netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", + index, + type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", + addr); + + igc_clear_mac_filter_hw(adapter, index); + } +} + +/** + * igc_add_vlan_prio_filter() - Add VLAN priority filter + * @adapter: Pointer to adapter where the filter should be added + * @prio: VLAN priority value + * @queue: Queue number which matching frames are assigned to + * + * Return: 0 in case of success, negative errno code otherwise. + */ +static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, + int queue) +{ + struct net_device *dev = adapter->netdev; + struct igc_hw *hw = &adapter->hw; + u32 vlanpqf; + + vlanpqf = rd32(IGC_VLANPQF); + + if (vlanpqf & IGC_VLANPQF_VALID(prio)) { + netdev_dbg(dev, "VLAN priority filter already in use\n"); + return -EEXIST; + } + + vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); + vlanpqf |= IGC_VLANPQF_VALID(prio); + + wr32(IGC_VLANPQF, vlanpqf); + + netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", + prio, queue); + return 0; +} + +/** + * igc_del_vlan_prio_filter() - Delete VLAN priority filter + * @adapter: Pointer to adapter where the filter should be deleted from + * @prio: VLAN priority value + */ +static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) +{ + struct igc_hw *hw = &adapter->hw; + u32 vlanpqf; + + vlanpqf = rd32(IGC_VLANPQF); + + vlanpqf &= ~IGC_VLANPQF_VALID(prio); + vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); + + wr32(IGC_VLANPQF, vlanpqf); + + netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", + prio); +} + +static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; int i; - if (is_zero_ether_addr(addr)) - return -EINVAL; + for (i = 0; i < MAX_ETYPE_FILTER; i++) { + u32 etqf = rd32(IGC_ETQF(i)); - /* Search for the first empty entry in the MAC table. - * Do not touch entries at the end of the table reserved for the VF MAC - * addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], - addr, 0)) - continue; + if (!(etqf & IGC_ETQF_FILTER_ENABLE)) + return i; + } - ether_addr_copy(adapter->mac_table[i].addr, addr); - adapter->mac_table[i].queue = queue; - adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE; + return -1; +} - igc_rar_set_index(adapter, i); - return i; +/** + * igc_add_etype_filter() - Add ethertype filter + * @adapter: Pointer to adapter where the filter should be added + * @etype: Ethertype value + * @queue: If non-negative, queue assignment feature is enabled and frames + * matching the filter are enqueued onto 'queue'. Otherwise, queue + * assignment is disabled. + * + * Return: 0 in case of success, negative errno code otherwise. + */ +static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, + int queue) +{ + struct igc_hw *hw = &adapter->hw; + int index; + u32 etqf; + + index = igc_get_avail_etype_filter_slot(adapter); + if (index < 0) + return -ENOSPC; + + etqf = rd32(IGC_ETQF(index)); + + etqf &= ~IGC_ETQF_ETYPE_MASK; + etqf |= etype; + + if (queue >= 0) { + etqf &= ~IGC_ETQF_QUEUE_MASK; + etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); + etqf |= IGC_ETQF_QUEUE_ENABLE; } - return -ENOSPC; + etqf |= IGC_ETQF_FILTER_ENABLE; + + wr32(IGC_ETQF(index), etqf); + + netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", + etype, queue); + return 0; } -/* Remove a MAC filter for 'addr' directing matching traffic to - * 'queue', 'flags' is used to indicate what kind of match need to be - * removed, match is by default for the destination address, if - * matching by source address is to be removed the flag - * IGC_MAC_STATE_SRC_ADDR can be used. +static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) +{ + struct igc_hw *hw = &adapter->hw; + int i; + + for (i = 0; i < MAX_ETYPE_FILTER; i++) { + u32 etqf = rd32(IGC_ETQF(i)); + + if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) + return i; + } + + return -1; +} + +/** + * igc_del_etype_filter() - Delete ethertype filter + * @adapter: Pointer to adapter where the filter should be deleted from + * @etype: Ethertype value */ -static int igc_del_mac_filter(struct igc_adapter *adapter, - const u8 *addr, const u8 queue) +static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) +{ + struct igc_hw *hw = &adapter->hw; + int index; + + index = igc_find_etype_filter(adapter, etype); + if (index < 0) + return; + + wr32(IGC_ETQF(index), 0); + + netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", + etype); +} + +static int igc_flex_filter_select(struct igc_adapter *adapter, + struct igc_flex_filter *input, + u32 *fhft) +{ + struct igc_hw *hw = &adapter->hw; + u8 fhft_index; + u32 fhftsl; + + if (input->index >= MAX_FLEX_FILTER) { + dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n"); + return -EINVAL; + } + + /* Indirect table select register */ + fhftsl = rd32(IGC_FHFTSL); + fhftsl &= ~IGC_FHFTSL_FTSL_MASK; + switch (input->index) { + case 0 ... 7: + fhftsl |= 0x00; + break; + case 8 ... 15: + fhftsl |= 0x01; + break; + case 16 ... 23: + fhftsl |= 0x02; + break; + case 24 ... 31: + fhftsl |= 0x03; + break; + } + wr32(IGC_FHFTSL, fhftsl); + + /* Normalize index down to host table register */ + fhft_index = input->index % 8; + + *fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) : + IGC_FHFT_EXT(fhft_index - 4); + + return 0; +} + +static int igc_write_flex_filter_ll(struct igc_adapter *adapter, + struct igc_flex_filter *input) { + struct device *dev = &adapter->pdev->dev; struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; + u8 *data = input->data; + u8 *mask = input->mask; + u32 queuing; + u32 fhft; + u32 wufc; + int ret; int i; - if (is_zero_ether_addr(addr)) + /* Length has to be aligned to 8. Otherwise the filter will fail. Bail + * out early to avoid surprises later. + */ + if (input->length % 8 != 0) { + dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n"); return -EINVAL; + } + + /* Select corresponding flex filter register and get base for host table. */ + ret = igc_flex_filter_select(adapter, input, &fhft); + if (ret) + return ret; - /* Search for matching entry in the MAC table based on given address - * and queue. Do not touch entries at the end of the table reserved - * for the VF MAC addresses. + /* When adding a filter globally disable flex filter feature. That is + * recommended within the datasheet. */ - for (i = 0; i < rar_entries; i++) { - if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) - continue; - if (adapter->mac_table[i].state != 0) - continue; - if (adapter->mac_table[i].queue != queue) - continue; - if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) + wufc = rd32(IGC_WUFC); + wufc &= ~IGC_WUFC_FLEX_HQ; + wr32(IGC_WUFC, wufc); + + /* Configure filter */ + queuing = input->length & IGC_FHFT_LENGTH_MASK; + queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK; + queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK; + + if (input->immediate_irq) + queuing |= IGC_FHFT_IMM_INT; + + if (input->drop) + queuing |= IGC_FHFT_DROP; + + wr32(fhft + 0xFC, queuing); + + /* Write data (128 byte) and mask (128 bit) */ + for (i = 0; i < 16; ++i) { + const size_t data_idx = i * 8; + const size_t row_idx = i * 16; + u32 dw0 = + (data[data_idx + 0] << 0) | + (data[data_idx + 1] << 8) | + (data[data_idx + 2] << 16) | + (data[data_idx + 3] << 24); + u32 dw1 = + (data[data_idx + 4] << 0) | + (data[data_idx + 5] << 8) | + (data[data_idx + 6] << 16) | + (data[data_idx + 7] << 24); + u32 tmp; + + /* Write row: dw0, dw1 and mask */ + wr32(fhft + row_idx, dw0); + wr32(fhft + row_idx + 4, dw1); + + /* mask is only valid for MASK(7, 0) */ + tmp = rd32(fhft + row_idx + 8); + tmp &= ~GENMASK(7, 0); + tmp |= mask[i]; + wr32(fhft + row_idx + 8, tmp); + } + + /* Enable filter. */ + wufc |= IGC_WUFC_FLEX_HQ; + if (input->index > 8) { + /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */ + u32 wufc_ext = rd32(IGC_WUFC_EXT); + + wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8)); + + wr32(IGC_WUFC_EXT, wufc_ext); + } else { + wufc |= (IGC_WUFC_FLX0 << input->index); + } + wr32(IGC_WUFC, wufc); + + dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n", + input->index); + + return 0; +} + +static void igc_flex_filter_add_field(struct igc_flex_filter *flex, + const void *src, unsigned int offset, + size_t len, const void *mask) +{ + int i; + + /* data */ + memcpy(&flex->data[offset], src, len); + + /* mask */ + for (i = 0; i < len; ++i) { + const unsigned int idx = i + offset; + const u8 *ptr = mask; + + if (mask) { + if (ptr[i] & 0xff) + flex->mask[idx / 8] |= BIT(idx % 8); + continue; + } - /* When a filter for the default address is "deleted", - * we return it to its initial configuration - */ - if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { - adapter->mac_table[i].state = - IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - adapter->mac_table[i].queue = 0; + flex->mask[idx / 8] |= BIT(idx % 8); + } +} + +static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 wufc, wufc_ext; + int i; + + wufc = rd32(IGC_WUFC); + wufc_ext = rd32(IGC_WUFC_EXT); + + for (i = 0; i < MAX_FLEX_FILTER; i++) { + if (i < 8) { + if (!(wufc & (IGC_WUFC_FLX0 << i))) + return i; } else { - adapter->mac_table[i].state = 0; - adapter->mac_table[i].queue = 0; - memset(adapter->mac_table[i].addr, 0, ETH_ALEN); + if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8)))) + return i; } + } - igc_rar_set_index(adapter, i); - return 0; + return -ENOSPC; +} + +static bool igc_flex_filter_in_use(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 wufc, wufc_ext; + + wufc = rd32(IGC_WUFC); + wufc_ext = rd32(IGC_WUFC_EXT); + + if (wufc & IGC_WUFC_FILTER_MASK) + return true; + + if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK) + return true; + + return false; +} + +static int igc_add_flex_filter(struct igc_adapter *adapter, + struct igc_nfc_rule *rule) +{ + struct igc_flex_filter flex = { }; + struct igc_nfc_filter *filter = &rule->filter; + unsigned int eth_offset, user_offset; + int ret, index; + bool vlan; + + index = igc_find_avail_flex_filter_slot(adapter); + if (index < 0) + return -ENOSPC; + + /* Construct the flex filter: + * -> dest_mac [6] + * -> src_mac [6] + * -> tpid [2] + * -> vlan tci [2] + * -> ether type [2] + * -> user data [8] + * -> = 26 bytes => 32 length + */ + flex.index = index; + flex.length = 32; + flex.rx_queue = rule->action; + + vlan = rule->filter.vlan_tci || rule->filter.vlan_etype; + eth_offset = vlan ? 16 : 12; + user_offset = vlan ? 18 : 14; + + /* Add destination MAC */ + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) + igc_flex_filter_add_field(&flex, &filter->dst_addr, 0, + ETH_ALEN, NULL); + + /* Add source MAC */ + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) + igc_flex_filter_add_field(&flex, &filter->src_addr, 6, + ETH_ALEN, NULL); + + /* Add VLAN etype */ + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE) + igc_flex_filter_add_field(&flex, &filter->vlan_etype, 12, + sizeof(filter->vlan_etype), + NULL); + + /* Add VLAN TCI */ + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) + igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14, + sizeof(filter->vlan_tci), NULL); + + /* Add Ether type */ + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { + __be16 etype = cpu_to_be16(filter->etype); + + igc_flex_filter_add_field(&flex, &etype, eth_offset, + sizeof(etype), NULL); } - return -ENOENT; + /* Add user data */ + if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA) + igc_flex_filter_add_field(&flex, &filter->user_data, + user_offset, + sizeof(filter->user_data), + filter->user_mask); + + /* Add it down to the hardware and enable it. */ + ret = igc_write_flex_filter_ll(adapter, &flex); + if (ret) + return ret; + + filter->flex_index = index; + + return 0; +} + +static void igc_del_flex_filter(struct igc_adapter *adapter, + u16 reg_index) +{ + struct igc_hw *hw = &adapter->hw; + u32 wufc; + + /* Just disable the filter. The filter table itself is kept + * intact. Another flex_filter_add() should override the "old" data + * then. + */ + if (reg_index > 8) { + u32 wufc_ext = rd32(IGC_WUFC_EXT); + + wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8)); + wr32(IGC_WUFC_EXT, wufc_ext); + } else { + wufc = rd32(IGC_WUFC); + + wufc &= ~(IGC_WUFC_FLX0 << reg_index); + wr32(IGC_WUFC, wufc); + } + + if (igc_flex_filter_in_use(adapter)) + return; + + /* No filters are in use, we may disable flex filters */ + wufc = rd32(IGC_WUFC); + wufc &= ~IGC_WUFC_FLEX_HQ; + wr32(IGC_WUFC, wufc); +} + +static int igc_enable_nfc_rule(struct igc_adapter *adapter, + struct igc_nfc_rule *rule) +{ + int err; + + if (rule->flex) { + return igc_add_flex_filter(adapter, rule); + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { + err = igc_add_etype_filter(adapter, rule->filter.etype, + rule->action); + if (err) + return err; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { + err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, + rule->filter.src_addr, rule->action); + if (err) + return err; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { + err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, + rule->filter.dst_addr, rule->action); + if (err) + return err; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { + int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> + VLAN_PRIO_SHIFT; + + err = igc_add_vlan_prio_filter(adapter, prio, rule->action); + if (err) + return err; + } + + return 0; +} + +static void igc_disable_nfc_rule(struct igc_adapter *adapter, + const struct igc_nfc_rule *rule) +{ + if (rule->flex) { + igc_del_flex_filter(adapter, rule->filter.flex_index); + return; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) + igc_del_etype_filter(adapter, rule->filter.etype); + + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { + int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> + VLAN_PRIO_SHIFT; + + igc_del_vlan_prio_filter(adapter, prio); + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) + igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, + rule->filter.src_addr); + + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) + igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, + rule->filter.dst_addr); +} + +/** + * igc_get_nfc_rule() - Get NFC rule + * @adapter: Pointer to adapter + * @location: Rule location + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + * + * Return: Pointer to NFC rule at @location. If not found, NULL. + */ +struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, + u32 location) +{ + struct igc_nfc_rule *rule; + + list_for_each_entry(rule, &adapter->nfc_rule_list, list) { + if (rule->location == location) + return rule; + if (rule->location > location) + break; + } + + return NULL; +} + +/** + * igc_del_nfc_rule() - Delete NFC rule + * @adapter: Pointer to adapter + * @rule: Pointer to rule to be deleted + * + * Disable NFC rule in hardware and delete it from adapter. + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + */ +void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) +{ + igc_disable_nfc_rule(adapter, rule); + + list_del(&rule->list); + adapter->nfc_rule_count--; + + kfree(rule); +} + +static void igc_flush_nfc_rules(struct igc_adapter *adapter) +{ + struct igc_nfc_rule *rule, *tmp; + + mutex_lock(&adapter->nfc_rule_lock); + + list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) + igc_del_nfc_rule(adapter, rule); + + mutex_unlock(&adapter->nfc_rule_lock); +} + +/** + * igc_add_nfc_rule() - Add NFC rule + * @adapter: Pointer to adapter + * @rule: Pointer to rule to be added + * + * Enable NFC rule in hardware and add it to adapter. + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + * + * Return: 0 on success, negative errno on failure. + */ +int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) +{ + struct igc_nfc_rule *pred, *cur; + int err; + + err = igc_enable_nfc_rule(adapter, rule); + if (err) + return err; + + pred = NULL; + list_for_each_entry(cur, &adapter->nfc_rule_list, list) { + if (cur->location >= rule->location) + break; + pred = cur; + } + + list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); + adapter->nfc_rule_count++; + return 0; +} + +static void igc_restore_nfc_rules(struct igc_adapter *adapter) +{ + struct igc_nfc_rule *rule; + + mutex_lock(&adapter->nfc_rule_lock); + + list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) + igc_enable_nfc_rule(adapter, rule); + + mutex_unlock(&adapter->nfc_rule_lock); } static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) { struct igc_adapter *adapter = netdev_priv(netdev); - int ret; - - ret = igc_add_mac_filter(adapter, addr, adapter->num_rx_queues); - return min_t(int, ret, 0); + return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); } static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) { struct igc_adapter *adapter = netdev_priv(netdev); - igc_del_mac_filter(adapter, addr, adapter->num_rx_queues); - + igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); return 0; } @@ -2321,11 +3687,15 @@ static void igc_configure(struct igc_adapter *adapter) igc_get_hw_control(adapter); igc_set_rx_mode(netdev); + igc_restore_vlan(adapter); + igc_setup_tctl(adapter); igc_setup_mrqc(adapter); igc_setup_rctl(adapter); - igc_nfc_filter_restore(adapter); + igc_set_default_mac_filter(adapter); + igc_restore_nfc_rules(adapter); + igc_configure_tx(adapter); igc_configure_rx(adapter); @@ -2338,7 +3708,10 @@ static void igc_configure(struct igc_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { struct igc_ring *ring = adapter->rx_ring[i]; - igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); + if (ring->xsk_pool) + igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); + else + igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); } } @@ -2518,12 +3891,7 @@ void igc_set_flag_queue_pairs(struct igc_adapter *adapter, unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) { - unsigned int max_rss_queues; - - /* Determine the maximum number of RSS queues supported. */ - max_rss_queues = IGC_MAX_RX_QUEUES; - - return max_rss_queues; + return IGC_MAX_RX_QUEUES; } static void igc_init_queue_configuration(struct igc_adapter *adapter) @@ -2939,7 +4307,6 @@ static void igc_cache_ring_register(struct igc_adapter *adapter) switch (adapter->hw.mac.type) { case igc_i225: - /* Fall through */ default: for (; i < adapter->num_rx_queues; i++) adapter->rx_ring[i]->reg_idx = i; @@ -2959,14 +4326,17 @@ static int igc_poll(struct napi_struct *napi, int budget) struct igc_q_vector *q_vector = container_of(napi, struct igc_q_vector, napi); + struct igc_ring *rx_ring = q_vector->rx.ring; bool clean_complete = true; int work_done = 0; if (q_vector->tx.ring) clean_complete = igc_clean_tx_irq(q_vector, budget); - if (q_vector->rx.ring) { - int cleaned = igc_clean_rx_irq(q_vector, budget); + if (rx_ring) { + int cleaned = rx_ring->xsk_pool ? + igc_clean_rx_irq_zc(q_vector, budget) : + igc_clean_rx_irq(q_vector, budget); work_done += cleaned; if (cleaned >= budget) @@ -3024,8 +4394,7 @@ static int igc_alloc_q_vector(struct igc_adapter *adapter, return -ENOMEM; /* initialize NAPI */ - netif_napi_add(adapter->netdev, &q_vector->napi, - igc_poll, 64); + netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll); /* tie q_vector and adapter together */ adapter->q_vector[v_idx] = q_vector; @@ -3164,14 +4533,14 @@ err_out: */ static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; int err = 0; igc_set_interrupt_capability(adapter, msix); err = igc_alloc_q_vectors(adapter); if (err) { - dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); + netdev_err(dev, "Unable to allocate memory for vectors\n"); goto err_alloc_q_vectors; } @@ -3198,8 +4567,6 @@ static int igc_sw_init(struct igc_adapter *adapter) struct pci_dev *pdev = adapter->pdev; struct igc_hw *hw = &adapter->hw; - int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; - pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); /* set default ring sizes */ @@ -3218,20 +4585,19 @@ static int igc_sw_init(struct igc_adapter *adapter) VLAN_HLEN; adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; - spin_lock_init(&adapter->nfc_lock); + mutex_init(&adapter->nfc_rule_lock); + INIT_LIST_HEAD(&adapter->nfc_rule_list); + adapter->nfc_rule_count = 0; + spin_lock_init(&adapter->stats64_lock); /* Assume MSI-X interrupts, will be checked during IRQ allocation */ adapter->flags |= IGC_FLAG_HAS_MSIX; - adapter->mac_table = kzalloc(size, GFP_ATOMIC); - if (!adapter->mac_table) - return -ENOMEM; - igc_init_queue_configuration(adapter); /* This call may decrease the number of queues */ if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; } @@ -3272,7 +4638,7 @@ void igc_up(struct igc_adapter *adapter) netif_tx_start_all_queues(adapter->netdev); /* start the watchdog. */ - hw->mac.get_link_status = 1; + hw->mac.get_link_status = true; schedule_work(&adapter->watchdog_task); } @@ -3359,8 +4725,9 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.prc511 += rd32(IGC_PRC511); adapter->stats.prc1023 += rd32(IGC_PRC1023); adapter->stats.prc1522 += rd32(IGC_PRC1522); - adapter->stats.symerrs += rd32(IGC_SYMERRS); - adapter->stats.sec += rd32(IGC_SEC); + adapter->stats.tlpic += rd32(IGC_TLPIC); + adapter->stats.rlpic += rd32(IGC_RLPIC); + adapter->stats.hgptc += rd32(IGC_HGPTC); mpc = rd32(IGC_MPC); adapter->stats.mpc += mpc; @@ -3399,21 +4766,13 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.tpt += rd32(IGC_TPT); adapter->stats.colc += rd32(IGC_COLC); + adapter->stats.colc += rd32(IGC_RERC); adapter->stats.algnerrc += rd32(IGC_ALGNERRC); adapter->stats.tsctc += rd32(IGC_TSCTC); - adapter->stats.tsctfc += rd32(IGC_TSCTFC); adapter->stats.iac += rd32(IGC_IAC); - adapter->stats.icrxoc += rd32(IGC_ICRXOC); - adapter->stats.icrxptc += rd32(IGC_ICRXPTC); - adapter->stats.icrxatc += rd32(IGC_ICRXATC); - adapter->stats.ictxptc += rd32(IGC_ICTXPTC); - adapter->stats.ictxatc += rd32(IGC_ICTXATC); - adapter->stats.ictxqec += rd32(IGC_ICTXQEC); - adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); - adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); /* Fill out the OS statistics structure */ net_stats->multicast = adapter->stats.mprc; @@ -3449,21 +4808,6 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.mgpdc += rd32(IGC_MGTPDC); } -static void igc_nfc_filter_exit(struct igc_adapter *adapter) -{ - struct igc_nfc_filter *rule; - - spin_lock(&adapter->nfc_lock); - - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) - igc_erase_filter(adapter, rule); - - hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) - igc_erase_filter(adapter, rule); - - spin_unlock(&adapter->nfc_lock); -} - /** * igc_down - Close the interface * @adapter: board private structure @@ -3477,28 +4821,31 @@ void igc_down(struct igc_adapter *adapter) set_bit(__IGC_DOWN, &adapter->state); - /* disable receives in the hardware */ - rctl = rd32(IGC_RCTL); - wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); - /* flush and sleep below */ - - igc_nfc_filter_exit(adapter); + igc_ptp_suspend(adapter); + if (pci_device_is_present(adapter->pdev)) { + /* disable receives in the hardware */ + rctl = rd32(IGC_RCTL); + wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); + /* flush and sleep below */ + } /* set trans_start so we don't get spurious watchdogs during reset */ netif_trans_update(netdev); netif_carrier_off(netdev); netif_tx_stop_all_queues(netdev); - /* disable transmits in the hardware */ - tctl = rd32(IGC_TCTL); - tctl &= ~IGC_TCTL_EN; - wr32(IGC_TCTL, tctl); - /* flush both disables and wait for them to finish */ - wrfl(); - usleep_range(10000, 20000); + if (pci_device_is_present(adapter->pdev)) { + /* disable transmits in the hardware */ + tctl = rd32(IGC_TCTL); + tctl &= ~IGC_TCTL_EN; + wr32(IGC_TCTL, tctl); + /* flush both disables and wait for them to finish */ + wrfl(); + usleep_range(10000, 20000); - igc_irq_disable(adapter); + igc_irq_disable(adapter); + } adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; @@ -3532,7 +4879,6 @@ void igc_down(struct igc_adapter *adapter) void igc_reinit_locked(struct igc_adapter *adapter) { - WARN_ON(in_interrupt()); while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) usleep_range(1000, 2000); igc_down(adapter); @@ -3546,8 +4892,19 @@ static void igc_reset_task(struct work_struct *work) adapter = container_of(work, struct igc_adapter, reset_task); + rtnl_lock(); + /* If we're already down or resetting, just bail */ + if (test_bit(__IGC_DOWN, &adapter->state) || + test_bit(__IGC_RESETTING, &adapter->state)) { + rtnl_unlock(); + return; + } + + igc_rings_dump(adapter); + igc_regs_dump(adapter); netdev_err(adapter->netdev, "Reset adapter\n"); igc_reinit_locked(adapter); + rtnl_unlock(); } /** @@ -3562,6 +4919,11 @@ static int igc_change_mtu(struct net_device *netdev, int new_mtu) int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; struct igc_adapter *adapter = netdev_priv(netdev); + if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) { + netdev_dbg(netdev, "Jumbo frames not supported with XDP"); + return -EINVAL; + } + /* adjust max frame to be at least the size of a standard frame */ if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; @@ -3575,8 +4937,7 @@ static int igc_change_mtu(struct net_device *netdev, int new_mtu) if (netif_running(netdev)) igc_down(adapter); - netdev_dbg(netdev, "changing MTU from %d to %d\n", - netdev->mtu, new_mtu); + netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); netdev->mtu = new_mtu; if (netif_running(netdev)) @@ -3590,21 +4951,23 @@ static int igc_change_mtu(struct net_device *netdev, int new_mtu) } /** - * igc_get_stats - Get System Network Statistics + * igc_get_stats64 - Get System Network Statistics * @netdev: network interface device structure + * @stats: rtnl_link_stats64 pointer * * Returns the address of the device statistics structure. * The statistics are updated here and also from the timer callback. */ -static struct net_device_stats *igc_get_stats(struct net_device *netdev) +static void igc_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) { struct igc_adapter *adapter = netdev_priv(netdev); + spin_lock(&adapter->stats64_lock); if (!test_bit(__IGC_RESETTING, &adapter->state)) igc_update_stats(adapter); - - /* only return the current stats */ - return &netdev->stats; + memcpy(stats, &adapter->stats64, sizeof(*stats)); + spin_unlock(&adapter->stats64_lock); } static netdev_features_t igc_fix_features(struct net_device *netdev, @@ -3627,24 +4990,15 @@ static int igc_set_features(struct net_device *netdev, netdev_features_t changed = netdev->features ^ features; struct igc_adapter *adapter = netdev_priv(netdev); + if (changed & NETIF_F_HW_VLAN_CTAG_RX) + igc_vlan_mode(netdev, features); + /* Add VLAN support */ if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) return 0; - if (!(features & NETIF_F_NTUPLE)) { - struct hlist_node *node2; - struct igc_nfc_filter *rule; - - spin_lock(&adapter->nfc_lock); - hlist_for_each_entry_safe(rule, node2, - &adapter->nfc_filter_list, nfc_node) { - igc_erase_filter(adapter, rule); - hlist_del(&rule->nfc_node); - kfree(rule); - } - spin_unlock(&adapter->nfc_lock); - adapter->nfc_filter_count = 0; - } + if (!(features & NETIF_F_NTUPLE)) + igc_flush_nfc_rules(adapter); netdev->features = features; @@ -3687,116 +5041,75 @@ igc_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -/* Add a MAC filter for 'addr' directing matching traffic to 'queue', - * 'flags' is used to indicate what kind of match is made, match is by - * default for the destination address, if matching by source address - * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_add_mac_filter_flags(struct igc_adapter *adapter, - const u8 *addr, const u8 queue, - const u8 flags) +static void igc_tsync_interrupt(struct igc_adapter *adapter) { + u32 ack, tsauxc, sec, nsec, tsicr; struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; - - if (is_zero_ether_addr(addr)) - return -EINVAL; - - /* Search for the first empty entry in the MAC table. - * Do not touch entries at the end of the table reserved for the VF MAC - * addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], - addr, flags)) - continue; + struct ptp_clock_event event; + struct timespec64 ts; - ether_addr_copy(adapter->mac_table[i].addr, addr); - adapter->mac_table[i].queue = queue; - adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; + tsicr = rd32(IGC_TSICR); + ack = 0; - igc_rar_set_index(adapter, i); - return i; + if (tsicr & IGC_TSICR_SYS_WRAP) { + event.type = PTP_CLOCK_PPS; + if (adapter->ptp_caps.pps) + ptp_clock_event(adapter->ptp_clock, &event); + ack |= IGC_TSICR_SYS_WRAP; } - return -ENOSPC; -} - -int igc_add_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags) -{ - return igc_add_mac_filter_flags(adapter, addr, queue, - IGC_MAC_STATE_QUEUE_STEERING | flags); -} - -/* Remove a MAC filter for 'addr' directing matching traffic to - * 'queue', 'flags' is used to indicate what kind of match need to be - * removed, match is by default for the destination address, if - * matching by source address is to be removed the flag - * IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_del_mac_filter_flags(struct igc_adapter *adapter, - const u8 *addr, const u8 queue, - const u8 flags) -{ - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; - - if (is_zero_ether_addr(addr)) - return -EINVAL; - - /* Search for matching entry in the MAC table based on given address - * and queue. Do not touch entries at the end of the table reserved - * for the VF MAC addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) - continue; - if ((adapter->mac_table[i].state & flags) != flags) - continue; - if (adapter->mac_table[i].queue != queue) - continue; - if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) - continue; - - /* When a filter for the default address is "deleted", - * we return it to its initial configuration - */ - if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { - adapter->mac_table[i].state = - IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - } else { - adapter->mac_table[i].state = 0; - adapter->mac_table[i].queue = 0; - memset(adapter->mac_table[i].addr, 0, ETH_ALEN); - } - - igc_rar_set_index(adapter, i); - return 0; + if (tsicr & IGC_TSICR_TXTS) { + /* retrieve hardware timestamp */ + schedule_work(&adapter->ptp_tx_work); + ack |= IGC_TSICR_TXTS; } - return -ENOENT; -} + if (tsicr & IGC_TSICR_TT0) { + spin_lock(&adapter->tmreg_lock); + ts = timespec64_add(adapter->perout[0].start, + adapter->perout[0].period); + wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); + wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec); + tsauxc = rd32(IGC_TSAUXC); + tsauxc |= IGC_TSAUXC_EN_TT0; + wr32(IGC_TSAUXC, tsauxc); + adapter->perout[0].start = ts; + spin_unlock(&adapter->tmreg_lock); + ack |= IGC_TSICR_TT0; + } -int igc_del_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags) -{ - return igc_del_mac_filter_flags(adapter, addr, queue, - IGC_MAC_STATE_QUEUE_STEERING | flags); -} + if (tsicr & IGC_TSICR_TT1) { + spin_lock(&adapter->tmreg_lock); + ts = timespec64_add(adapter->perout[1].start, + adapter->perout[1].period); + wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); + wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec); + tsauxc = rd32(IGC_TSAUXC); + tsauxc |= IGC_TSAUXC_EN_TT1; + wr32(IGC_TSAUXC, tsauxc); + adapter->perout[1].start = ts; + spin_unlock(&adapter->tmreg_lock); + ack |= IGC_TSICR_TT1; + } -static void igc_tsync_interrupt(struct igc_adapter *adapter) -{ - struct igc_hw *hw = &adapter->hw; - u32 tsicr = rd32(IGC_TSICR); - u32 ack = 0; + if (tsicr & IGC_TSICR_AUTT0) { + nsec = rd32(IGC_AUXSTMPL0); + sec = rd32(IGC_AUXSTMPH0); + event.type = PTP_CLOCK_EXTTS; + event.index = 0; + event.timestamp = sec * NSEC_PER_SEC + nsec; + ptp_clock_event(adapter->ptp_clock, &event); + ack |= IGC_TSICR_AUTT0; + } - if (tsicr & IGC_TSICR_TXTS) { - /* retrieve hardware timestamp */ - schedule_work(&adapter->ptp_tx_work); - ack |= IGC_TSICR_TXTS; + if (tsicr & IGC_TSICR_AUTT1) { + nsec = rd32(IGC_AUXSTMPL1); + sec = rd32(IGC_AUXSTMPH1); + event.type = PTP_CLOCK_EXTTS; + event.index = 1; + event.timestamp = sec * NSEC_PER_SEC + nsec; + ptp_clock_event(adapter->ptp_clock, &event); + ack |= IGC_TSICR_AUTT1; } /* acknowledge the interrupts */ @@ -3824,7 +5137,7 @@ static irqreturn_t igc_msix_other(int irq, void *data) } if (icr & IGC_ICR_LSC) { - hw->mac.get_link_status = 1; + hw->mac.get_link_status = true; /* guard against interrupt when we're going down */ if (!test_bit(__IGC_DOWN, &adapter->state)) mod_timer(&adapter->watchdog_timer, jiffies + 1); @@ -3875,6 +5188,7 @@ static irqreturn_t igc_msix_ring(int irq, void *data) */ static int igc_request_msix(struct igc_adapter *adapter) { + unsigned int num_q_vectors = adapter->num_q_vectors; int i = 0, err = 0, vector = 0, free_vector = 0; struct net_device *netdev = adapter->netdev; @@ -3883,7 +5197,13 @@ static int igc_request_msix(struct igc_adapter *adapter) if (err) goto err_out; - for (i = 0; i < adapter->num_q_vectors; i++) { + if (num_q_vectors > MAX_Q_VECTORS) { + num_q_vectors = MAX_Q_VECTORS; + dev_warn(&adapter->pdev->dev, + "The number of queue vectors (%d) is higher than max allowed (%d)\n", + adapter->num_q_vectors, MAX_Q_VECTORS); + } + for (i = 0; i < num_q_vectors; i++) { struct igc_q_vector *q_vector = adapter->q_vector[i]; vector++; @@ -3962,20 +5282,12 @@ bool igc_has_link(struct igc_adapter *adapter) * false until the igc_check_for_link establishes link * for copper adapters ONLY */ - switch (hw->phy.media_type) { - case igc_media_type_copper: - if (!hw->mac.get_link_status) - return true; - hw->mac.ops.check_for_link(hw); - link_active = !hw->mac.get_link_status; - break; - default: - case igc_media_type_unknown: - break; - } + if (!hw->mac.get_link_status) + return true; + hw->mac.ops.check_for_link(hw); + link_active = !hw->mac.get_link_status; - if (hw->mac.type == igc_i225 && - hw->phy.id == I225_I_PHY_ID) { + if (hw->mac.type == igc_i225) { if (!netif_carrier_ok(adapter->netdev)) { adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { @@ -4007,7 +5319,6 @@ static void igc_watchdog_task(struct work_struct *work) struct igc_hw *hw = &adapter->hw; struct igc_phy_info *phy = &hw->phy; u16 phy_data, retry_count = 20; - u32 connsw; u32 link; int i; @@ -4020,15 +5331,10 @@ static void igc_watchdog_task(struct work_struct *work) link = false; } - /* Force link down if we have fiber to swap to */ - if (adapter->flags & IGC_FLAG_MAS_ENABLE) { - if (hw->phy.media_type == igc_media_type_copper) { - connsw = rd32(IGC_CONNSW); - if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) - link = 0; - } - } if (link) { + /* Cancel scheduled suspend requests. */ + pm_runtime_resume(netdev->dev.parent); + if (!netif_carrier_ok(netdev)) { u32 ctrl; @@ -4039,8 +5345,7 @@ static void igc_watchdog_task(struct work_struct *work) ctrl = rd32(IGC_CTRL); /* Link status message must follow this format */ netdev_info(netdev, - "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", - netdev->name, + "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", adapter->link_speed, adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half", @@ -4049,6 +5354,15 @@ static void igc_watchdog_task(struct work_struct *work) (ctrl & IGC_CTRL_RFCE) ? "RX" : (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); + /* disable EEE if enabled */ + if ((adapter->flags & IGC_FLAG_EEE) && + adapter->link_duplex == HALF_DUPLEX) { + netdev_info(netdev, + "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); + adapter->hw.dev_spec._base.eee_enable = false; + adapter->flags &= ~IGC_FLAG_EEE; + } + /* check if SmartSpeed worked */ igc_check_downshift(hw); if (phy->speed_downgraded) @@ -4061,7 +5375,9 @@ static void igc_watchdog_task(struct work_struct *work) adapter->tx_timeout_factor = 14; break; case SPEED_100: - /* maybe add some timeout factor ? */ + case SPEED_1000: + case SPEED_2500: + adapter->tx_timeout_factor = 7; break; } @@ -4078,10 +5394,10 @@ retry_read_status: retry_count--; goto retry_read_status; } else if (!retry_count) { - dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); + netdev_err(netdev, "exceed max 2 second\n"); } } else { - dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); + netdev_err(netdev, "read 1000Base-T Status Reg\n"); } no_wait: netif_carrier_on(netdev); @@ -4097,8 +5413,7 @@ no_wait: adapter->link_duplex = 0; /* Links status message must follow this format */ - netdev_info(netdev, "igc: %s NIC Link is Down\n", - netdev->name); + netdev_info(netdev, "NIC Link is Down\n"); netif_carrier_off(netdev); /* link state has changed, schedule phy info update */ @@ -4114,6 +5429,8 @@ no_wait: return; } } + pm_schedule_suspend(netdev->dev.parent, + MSEC_PER_SEC * 5); /* also check for alternate media here */ } else if (!netif_carrier_ok(netdev) && @@ -4199,11 +5516,14 @@ static irqreturn_t igc_intr_msi(int irq, void *data) } if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { - hw->mac.get_link_status = 1; + hw->mac.get_link_status = true; if (!test_bit(__IGC_DOWN, &adapter->state)) mod_timer(&adapter->watchdog_timer, jiffies + 1); } + if (icr & IGC_ICR_TS) + igc_tsync_interrupt(adapter); + napi_schedule(&q_vector->napi); return IRQ_HANDLED; @@ -4241,12 +5561,15 @@ static irqreturn_t igc_intr(int irq, void *data) } if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { - hw->mac.get_link_status = 1; + hw->mac.get_link_status = true; /* guard against interrupt when we're going down */ if (!test_bit(__IGC_DOWN, &adapter->state)) mod_timer(&adapter->watchdog_timer, jiffies + 1); } + if (icr & IGC_ICR_TS) + igc_tsync_interrupt(adapter); + napi_schedule(&q_vector->napi); return IRQ_HANDLED; @@ -4314,8 +5637,7 @@ static int igc_request_irq(struct igc_adapter *adapter) netdev->name, adapter); if (err) - dev_err(&pdev->dev, "Error %d getting interrupt\n", - err); + netdev_err(netdev, "Error %d getting interrupt\n", err); request_done: return err; @@ -4337,6 +5659,7 @@ request_done: static int __igc_open(struct net_device *netdev, bool resuming) { struct igc_adapter *adapter = netdev_priv(netdev); + struct pci_dev *pdev = adapter->pdev; struct igc_hw *hw = &adapter->hw; int err = 0; int i = 0; @@ -4348,6 +5671,9 @@ static int __igc_open(struct net_device *netdev, bool resuming) return -EBUSY; } + if (!resuming) + pm_runtime_get_sync(&pdev->dev); + netif_carrier_off(netdev); /* allocate transmit descriptors */ @@ -4386,10 +5712,13 @@ static int __igc_open(struct net_device *netdev, bool resuming) rd32(IGC_ICR); igc_irq_enable(adapter); + if (!resuming) + pm_runtime_put(&pdev->dev); + netif_tx_start_all_queues(netdev); /* start the watchdog. */ - hw->mac.get_link_status = 1; + hw->mac.get_link_status = true; schedule_work(&adapter->watchdog_task); return IGC_SUCCESS; @@ -4398,17 +5727,19 @@ err_set_queues: igc_free_irq(adapter); err_req_irq: igc_release_hw_control(adapter); - igc_power_down_link(adapter); + igc_power_down_phy_copper_base(&adapter->hw); igc_free_all_rx_resources(adapter); err_setup_rx: igc_free_all_tx_resources(adapter); err_setup_tx: igc_reset(adapter); + if (!resuming) + pm_runtime_put(&pdev->dev); return err; } -static int igc_open(struct net_device *netdev) +int igc_open(struct net_device *netdev) { return __igc_open(netdev, false); } @@ -4428,9 +5759,13 @@ static int igc_open(struct net_device *netdev) static int __igc_close(struct net_device *netdev, bool suspending) { struct igc_adapter *adapter = netdev_priv(netdev); + struct pci_dev *pdev = adapter->pdev; WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); + if (!suspending) + pm_runtime_get_sync(&pdev->dev); + igc_down(adapter); igc_release_hw_control(adapter); @@ -4440,10 +5775,13 @@ static int __igc_close(struct net_device *netdev, bool suspending) igc_free_all_tx_resources(adapter); igc_free_all_rx_resources(adapter); + if (!suspending) + pm_runtime_put_sync(&pdev->dev); + return 0; } -static int igc_close(struct net_device *netdev) +int igc_close(struct net_device *netdev) { if (netif_device_present(netdev) || netdev->dismantle) return __igc_close(netdev, false); @@ -4453,7 +5791,7 @@ static int igc_close(struct net_device *netdev) /** * igc_ioctl - Access the hwtstamp interface * @netdev: network interface device structure - * @ifreq: interface request data + * @ifr: interface request data * @cmd: ioctl command **/ static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) @@ -4468,6 +5806,356 @@ static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) } } +static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, + bool enable) +{ + struct igc_ring *ring; + + if (queue < 0 || queue >= adapter->num_tx_queues) + return -EINVAL; + + ring = adapter->tx_ring[queue]; + ring->launchtime_enable = enable; + + return 0; +} + +static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) +{ + struct timespec64 b; + + b = ktime_to_timespec64(base_time); + + return timespec64_compare(now, &b) > 0; +} + +static bool validate_schedule(struct igc_adapter *adapter, + const struct tc_taprio_qopt_offload *qopt) +{ + int queue_uses[IGC_MAX_TX_QUEUES] = { }; + struct timespec64 now; + size_t n; + + if (qopt->cycle_time_extension) + return false; + + igc_ptp_read(adapter, &now); + + /* If we program the controller's BASET registers with a time + * in the future, it will hold all the packets until that + * time, causing a lot of TX Hangs, so to avoid that, we + * reject schedules that would start in the future. + */ + if (!is_base_time_past(qopt->base_time, &now)) + return false; + + for (n = 0; n < qopt->num_entries; n++) { + const struct tc_taprio_sched_entry *e, *prev; + int i; + + prev = n ? &qopt->entries[n - 1] : NULL; + e = &qopt->entries[n]; + + /* i225 only supports "global" frame preemption + * settings. + */ + if (e->command != TC_TAPRIO_CMD_SET_GATES) + return false; + + for (i = 0; i < adapter->num_tx_queues; i++) { + if (e->gate_mask & BIT(i)) + queue_uses[i]++; + + /* There are limitations: A single queue cannot be + * opened and closed multiple times per cycle unless the + * gate stays open. Check for it. + */ + if (queue_uses[i] > 1 && + !(prev->gate_mask & BIT(i))) + return false; + } + } + + return true; +} + +static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, + struct tc_etf_qopt_offload *qopt) +{ + struct igc_hw *hw = &adapter->hw; + int err; + + if (hw->mac.type != igc_i225) + return -EOPNOTSUPP; + + err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); + if (err) + return err; + + return igc_tsn_offload_apply(adapter); +} + +static int igc_tsn_clear_schedule(struct igc_adapter *adapter) +{ + int i; + + adapter->base_time = 0; + adapter->cycle_time = NSEC_PER_SEC; + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + ring->start_time = 0; + ring->end_time = NSEC_PER_SEC; + } + + return 0; +} + +static int igc_save_qbv_schedule(struct igc_adapter *adapter, + struct tc_taprio_qopt_offload *qopt) +{ + bool queue_configured[IGC_MAX_TX_QUEUES] = { }; + u32 start_time = 0, end_time = 0; + size_t n; + + if (!qopt->enable) + return igc_tsn_clear_schedule(adapter); + + if (adapter->base_time) + return -EALREADY; + + if (!validate_schedule(adapter, qopt)) + return -EINVAL; + + adapter->cycle_time = qopt->cycle_time; + adapter->base_time = qopt->base_time; + + for (n = 0; n < qopt->num_entries; n++) { + struct tc_taprio_sched_entry *e = &qopt->entries[n]; + int i; + + end_time += e->interval; + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + if (!(e->gate_mask & BIT(i))) + continue; + + /* Check whether a queue stays open for more than one + * entry. If so, keep the start and advance the end + * time. + */ + if (!queue_configured[i]) + ring->start_time = start_time; + ring->end_time = end_time; + + queue_configured[i] = true; + } + + start_time += e->interval; + } + + return 0; +} + +static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, + struct tc_taprio_qopt_offload *qopt) +{ + struct igc_hw *hw = &adapter->hw; + int err; + + if (hw->mac.type != igc_i225) + return -EOPNOTSUPP; + + err = igc_save_qbv_schedule(adapter, qopt); + if (err) + return err; + + return igc_tsn_offload_apply(adapter); +} + +static int igc_save_cbs_params(struct igc_adapter *adapter, int queue, + bool enable, int idleslope, int sendslope, + int hicredit, int locredit) +{ + bool cbs_status[IGC_MAX_SR_QUEUES] = { false }; + struct net_device *netdev = adapter->netdev; + struct igc_ring *ring; + int i; + + /* i225 has two sets of credit-based shaper logic. + * Supporting it only on the top two priority queues + */ + if (queue < 0 || queue > 1) + return -EINVAL; + + ring = adapter->tx_ring[queue]; + + for (i = 0; i < IGC_MAX_SR_QUEUES; i++) + if (adapter->tx_ring[i]) + cbs_status[i] = adapter->tx_ring[i]->cbs_enable; + + /* CBS should be enabled on the highest priority queue first in order + * for the CBS algorithm to operate as intended. + */ + if (enable) { + if (queue == 1 && !cbs_status[0]) { + netdev_err(netdev, + "Enabling CBS on queue1 before queue0\n"); + return -EINVAL; + } + } else { + if (queue == 0 && cbs_status[1]) { + netdev_err(netdev, + "Disabling CBS on queue0 before queue1\n"); + return -EINVAL; + } + } + + ring->cbs_enable = enable; + ring->idleslope = idleslope; + ring->sendslope = sendslope; + ring->hicredit = hicredit; + ring->locredit = locredit; + + return 0; +} + +static int igc_tsn_enable_cbs(struct igc_adapter *adapter, + struct tc_cbs_qopt_offload *qopt) +{ + struct igc_hw *hw = &adapter->hw; + int err; + + if (hw->mac.type != igc_i225) + return -EOPNOTSUPP; + + if (qopt->queue < 0 || qopt->queue > 1) + return -EINVAL; + + err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable, + qopt->idleslope, qopt->sendslope, + qopt->hicredit, qopt->locredit); + if (err) + return err; + + return igc_tsn_offload_apply(adapter); +} + +static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, + void *type_data) +{ + struct igc_adapter *adapter = netdev_priv(dev); + + switch (type) { + case TC_SETUP_QDISC_TAPRIO: + return igc_tsn_enable_qbv_scheduling(adapter, type_data); + + case TC_SETUP_QDISC_ETF: + return igc_tsn_enable_launchtime(adapter, type_data); + + case TC_SETUP_QDISC_CBS: + return igc_tsn_enable_cbs(adapter, type_data); + + default: + return -EOPNOTSUPP; + } +} + +static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf) +{ + struct igc_adapter *adapter = netdev_priv(dev); + + switch (bpf->command) { + case XDP_SETUP_PROG: + return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack); + case XDP_SETUP_XSK_POOL: + return igc_xdp_setup_pool(adapter, bpf->xsk.pool, + bpf->xsk.queue_id); + default: + return -EOPNOTSUPP; + } +} + +static int igc_xdp_xmit(struct net_device *dev, int num_frames, + struct xdp_frame **frames, u32 flags) +{ + struct igc_adapter *adapter = netdev_priv(dev); + int cpu = smp_processor_id(); + struct netdev_queue *nq; + struct igc_ring *ring; + int i, drops; + + if (unlikely(test_bit(__IGC_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + ring = igc_xdp_get_tx_ring(adapter, cpu); + nq = txring_txq(ring); + + __netif_tx_lock(nq, cpu); + + drops = 0; + for (i = 0; i < num_frames; i++) { + int err; + struct xdp_frame *xdpf = frames[i]; + + err = igc_xdp_init_tx_descriptor(ring, xdpf); + if (err) { + xdp_return_frame_rx_napi(xdpf); + drops++; + } + } + + if (flags & XDP_XMIT_FLUSH) + igc_flush_tx_descriptors(ring); + + __netif_tx_unlock(nq); + + return num_frames - drops; +} + +static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter, + struct igc_q_vector *q_vector) +{ + struct igc_hw *hw = &adapter->hw; + u32 eics = 0; + + eics |= q_vector->eims_value; + wr32(IGC_EICS, eics); +} + +int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) +{ + struct igc_adapter *adapter = netdev_priv(dev); + struct igc_q_vector *q_vector; + struct igc_ring *ring; + + if (test_bit(__IGC_DOWN, &adapter->state)) + return -ENETDOWN; + + if (!igc_xdp_is_enabled(adapter)) + return -ENXIO; + + if (queue_id >= adapter->num_rx_queues) + return -EINVAL; + + ring = adapter->rx_ring[queue_id]; + + if (!ring->xsk_pool) + return -ENXIO; + + q_vector = adapter->q_vector[queue_id]; + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + igc_trigger_rxtxq_interrupt(adapter, q_vector); + + return 0; +} + static const struct net_device_ops igc_netdev_ops = { .ndo_open = igc_open, .ndo_stop = igc_close, @@ -4475,11 +6163,15 @@ static const struct net_device_ops igc_netdev_ops = { .ndo_set_rx_mode = igc_set_rx_mode, .ndo_set_mac_address = igc_set_mac, .ndo_change_mtu = igc_change_mtu, - .ndo_get_stats = igc_get_stats, + .ndo_get_stats64 = igc_get_stats64, .ndo_fix_features = igc_fix_features, .ndo_set_features = igc_set_features, .ndo_features_check = igc_features_check, - .ndo_do_ioctl = igc_ioctl, + .ndo_eth_ioctl = igc_ioctl, + .ndo_setup_tc = igc_setup_tc, + .ndo_bpf = igc_bpf, + .ndo_xdp_xmit = igc_xdp_xmit, + .ndo_xsk_wakeup = igc_xsk_wakeup, }; /* PCIe configuration access */ @@ -4546,57 +6238,6 @@ u32 igc_rd32(struct igc_hw *hw, u32 reg) return value; } -int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) -{ - struct pci_dev *pdev = adapter->pdev; - struct igc_mac_info *mac = &adapter->hw.mac; - - mac->autoneg = 0; - - /* Make sure dplx is at most 1 bit and lsb of speed is not set - * for the switch() below to work - */ - if ((spd & 1) || (dplx & ~1)) - goto err_inval; - - switch (spd + dplx) { - case SPEED_10 + DUPLEX_HALF: - mac->forced_speed_duplex = ADVERTISE_10_HALF; - break; - case SPEED_10 + DUPLEX_FULL: - mac->forced_speed_duplex = ADVERTISE_10_FULL; - break; - case SPEED_100 + DUPLEX_HALF: - mac->forced_speed_duplex = ADVERTISE_100_HALF; - break; - case SPEED_100 + DUPLEX_FULL: - mac->forced_speed_duplex = ADVERTISE_100_FULL; - break; - case SPEED_1000 + DUPLEX_FULL: - mac->autoneg = 1; - adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; - break; - case SPEED_1000 + DUPLEX_HALF: /* not supported */ - goto err_inval; - case SPEED_2500 + DUPLEX_FULL: - mac->autoneg = 1; - adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; - break; - case SPEED_2500 + DUPLEX_HALF: /* not supported */ - default: - goto err_inval; - } - - /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ - adapter->hw.phy.mdix = AUTO_ALL_MODES; - - return 0; - -err_inval: - dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); - return -EINVAL; -} - /** * igc_probe - Device Initialization Routine * @pdev: PCI device information struct @@ -4615,23 +6256,17 @@ static int igc_probe(struct pci_dev *pdev, struct net_device *netdev; struct igc_hw *hw; const struct igc_info *ei = igc_info_tbl[ent->driver_data]; - int err, pci_using_dac; + int err; err = pci_enable_device_mem(pdev); if (err) return err; - pci_using_dac = 0; err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (!err) { - pci_using_dac = 1; - } else { - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, - "No usable DMA configuration, aborting\n"); - goto err_dma; - } + if (err) { + dev_err(&pdev->dev, + "No usable DMA configuration, aborting\n"); + goto err_dma; } err = pci_request_mem_regions(pdev, igc_driver_name); @@ -4640,6 +6275,10 @@ static int igc_probe(struct pci_dev *pdev, pci_enable_pcie_error_reporting(pdev); + err = pci_enable_ptm(pdev, NULL); + if (err < 0) + dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n"); + pci_set_master(pdev); err = -ENOMEM; @@ -4674,7 +6313,7 @@ static int igc_probe(struct pci_dev *pdev, hw->hw_addr = adapter->io_addr; netdev->netdev_ops = &igc_netdev_ops; - igc_set_ethtool_ops(netdev); + igc_ethtool_set_ops(netdev); netdev->watchdog_timeo = 5 * HZ; netdev->mem_start = pci_resource_start(pdev, 0); @@ -4700,9 +6339,21 @@ static int igc_probe(struct pci_dev *pdev, netdev->features |= NETIF_F_SG; netdev->features |= NETIF_F_TSO; netdev->features |= NETIF_F_TSO6; + netdev->features |= NETIF_F_TSO_ECN; netdev->features |= NETIF_F_RXCSUM; netdev->features |= NETIF_F_HW_CSUM; netdev->features |= NETIF_F_SCTP_CRC; + netdev->features |= NETIF_F_HW_TC; + +#define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ + NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_IPXIP4 | \ + NETIF_F_GSO_IPXIP6 | \ + NETIF_F_GSO_UDP_TUNNEL | \ + NETIF_F_GSO_UDP_TUNNEL_CSUM) + + netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; + netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; /* setup the private structure */ err = igc_sw_init(adapter); @@ -4711,10 +6362,15 @@ static int igc_probe(struct pci_dev *pdev, /* copy netdev features into list of user selectable features */ netdev->hw_features |= NETIF_F_NTUPLE; + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; netdev->hw_features |= netdev->features; - if (pci_using_dac) - netdev->features |= NETIF_F_HIGHDMA; + netdev->features |= NETIF_F_HIGHDMA; + + netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; + netdev->mpls_features |= NETIF_F_HW_CSUM; + netdev->hw_enc_features |= netdev->vlan_features; /* MTU range: 68 - 9216 */ netdev->min_mtu = ETH_MIN_MTU; @@ -4727,8 +6383,7 @@ static int igc_probe(struct pci_dev *pdev, if (igc_get_flash_presence_i225(hw)) { if (hw->nvm.ops.validate(hw) < 0) { - dev_err(&pdev->dev, - "The NVM Checksum Is Not Valid\n"); + dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); err = -EIO; goto err_eeprom; } @@ -4740,7 +6395,7 @@ static int igc_probe(struct pci_dev *pdev, dev_err(&pdev->dev, "NVM Read Error\n"); } - memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); + eth_hw_addr_set(netdev, hw->mac.addr); if (!is_valid_ether_addr(netdev->dev_addr)) { dev_err(&pdev->dev, "Invalid MAC Address\n"); @@ -4766,6 +6421,20 @@ static int igc_probe(struct pci_dev *pdev, hw->fc.requested_mode = igc_fc_default; hw->fc.current_mode = igc_fc_default; + /* By default, support wake on port A */ + adapter->flags |= IGC_FLAG_WOL_SUPPORTED; + + /* initialize the wol settings based on the eeprom settings */ + if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) + adapter->wol |= IGC_WUFC_MAG; + + device_set_wakeup_enable(&adapter->pdev->dev, + adapter->flags & IGC_FLAG_WOL_SUPPORTED); + + igc_ptp_init(adapter); + + igc_tsn_clear_schedule(adapter); + /* reset the hardware with the new settings */ igc_reset(adapter); @@ -4782,9 +6451,6 @@ static int igc_probe(struct pci_dev *pdev, /* carrier off reporting is important to ethtool even BEFORE open */ netif_carrier_off(netdev); - /* do hw tstamp init after resetting */ - igc_ptp_init(adapter); - /* Check if Media Autosense is enabled */ adapter->ei = *ei; @@ -4792,6 +6458,14 @@ static int igc_probe(struct pci_dev *pdev, pcie_print_link_status(pdev); netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); + dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); + /* Disable EEE for internal PHY devices */ + hw->dev_spec._base.eee_enable = false; + adapter->flags &= ~IGC_FLAG_EEE; + igc_set_eee_i225(hw, false, false, false); + + pm_runtime_put_noidle(&pdev->dev); + return 0; err_register: @@ -4805,6 +6479,7 @@ err_sw_init: err_ioremap: free_netdev(netdev); err_alloc_etherdev: + pci_disable_pcie_error_reporting(pdev); pci_release_mem_regions(pdev); err_pci_reg: err_dma: @@ -4826,6 +6501,10 @@ static void igc_remove(struct pci_dev *pdev) struct net_device *netdev = pci_get_drvdata(pdev); struct igc_adapter *adapter = netdev_priv(netdev); + pm_runtime_get_noresume(&pdev->dev); + + igc_flush_nfc_rules(adapter); + igc_ptp_stop(adapter); set_bit(__IGC_DOWN, &adapter->state); @@ -4846,7 +6525,6 @@ static void igc_remove(struct pci_dev *pdev) pci_iounmap(pdev, adapter->io_addr); pci_release_mem_regions(pdev); - kfree(adapter->mac_table); free_netdev(netdev); pci_disable_pcie_error_reporting(pdev); @@ -4870,6 +6548,8 @@ static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, if (netif_running(netdev)) __igc_close(netdev, true); + igc_ptp_suspend(adapter); + igc_clear_interrupt_scheme(adapter); rtnl_unlock(); @@ -4904,7 +6584,7 @@ static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, wake = wufc || adapter->en_mng_pt; if (!wake) - igc_power_down_link(adapter); + igc_power_down_phy_copper_base(&adapter->hw); else igc_power_up_link(adapter); @@ -4973,8 +6653,7 @@ static int __maybe_unused igc_resume(struct device *dev) return -ENODEV; err = pci_enable_device_mem(pdev); if (err) { - dev_err(&pdev->dev, - "igc: Cannot enable PCI device from suspend\n"); + netdev_err(netdev, "Cannot enable PCI device from suspend\n"); return err; } pci_set_master(pdev); @@ -4983,7 +6662,7 @@ static int __maybe_unused igc_resume(struct device *dev) pci_enable_wake(pdev, PCI_D3cold, 0); if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; } @@ -5045,6 +6724,107 @@ static void igc_shutdown(struct pci_dev *pdev) } } +/** + * igc_io_error_detected - called when PCI error is detected + * @pdev: Pointer to PCI device + * @state: The current PCI connection state + * + * This function is called after a PCI bus error affecting + * this device has been detected. + **/ +static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct igc_adapter *adapter = netdev_priv(netdev); + + netif_device_detach(netdev); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + if (netif_running(netdev)) + igc_down(adapter); + pci_disable_device(pdev); + + /* Request a slot reset. */ + return PCI_ERS_RESULT_NEED_RESET; +} + +/** + * igc_io_slot_reset - called after the PCI bus has been reset. + * @pdev: Pointer to PCI device + * + * Restart the card from scratch, as if from a cold-boot. Implementation + * resembles the first-half of the igc_resume routine. + **/ +static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + pci_ers_result_t result; + + if (pci_enable_device_mem(pdev)) { + netdev_err(netdev, "Could not re-enable PCI device after reset\n"); + result = PCI_ERS_RESULT_DISCONNECT; + } else { + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + + pci_enable_wake(pdev, PCI_D3hot, 0); + pci_enable_wake(pdev, PCI_D3cold, 0); + + /* In case of PCI error, adapter loses its HW address + * so we should re-assign it here. + */ + hw->hw_addr = adapter->io_addr; + + igc_reset(adapter); + wr32(IGC_WUS, ~0); + result = PCI_ERS_RESULT_RECOVERED; + } + + return result; +} + +/** + * igc_io_resume - called when traffic can start to flow again. + * @pdev: Pointer to PCI device + * + * This callback is called when the error recovery driver tells us that + * its OK to resume normal operation. Implementation resembles the + * second-half of the igc_resume routine. + */ +static void igc_io_resume(struct pci_dev *pdev) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct igc_adapter *adapter = netdev_priv(netdev); + + rtnl_lock(); + if (netif_running(netdev)) { + if (igc_open(netdev)) { + netdev_err(netdev, "igc_open failed after reset\n"); + return; + } + } + + netif_device_attach(netdev); + + /* let the f/w know that the h/w is now under the control of the + * driver. + */ + igc_get_hw_control(adapter); + rtnl_unlock(); +} + +static const struct pci_error_handlers igc_err_handler = { + .error_detected = igc_io_error_detected, + .slot_reset = igc_io_slot_reset, + .resume = igc_io_resume, +}; + #ifdef CONFIG_PM static const struct dev_pm_ops igc_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) @@ -5062,6 +6842,7 @@ static struct pci_driver igc_driver = { .driver.pm = &igc_pm_ops, #endif .shutdown = igc_shutdown, + .err_handler = &igc_err_handler, }; /** @@ -5071,7 +6852,6 @@ static struct pci_driver igc_driver = { int igc_reinit_queues(struct igc_adapter *adapter) { struct net_device *netdev = adapter->netdev; - struct pci_dev *pdev = adapter->pdev; int err = 0; if (netif_running(netdev)) @@ -5080,7 +6860,7 @@ int igc_reinit_queues(struct igc_adapter *adapter) igc_reset_interrupt_capability(adapter); if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; } @@ -5103,6 +6883,61 @@ struct net_device *igc_get_hw_dev(struct igc_hw *hw) return adapter->netdev; } +static void igc_disable_rx_ring_hw(struct igc_ring *ring) +{ + struct igc_hw *hw = &ring->q_vector->adapter->hw; + u8 idx = ring->reg_idx; + u32 rxdctl; + + rxdctl = rd32(IGC_RXDCTL(idx)); + rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE; + rxdctl |= IGC_RXDCTL_SWFLUSH; + wr32(IGC_RXDCTL(idx), rxdctl); +} + +void igc_disable_rx_ring(struct igc_ring *ring) +{ + igc_disable_rx_ring_hw(ring); + igc_clean_rx_ring(ring); +} + +void igc_enable_rx_ring(struct igc_ring *ring) +{ + struct igc_adapter *adapter = ring->q_vector->adapter; + + igc_configure_rx_ring(adapter, ring); + + if (ring->xsk_pool) + igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); + else + igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); +} + +static void igc_disable_tx_ring_hw(struct igc_ring *ring) +{ + struct igc_hw *hw = &ring->q_vector->adapter->hw; + u8 idx = ring->reg_idx; + u32 txdctl; + + txdctl = rd32(IGC_TXDCTL(idx)); + txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE; + txdctl |= IGC_TXDCTL_SWFLUSH; + wr32(IGC_TXDCTL(idx), txdctl); +} + +void igc_disable_tx_ring(struct igc_ring *ring) +{ + igc_disable_tx_ring_hw(ring); + igc_clean_tx_ring(ring); +} + +void igc_enable_tx_ring(struct igc_ring *ring) +{ + struct igc_adapter *adapter = ring->q_vector->adapter; + + igc_configure_tx_ring(adapter, ring); +} + /** * igc_init_module - Driver Registration Routine * @@ -5113,9 +6948,7 @@ static int __init igc_init_module(void) { int ret; - pr_info("%s - version %s\n", - igc_driver_string, igc_driver_version); - + pr_info("%s\n", igc_driver_string); pr_info("%s\n", igc_copyright); ret = pci_register_driver(&igc_driver); diff --git a/drivers/net/ethernet/intel/igc/igc_phy.c b/drivers/net/ethernet/intel/igc/igc_phy.c index 8e1799508edc..53b77c969c85 100644 --- a/drivers/net/ethernet/intel/igc/igc_phy.c +++ b/drivers/net/ethernet/intel/igc/igc_phy.c @@ -141,24 +141,14 @@ void igc_power_down_phy_copper(struct igc_hw *hw) * igc_check_downshift - Checks whether a downshift in speed occurred * @hw: pointer to the HW structure * - * Success returns 0, Failure returns 1 - * * A downshift is detected by querying the PHY link health. */ -s32 igc_check_downshift(struct igc_hw *hw) +void igc_check_downshift(struct igc_hw *hw) { struct igc_phy_info *phy = &hw->phy; - s32 ret_val; - - switch (phy->type) { - case igc_phy_i225: - default: - /* speed downshift not supported */ - phy->speed_downgraded = false; - ret_val = 0; - } - return ret_val; + /* speed downshift not supported */ + phy->speed_downgraded = false; } /** @@ -249,8 +239,7 @@ static s32 igc_phy_setup_autoneg(struct igc_hw *hw) return ret_val; } - if ((phy->autoneg_mask & ADVERTISE_2500_FULL) && - hw->phy.id == I225_I_PHY_ID) { + if (phy->autoneg_mask & ADVERTISE_2500_FULL) { /* Read the MULTI GBT AN Control Register - reg 7.32 */ ret_val = phy->ops.read_reg(hw, (STANDARD_AN_REG_MASK << MMD_DEVADDR_SHIFT) | @@ -390,8 +379,7 @@ static s32 igc_phy_setup_autoneg(struct igc_hw *hw) ret_val = phy->ops.write_reg(hw, PHY_1000T_CTRL, mii_1000t_ctrl_reg); - if ((phy->autoneg_mask & ADVERTISE_2500_FULL) && - hw->phy.id == I225_I_PHY_ID) + if (phy->autoneg_mask & ADVERTISE_2500_FULL) ret_val = phy->ops.write_reg(hw, (STANDARD_AN_REG_MASK << MMD_DEVADDR_SHIFT) | @@ -583,7 +571,7 @@ static s32 igc_read_phy_reg_mdic(struct igc_hw *hw, u32 offset, u16 *data) * the lower time out */ for (i = 0; i < IGC_GEN_POLL_TIMEOUT; i++) { - usleep_range(500, 1000); + udelay(50); mdic = rd32(IGC_MDIC); if (mdic & IGC_MDIC_READY) break; @@ -640,7 +628,7 @@ static s32 igc_write_phy_reg_mdic(struct igc_hw *hw, u32 offset, u16 data) * the lower time out */ for (i = 0; i < IGC_GEN_POLL_TIMEOUT; i++) { - usleep_range(500, 1000); + udelay(50); mdic = rd32(IGC_MDIC); if (mdic & IGC_MDIC_READY) break; @@ -748,8 +736,6 @@ s32 igc_write_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 data) if (ret_val) return ret_val; ret_val = igc_write_phy_reg_mdic(hw, offset, data); - if (ret_val) - return ret_val; hw->phy.ops.release(hw); } else { ret_val = igc_write_xmdio_reg(hw, (u16)offset, dev_addr, @@ -781,8 +767,6 @@ s32 igc_read_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 *data) if (ret_val) return ret_val; ret_val = igc_read_phy_reg_mdic(hw, offset, data); - if (ret_val) - return ret_val; hw->phy.ops.release(hw); } else { ret_val = igc_read_xmdio_reg(hw, (u16)offset, dev_addr, @@ -791,3 +775,21 @@ s32 igc_read_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 *data) return ret_val; } + +/** + * igc_read_phy_fw_version - Read gPHY firmware version + * @hw: pointer to the HW structure + */ +u16 igc_read_phy_fw_version(struct igc_hw *hw) +{ + struct igc_phy_info *phy = &hw->phy; + u16 gphy_version = 0; + u16 ret_val; + + /* NVM image version is reported as firmware version for i225 device */ + ret_val = phy->ops.read_reg(hw, IGC_GPHY_VERSION, &gphy_version); + if (ret_val) + hw_dbg("igc_phy: read wrong gphy version\n"); + + return gphy_version; +} diff --git a/drivers/net/ethernet/intel/igc/igc_phy.h b/drivers/net/ethernet/intel/igc/igc_phy.h index 25cba33de7e2..832a7e359f18 100644 --- a/drivers/net/ethernet/intel/igc/igc_phy.h +++ b/drivers/net/ethernet/intel/igc/igc_phy.h @@ -11,11 +11,12 @@ s32 igc_phy_hw_reset(struct igc_hw *hw); s32 igc_get_phy_id(struct igc_hw *hw); s32 igc_phy_has_link(struct igc_hw *hw, u32 iterations, u32 usec_interval, bool *success); -s32 igc_check_downshift(struct igc_hw *hw); +void igc_check_downshift(struct igc_hw *hw); s32 igc_setup_copper_link(struct igc_hw *hw); void igc_power_up_phy_copper(struct igc_hw *hw); void igc_power_down_phy_copper(struct igc_hw *hw); s32 igc_write_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 data); s32 igc_read_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 *data); +u16 igc_read_phy_fw_version(struct igc_hw *hw); #endif diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index 693506587198..8dbb9f903ca7 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -8,25 +8,25 @@ #include <linux/pci.h> #include <linux/ptp_classify.h> #include <linux/clocksource.h> +#include <linux/ktime.h> +#include <linux/delay.h> +#include <linux/iopoll.h> #define INCVALUE_MASK 0x7fffffff #define ISGN 0x80000000 -#define IGC_SYSTIM_OVERFLOW_PERIOD (HZ * 60 * 9) #define IGC_PTP_TX_TIMEOUT (HZ * 15) +#define IGC_PTM_STAT_SLEEP 2 +#define IGC_PTM_STAT_TIMEOUT 100 + /* SYSTIM read access for I225 */ -static void igc_ptp_read_i225(struct igc_adapter *adapter, - struct timespec64 *ts) +void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts) { struct igc_hw *hw = &adapter->hw; u32 sec, nsec; - /* The timestamp latches on lowest register read. For I210/I211, the - * lowest register is SYSTIMR. Since we only need to provide nanosecond - * resolution, we can ignore it. - */ - rd32(IGC_SYSTIMR); + /* The timestamp is latched when SYSTIML is read. */ nsec = rd32(IGC_SYSTIML); sec = rd32(IGC_SYSTIMH); @@ -39,9 +39,6 @@ static void igc_ptp_write_i225(struct igc_adapter *adapter, { struct igc_hw *hw = &adapter->hw; - /* Writing the SYSTIMR register is not necessary as it only - * provides sub-nanosecond resolution. - */ wr32(IGC_SYSTIML, ts->tv_nsec); wr32(IGC_SYSTIMH, ts->tv_sec); } @@ -81,7 +78,7 @@ static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta) spin_lock_irqsave(&igc->tmreg_lock, flags); - igc_ptp_read_i225(igc, &now); + igc_ptp_read(igc, &now); now = timespec64_add(now, then); igc_ptp_write_i225(igc, (const struct timespec64 *)&now); @@ -102,10 +99,9 @@ static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp, spin_lock_irqsave(&igc->tmreg_lock, flags); ptp_read_system_prets(sts); - rd32(IGC_SYSTIMR); - ptp_read_system_postts(sts); ts->tv_nsec = rd32(IGC_SYSTIML); ts->tv_sec = rd32(IGC_SYSTIMH); + ptp_read_system_postts(sts); spin_unlock_irqrestore(&igc->tmreg_lock, flags); @@ -128,12 +124,289 @@ static int igc_ptp_settime_i225(struct ptp_clock_info *ptp, return 0; } +static void igc_pin_direction(int pin, int input, u32 *ctrl, u32 *ctrl_ext) +{ + u32 *ptr = pin < 2 ? ctrl : ctrl_ext; + static const u32 mask[IGC_N_SDP] = { + IGC_CTRL_SDP0_DIR, + IGC_CTRL_SDP1_DIR, + IGC_CTRL_EXT_SDP2_DIR, + IGC_CTRL_EXT_SDP3_DIR, + }; + + if (input) + *ptr &= ~mask[pin]; + else + *ptr |= mask[pin]; +} + +static void igc_pin_perout(struct igc_adapter *igc, int chan, int pin, int freq) +{ + static const u32 igc_aux0_sel_sdp[IGC_N_SDP] = { + IGC_AUX0_SEL_SDP0, IGC_AUX0_SEL_SDP1, IGC_AUX0_SEL_SDP2, IGC_AUX0_SEL_SDP3, + }; + static const u32 igc_aux1_sel_sdp[IGC_N_SDP] = { + IGC_AUX1_SEL_SDP0, IGC_AUX1_SEL_SDP1, IGC_AUX1_SEL_SDP2, IGC_AUX1_SEL_SDP3, + }; + static const u32 igc_ts_sdp_en[IGC_N_SDP] = { + IGC_TS_SDP0_EN, IGC_TS_SDP1_EN, IGC_TS_SDP2_EN, IGC_TS_SDP3_EN, + }; + static const u32 igc_ts_sdp_sel_tt0[IGC_N_SDP] = { + IGC_TS_SDP0_SEL_TT0, IGC_TS_SDP1_SEL_TT0, + IGC_TS_SDP2_SEL_TT0, IGC_TS_SDP3_SEL_TT0, + }; + static const u32 igc_ts_sdp_sel_tt1[IGC_N_SDP] = { + IGC_TS_SDP0_SEL_TT1, IGC_TS_SDP1_SEL_TT1, + IGC_TS_SDP2_SEL_TT1, IGC_TS_SDP3_SEL_TT1, + }; + static const u32 igc_ts_sdp_sel_fc0[IGC_N_SDP] = { + IGC_TS_SDP0_SEL_FC0, IGC_TS_SDP1_SEL_FC0, + IGC_TS_SDP2_SEL_FC0, IGC_TS_SDP3_SEL_FC0, + }; + static const u32 igc_ts_sdp_sel_fc1[IGC_N_SDP] = { + IGC_TS_SDP0_SEL_FC1, IGC_TS_SDP1_SEL_FC1, + IGC_TS_SDP2_SEL_FC1, IGC_TS_SDP3_SEL_FC1, + }; + static const u32 igc_ts_sdp_sel_clr[IGC_N_SDP] = { + IGC_TS_SDP0_SEL_FC1, IGC_TS_SDP1_SEL_FC1, + IGC_TS_SDP2_SEL_FC1, IGC_TS_SDP3_SEL_FC1, + }; + struct igc_hw *hw = &igc->hw; + u32 ctrl, ctrl_ext, tssdp = 0; + + ctrl = rd32(IGC_CTRL); + ctrl_ext = rd32(IGC_CTRL_EXT); + tssdp = rd32(IGC_TSSDP); + + igc_pin_direction(pin, 0, &ctrl, &ctrl_ext); + + /* Make sure this pin is not enabled as an input. */ + if ((tssdp & IGC_AUX0_SEL_SDP3) == igc_aux0_sel_sdp[pin]) + tssdp &= ~IGC_AUX0_TS_SDP_EN; + + if ((tssdp & IGC_AUX1_SEL_SDP3) == igc_aux1_sel_sdp[pin]) + tssdp &= ~IGC_AUX1_TS_SDP_EN; + + tssdp &= ~igc_ts_sdp_sel_clr[pin]; + if (freq) { + if (chan == 1) + tssdp |= igc_ts_sdp_sel_fc1[pin]; + else + tssdp |= igc_ts_sdp_sel_fc0[pin]; + } else { + if (chan == 1) + tssdp |= igc_ts_sdp_sel_tt1[pin]; + else + tssdp |= igc_ts_sdp_sel_tt0[pin]; + } + tssdp |= igc_ts_sdp_en[pin]; + + wr32(IGC_TSSDP, tssdp); + wr32(IGC_CTRL, ctrl); + wr32(IGC_CTRL_EXT, ctrl_ext); +} + +static void igc_pin_extts(struct igc_adapter *igc, int chan, int pin) +{ + static const u32 igc_aux0_sel_sdp[IGC_N_SDP] = { + IGC_AUX0_SEL_SDP0, IGC_AUX0_SEL_SDP1, IGC_AUX0_SEL_SDP2, IGC_AUX0_SEL_SDP3, + }; + static const u32 igc_aux1_sel_sdp[IGC_N_SDP] = { + IGC_AUX1_SEL_SDP0, IGC_AUX1_SEL_SDP1, IGC_AUX1_SEL_SDP2, IGC_AUX1_SEL_SDP3, + }; + static const u32 igc_ts_sdp_en[IGC_N_SDP] = { + IGC_TS_SDP0_EN, IGC_TS_SDP1_EN, IGC_TS_SDP2_EN, IGC_TS_SDP3_EN, + }; + struct igc_hw *hw = &igc->hw; + u32 ctrl, ctrl_ext, tssdp = 0; + + ctrl = rd32(IGC_CTRL); + ctrl_ext = rd32(IGC_CTRL_EXT); + tssdp = rd32(IGC_TSSDP); + + igc_pin_direction(pin, 1, &ctrl, &ctrl_ext); + + /* Make sure this pin is not enabled as an output. */ + tssdp &= ~igc_ts_sdp_en[pin]; + + if (chan == 1) { + tssdp &= ~IGC_AUX1_SEL_SDP3; + tssdp |= igc_aux1_sel_sdp[pin] | IGC_AUX1_TS_SDP_EN; + } else { + tssdp &= ~IGC_AUX0_SEL_SDP3; + tssdp |= igc_aux0_sel_sdp[pin] | IGC_AUX0_TS_SDP_EN; + } + + wr32(IGC_TSSDP, tssdp); + wr32(IGC_CTRL, ctrl); + wr32(IGC_CTRL_EXT, ctrl_ext); +} + static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp, struct ptp_clock_request *rq, int on) { + struct igc_adapter *igc = + container_of(ptp, struct igc_adapter, ptp_caps); + struct igc_hw *hw = &igc->hw; + unsigned long flags; + struct timespec64 ts; + int use_freq = 0, pin = -1; + u32 tsim, tsauxc, tsauxc_mask, tsim_mask, trgttiml, trgttimh, freqout; + s64 ns; + + switch (rq->type) { + case PTP_CLK_REQ_EXTTS: + /* Reject requests with unsupported flags */ + if (rq->extts.flags & ~(PTP_ENABLE_FEATURE | + PTP_RISING_EDGE | + PTP_FALLING_EDGE | + PTP_STRICT_FLAGS)) + return -EOPNOTSUPP; + + /* Reject requests failing to enable both edges. */ + if ((rq->extts.flags & PTP_STRICT_FLAGS) && + (rq->extts.flags & PTP_ENABLE_FEATURE) && + (rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES) + return -EOPNOTSUPP; + + if (on) { + pin = ptp_find_pin(igc->ptp_clock, PTP_PF_EXTTS, + rq->extts.index); + if (pin < 0) + return -EBUSY; + } + if (rq->extts.index == 1) { + tsauxc_mask = IGC_TSAUXC_EN_TS1; + tsim_mask = IGC_TSICR_AUTT1; + } else { + tsauxc_mask = IGC_TSAUXC_EN_TS0; + tsim_mask = IGC_TSICR_AUTT0; + } + spin_lock_irqsave(&igc->tmreg_lock, flags); + tsauxc = rd32(IGC_TSAUXC); + tsim = rd32(IGC_TSIM); + if (on) { + igc_pin_extts(igc, rq->extts.index, pin); + tsauxc |= tsauxc_mask; + tsim |= tsim_mask; + } else { + tsauxc &= ~tsauxc_mask; + tsim &= ~tsim_mask; + } + wr32(IGC_TSAUXC, tsauxc); + wr32(IGC_TSIM, tsim); + spin_unlock_irqrestore(&igc->tmreg_lock, flags); + return 0; + + case PTP_CLK_REQ_PEROUT: + /* Reject requests with unsupported flags */ + if (rq->perout.flags) + return -EOPNOTSUPP; + + if (on) { + pin = ptp_find_pin(igc->ptp_clock, PTP_PF_PEROUT, + rq->perout.index); + if (pin < 0) + return -EBUSY; + } + ts.tv_sec = rq->perout.period.sec; + ts.tv_nsec = rq->perout.period.nsec; + ns = timespec64_to_ns(&ts); + ns = ns >> 1; + if (on && (ns <= 70000000LL || ns == 125000000LL || + ns == 250000000LL || ns == 500000000LL)) { + if (ns < 8LL) + return -EINVAL; + use_freq = 1; + } + ts = ns_to_timespec64(ns); + if (rq->perout.index == 1) { + if (use_freq) { + tsauxc_mask = IGC_TSAUXC_EN_CLK1; + tsim_mask = 0; + } else { + tsauxc_mask = IGC_TSAUXC_EN_TT1; + tsim_mask = IGC_TSICR_TT1; + } + trgttiml = IGC_TRGTTIML1; + trgttimh = IGC_TRGTTIMH1; + freqout = IGC_FREQOUT1; + } else { + if (use_freq) { + tsauxc_mask = IGC_TSAUXC_EN_CLK0; + tsim_mask = 0; + } else { + tsauxc_mask = IGC_TSAUXC_EN_TT0; + tsim_mask = IGC_TSICR_TT0; + } + trgttiml = IGC_TRGTTIML0; + trgttimh = IGC_TRGTTIMH0; + freqout = IGC_FREQOUT0; + } + spin_lock_irqsave(&igc->tmreg_lock, flags); + tsauxc = rd32(IGC_TSAUXC); + tsim = rd32(IGC_TSIM); + if (rq->perout.index == 1) { + tsauxc &= ~(IGC_TSAUXC_EN_TT1 | IGC_TSAUXC_EN_CLK1); + tsim &= ~IGC_TSICR_TT1; + } else { + tsauxc &= ~(IGC_TSAUXC_EN_TT0 | IGC_TSAUXC_EN_CLK0); + tsim &= ~IGC_TSICR_TT0; + } + if (on) { + int i = rq->perout.index; + + igc_pin_perout(igc, i, pin, use_freq); + igc->perout[i].start.tv_sec = rq->perout.start.sec; + igc->perout[i].start.tv_nsec = rq->perout.start.nsec; + igc->perout[i].period.tv_sec = ts.tv_sec; + igc->perout[i].period.tv_nsec = ts.tv_nsec; + wr32(trgttimh, rq->perout.start.sec); + /* For now, always select timer 0 as source. */ + wr32(trgttiml, rq->perout.start.nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); + if (use_freq) + wr32(freqout, ns); + tsauxc |= tsauxc_mask; + tsim |= tsim_mask; + } + wr32(IGC_TSAUXC, tsauxc); + wr32(IGC_TSIM, tsim); + spin_unlock_irqrestore(&igc->tmreg_lock, flags); + return 0; + + case PTP_CLK_REQ_PPS: + spin_lock_irqsave(&igc->tmreg_lock, flags); + tsim = rd32(IGC_TSIM); + if (on) + tsim |= IGC_TSICR_SYS_WRAP; + else + tsim &= ~IGC_TSICR_SYS_WRAP; + igc->pps_sys_wrap_on = on; + wr32(IGC_TSIM, tsim); + spin_unlock_irqrestore(&igc->tmreg_lock, flags); + return 0; + + default: + break; + } + return -EOPNOTSUPP; } +static int igc_ptp_verify_pin(struct ptp_clock_info *ptp, unsigned int pin, + enum ptp_pin_function func, unsigned int chan) +{ + switch (func) { + case PTP_PF_NONE: + case PTP_PF_EXTTS: + case PTP_PF_PEROUT: + break; + case PTP_PF_PHYSYNC: + return -1; + } + return 0; +} + /** * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp * @adapter: board private structure @@ -160,123 +433,120 @@ static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter, } /** - * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp - * @q_vector: Pointer to interrupt specific structure - * @va: Pointer to address containing Rx buffer - * @skb: Buffer containing timestamp and packet + * igc_ptp_rx_pktstamp - Retrieve timestamp from Rx packet buffer + * @adapter: Pointer to adapter the packet buffer belongs to + * @buf: Pointer to packet buffer * - * This function is meant to retrieve the first timestamp from the - * first buffer of an incoming frame. The value is stored in little - * endian format starting on byte 0. There's a second timestamp - * starting on byte 8. - **/ -void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va, - struct sk_buff *skb) -{ - struct igc_adapter *adapter = q_vector->adapter; - __le64 *regval = (__le64 *)va; - int adjust = 0; - - /* The timestamp is recorded in little endian format. - * DWORD: | 0 | 1 | 2 | 3 - * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High - */ - igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), - le64_to_cpu(regval[0])); - - /* adjust timestamp for the RX latency based on link speed */ - if (adapter->hw.mac.type == igc_i225) { - switch (adapter->link_speed) { - case SPEED_10: - adjust = IGC_I225_RX_LATENCY_10; - break; - case SPEED_100: - adjust = IGC_I225_RX_LATENCY_100; - break; - case SPEED_1000: - adjust = IGC_I225_RX_LATENCY_1000; - break; - case SPEED_2500: - adjust = IGC_I225_RX_LATENCY_2500; - break; - } - } - skb_hwtstamps(skb)->hwtstamp = - ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust); -} - -/** - * igc_ptp_rx_rgtstamp - retrieve Rx timestamp stored in register - * @q_vector: Pointer to interrupt specific structure - * @skb: Buffer containing timestamp and packet + * This function retrieves the timestamp saved in the beginning of packet + * buffer. While two timestamps are available, one in timer0 reference and the + * other in timer1 reference, this function considers only the timestamp in + * timer0 reference. * - * This function is meant to retrieve a timestamp from the internal registers - * of the adapter and store it in the skb. + * Returns timestamp value. */ -void igc_ptp_rx_rgtstamp(struct igc_q_vector *q_vector, - struct sk_buff *skb) +ktime_t igc_ptp_rx_pktstamp(struct igc_adapter *adapter, __le32 *buf) { - struct igc_adapter *adapter = q_vector->adapter; - struct igc_hw *hw = &adapter->hw; - u64 regval; + ktime_t timestamp; + u32 secs, nsecs; + int adjust; - /* If this bit is set, then the RX registers contain the time - * stamp. No other packet will be time stamped until we read - * these registers, so read the registers to make them - * available again. Because only one packet can be time - * stamped at a time, we know that the register values must - * belong to this one here and therefore we don't need to - * compare any of the additional attributes stored for it. + /* Timestamps are saved in little endian at the beginning of the packet + * buffer following the layout: + * + * DWORD: | 0 | 1 | 2 | 3 | + * Field: | Timer1 SYSTIML | Timer1 SYSTIMH | Timer0 SYSTIML | Timer0 SYSTIMH | * - * If nothing went wrong, then it should have a shared - * tx_flags that we can turn into a skb_shared_hwtstamps. + * SYSTIML holds the nanoseconds part while SYSTIMH holds the seconds + * part of the timestamp. */ - if (!(rd32(IGC_TSYNCRXCTL) & IGC_TSYNCRXCTL_VALID)) - return; + nsecs = le32_to_cpu(buf[2]); + secs = le32_to_cpu(buf[3]); - regval = rd32(IGC_RXSTMPL); - regval |= (u64)rd32(IGC_RXSTMPH) << 32; + timestamp = ktime_set(secs, nsecs); - igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval); + /* Adjust timestamp for the RX latency based on link speed */ + switch (adapter->link_speed) { + case SPEED_10: + adjust = IGC_I225_RX_LATENCY_10; + break; + case SPEED_100: + adjust = IGC_I225_RX_LATENCY_100; + break; + case SPEED_1000: + adjust = IGC_I225_RX_LATENCY_1000; + break; + case SPEED_2500: + adjust = IGC_I225_RX_LATENCY_2500; + break; + default: + adjust = 0; + netdev_warn_once(adapter->netdev, "Imprecise timestamp\n"); + break; + } - /* Update the last_rx_timestamp timer in order to enable watchdog check - * for error case of latched timestamp on a dropped packet. - */ - adapter->last_rx_timestamp = jiffies; + return ktime_sub_ns(timestamp, adjust); } -/** - * igc_ptp_enable_tstamp_rxqueue - Enable RX timestamp for a queue - * @rx_ring: Pointer to RX queue - * @timer: Index for timer - * - * This function enables RX timestamping for a queue, and selects - * which 1588 timer will provide the timestamp. - */ -static void igc_ptp_enable_tstamp_rxqueue(struct igc_adapter *adapter, - struct igc_ring *rx_ring, u8 timer) +static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; - int reg_idx = rx_ring->reg_idx; - u32 srrctl = rd32(IGC_SRRCTL(reg_idx)); + u32 val; + int i; - srrctl |= IGC_SRRCTL_TIMESTAMP; - srrctl |= IGC_SRRCTL_TIMER1SEL(timer); - srrctl |= IGC_SRRCTL_TIMER0SEL(timer); + wr32(IGC_TSYNCRXCTL, 0); - wr32(IGC_SRRCTL(reg_idx), srrctl); + for (i = 0; i < adapter->num_rx_queues; i++) { + val = rd32(IGC_SRRCTL(i)); + val &= ~IGC_SRRCTL_TIMESTAMP; + wr32(IGC_SRRCTL(i), val); + } + + val = rd32(IGC_RXPBS); + val &= ~IGC_RXPBS_CFG_TS_EN; + wr32(IGC_RXPBS, val); } -static void igc_ptp_enable_tstamp_all_rxqueues(struct igc_adapter *adapter, - u8 timer) +static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter) { + struct igc_hw *hw = &adapter->hw; + u32 val; int i; - for (i = 0; i < adapter->num_rx_queues; i++) { - struct igc_ring *ring = adapter->rx_ring[i]; + val = rd32(IGC_RXPBS); + val |= IGC_RXPBS_CFG_TS_EN; + wr32(IGC_RXPBS, val); - igc_ptp_enable_tstamp_rxqueue(adapter, ring, timer); + for (i = 0; i < adapter->num_rx_queues; i++) { + val = rd32(IGC_SRRCTL(i)); + /* FIXME: For now, only support retrieving RX timestamps from + * timer 0. + */ + val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) | + IGC_SRRCTL_TIMESTAMP; + wr32(IGC_SRRCTL(i), val); } + + val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL | + IGC_TSYNCRXCTL_RXSYNSIG; + wr32(IGC_TSYNCRXCTL, val); +} + +static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + + wr32(IGC_TSYNCTXCTL, 0); +} + +static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + + wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG); + + /* Read TXSTMP registers to discard any timestamp previously stored. */ + rd32(IGC_TXSTMPL); + rd32(IGC_TXSTMPH); } /** @@ -284,38 +554,17 @@ static void igc_ptp_enable_tstamp_all_rxqueues(struct igc_adapter *adapter, * @adapter: networking device structure * @config: hwtstamp configuration * - * Outgoing time stamping can be enabled and disabled. Play nice and - * disable it when requested, although it shouldn't case any overhead - * when no packet needs it. At most one packet in the queue may be - * marked for time stamping, otherwise it would be impossible to tell - * for sure to which packet the hardware time stamp belongs. - * - * Incoming time stamping has to be configured via the hardware - * filters. Not all combinations are supported, in particular event - * type has to be specified. Matching the kind of event packet is - * not supported, with the exception of "all V2 events regardless of - * level 2 or 4". - * + * Return: 0 in case of success, negative errno code otherwise. */ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, struct hwtstamp_config *config) { - u32 tsync_tx_ctl = IGC_TSYNCTXCTL_ENABLED; - u32 tsync_rx_ctl = IGC_TSYNCRXCTL_ENABLED; - struct igc_hw *hw = &adapter->hw; - u32 tsync_rx_cfg = 0; - bool is_l4 = false; - bool is_l2 = false; - u32 regval; - - /* reserved for future extensions */ - if (config->flags) - return -EINVAL; - switch (config->tx_type) { case HWTSTAMP_TX_OFF: - tsync_tx_ctl = 0; + igc_ptp_disable_tx_timestamp(adapter); + break; case HWTSTAMP_TX_ON: + igc_ptp_enable_tx_timestamp(adapter); break; default: return -ERANGE; @@ -323,18 +572,10 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, switch (config->rx_filter) { case HWTSTAMP_FILTER_NONE: - tsync_rx_ctl = 0; + igc_ptp_disable_rx_timestamp(adapter); break; case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: - tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_L4_V1; - tsync_rx_cfg = IGC_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE; - is_l4 = true; - break; case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: - tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_L4_V1; - tsync_rx_cfg = IGC_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE; - is_l4 = true; - break; case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: @@ -344,110 +585,36 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: - tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_EVENT_V2; - config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - is_l2 = true; - is_l4 = true; - break; case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: case HWTSTAMP_FILTER_NTP_ALL: case HWTSTAMP_FILTER_ALL: - tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_ALL; + igc_ptp_enable_rx_timestamp(adapter); config->rx_filter = HWTSTAMP_FILTER_ALL; break; - /* fall through */ default: - config->rx_filter = HWTSTAMP_FILTER_NONE; return -ERANGE; } - /* Per-packet timestamping only works if all packets are - * timestamped, so enable timestamping in all packets as long - * as one Rx filter was configured. - */ - if (tsync_rx_ctl) { - tsync_rx_ctl = IGC_TSYNCRXCTL_ENABLED; - tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_ALL; - tsync_rx_ctl |= IGC_TSYNCRXCTL_RXSYNSIG; - config->rx_filter = HWTSTAMP_FILTER_ALL; - is_l2 = true; - is_l4 = true; - - if (hw->mac.type == igc_i225) { - regval = rd32(IGC_RXPBS); - regval |= IGC_RXPBS_CFG_TS_EN; - wr32(IGC_RXPBS, regval); - - /* FIXME: For now, only support retrieving RX - * timestamps from timer 0 - */ - igc_ptp_enable_tstamp_all_rxqueues(adapter, 0); - } - } - - if (tsync_tx_ctl) { - tsync_tx_ctl = IGC_TSYNCTXCTL_ENABLED; - tsync_tx_ctl |= IGC_TSYNCTXCTL_TXSYNSIG; - } - - /* enable/disable TX */ - regval = rd32(IGC_TSYNCTXCTL); - regval &= ~IGC_TSYNCTXCTL_ENABLED; - regval |= tsync_tx_ctl; - wr32(IGC_TSYNCTXCTL, regval); - - /* enable/disable RX */ - regval = rd32(IGC_TSYNCRXCTL); - regval &= ~(IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_MASK); - regval |= tsync_rx_ctl; - wr32(IGC_TSYNCRXCTL, regval); - - /* define which PTP packets are time stamped */ - wr32(IGC_TSYNCRXCFG, tsync_rx_cfg); - - /* define ethertype filter for timestamped packets */ - if (is_l2) - wr32(IGC_ETQF(3), - (IGC_ETQF_FILTER_ENABLE | /* enable filter */ - IGC_ETQF_1588 | /* enable timestamping */ - ETH_P_1588)); /* 1588 eth protocol type */ - else - wr32(IGC_ETQF(3), 0); - - /* L4 Queue Filter[3]: filter by destination port and protocol */ - if (is_l4) { - u32 ftqf = (IPPROTO_UDP /* UDP */ - | IGC_FTQF_VF_BP /* VF not compared */ - | IGC_FTQF_1588_TIME_STAMP /* Enable Timestamp */ - | IGC_FTQF_MASK); /* mask all inputs */ - ftqf &= ~IGC_FTQF_MASK_PROTO_BP; /* enable protocol check */ - - wr32(IGC_IMIR(3), htons(PTP_EV_PORT)); - wr32(IGC_IMIREXT(3), - (IGC_IMIREXT_SIZE_BP | IGC_IMIREXT_CTRL_BP)); - wr32(IGC_FTQF(3), ftqf); - } else { - wr32(IGC_FTQF(3), IGC_FTQF_MASK); - } - wrfl(); + return 0; +} - /* clear TX/RX time stamp registers, just to be sure */ - regval = rd32(IGC_TXSTMPL); - regval = rd32(IGC_TXSTMPH); - regval = rd32(IGC_RXSTMPL); - regval = rd32(IGC_RXSTMPH); +static void igc_ptp_tx_timeout(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; - return 0; + dev_kfree_skb_any(adapter->ptp_tx_skb); + adapter->ptp_tx_skb = NULL; + adapter->tx_hwtstamp_timeouts++; + clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + /* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */ + rd32(IGC_TXSTMPH); + netdev_warn(adapter->netdev, "Tx timestamp timeout\n"); } void igc_ptp_tx_hang(struct igc_adapter *adapter) { bool timeout = time_is_before_jiffies(adapter->ptp_tx_start + IGC_PTP_TX_TIMEOUT); - struct igc_hw *hw = &adapter->hw; - - if (!adapter->ptp_tx_skb) - return; if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state)) return; @@ -458,15 +625,7 @@ void igc_ptp_tx_hang(struct igc_adapter *adapter) */ if (timeout) { cancel_work_sync(&adapter->ptp_tx_work); - dev_kfree_skb_any(adapter->ptp_tx_skb); - adapter->ptp_tx_skb = NULL; - clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); - adapter->tx_hwtstamp_timeouts++; - /* Clear the Tx valid bit in TSYNCTXCTL register to enable - * interrupt - */ - rd32(IGC_TXSTMPH); - dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n"); + igc_ptp_tx_timeout(adapter); } } @@ -483,12 +642,34 @@ static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter) struct sk_buff *skb = adapter->ptp_tx_skb; struct skb_shared_hwtstamps shhwtstamps; struct igc_hw *hw = &adapter->hw; + int adjust = 0; u64 regval; + if (WARN_ON_ONCE(!skb)) + return; + regval = rd32(IGC_TXSTMPL); regval |= (u64)rd32(IGC_TXSTMPH) << 32; igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval); + switch (adapter->link_speed) { + case SPEED_10: + adjust = IGC_I225_TX_LATENCY_10; + break; + case SPEED_100: + adjust = IGC_I225_TX_LATENCY_100; + break; + case SPEED_1000: + adjust = IGC_I225_TX_LATENCY_1000; + break; + case SPEED_2500: + adjust = IGC_I225_TX_LATENCY_2500; + break; + } + + shhwtstamps.hwtstamp = + ktime_add_ns(shhwtstamps.hwtstamp, adjust); + /* Clear the lock early before calling skb_tstamp_tx so that * applications are not woken up before the lock bit is clear. We use * a copy of the skb pointer to ensure other threads can't change it @@ -509,42 +690,27 @@ static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter) * This work function polls the TSYNCTXCTL valid bit to determine when a * timestamp has been taken for the current stored skb. */ -void igc_ptp_tx_work(struct work_struct *work) +static void igc_ptp_tx_work(struct work_struct *work) { struct igc_adapter *adapter = container_of(work, struct igc_adapter, ptp_tx_work); struct igc_hw *hw = &adapter->hw; u32 tsynctxctl; - if (!adapter->ptp_tx_skb) + if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state)) return; - if (time_is_before_jiffies(adapter->ptp_tx_start + - IGC_PTP_TX_TIMEOUT)) { - dev_kfree_skb_any(adapter->ptp_tx_skb); - adapter->ptp_tx_skb = NULL; - clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); - adapter->tx_hwtstamp_timeouts++; - /* Clear the tx valid bit in TSYNCTXCTL register to enable - * interrupt - */ - rd32(IGC_TXSTMPH); - dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n"); + tsynctxctl = rd32(IGC_TSYNCTXCTL); + if (WARN_ON_ONCE(!(tsynctxctl & IGC_TSYNCTXCTL_TXTT_0))) return; - } - tsynctxctl = rd32(IGC_TSYNCTXCTL); - if (tsynctxctl & IGC_TSYNCTXCTL_VALID) - igc_ptp_tx_hwtstamp(adapter); - else - /* reschedule to check later */ - schedule_work(&adapter->ptp_tx_work); + igc_ptp_tx_hwtstamp(adapter); } /** * igc_ptp_set_ts_config - set hardware time stamping config * @netdev: network interface device structure - * @ifreq: interface request data + * @ifr: interface request data * **/ int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr) @@ -571,7 +737,7 @@ int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr) /** * igc_ptp_get_ts_config - get hardware time stamping config * @netdev: network interface device structure - * @ifreq: interface request data + * @ifr: interface request data * * Get the hwtstamp_config settings to return to the user. Rather than attempt * to deconstruct the settings from the registers, just return a shadow copy @@ -586,6 +752,160 @@ int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr) -EFAULT : 0; } +/* The two conditions below must be met for cross timestamping via + * PCIe PTM: + * + * 1. We have an way to convert the timestamps in the PTM messages + * to something related to the system clocks (right now, only + * X86 systems with support for the Always Running Timer allow that); + * + * 2. We have PTM enabled in the path from the device to the PCIe root port. + */ +static bool igc_is_crosststamp_supported(struct igc_adapter *adapter) +{ + if (!IS_ENABLED(CONFIG_X86_TSC)) + return false; + + /* FIXME: it was noticed that enabling support for PCIe PTM in + * some i225-V models could cause lockups when bringing the + * interface up/down. There should be no downsides to + * disabling crosstimestamping support for i225-V, as it + * doesn't have any PTP support. That way we gain some time + * while root causing the issue. + */ + if (adapter->pdev->device == IGC_DEV_ID_I225_V) + return false; + + return pcie_ptm_enabled(adapter->pdev); +} + +static struct system_counterval_t igc_device_tstamp_to_system(u64 tstamp) +{ +#if IS_ENABLED(CONFIG_X86_TSC) && !defined(CONFIG_UML) + return convert_art_ns_to_tsc(tstamp); +#else + return (struct system_counterval_t) { }; +#endif +} + +static void igc_ptm_log_error(struct igc_adapter *adapter, u32 ptm_stat) +{ + struct net_device *netdev = adapter->netdev; + + switch (ptm_stat) { + case IGC_PTM_STAT_RET_ERR: + netdev_err(netdev, "PTM Error: Root port timeout\n"); + break; + case IGC_PTM_STAT_BAD_PTM_RES: + netdev_err(netdev, "PTM Error: Bad response, PTM Response Data expected\n"); + break; + case IGC_PTM_STAT_T4M1_OVFL: + netdev_err(netdev, "PTM Error: T4 minus T1 overflow\n"); + break; + case IGC_PTM_STAT_ADJUST_1ST: + netdev_err(netdev, "PTM Error: 1588 timer adjusted during first PTM cycle\n"); + break; + case IGC_PTM_STAT_ADJUST_CYC: + netdev_err(netdev, "PTM Error: 1588 timer adjusted during non-first PTM cycle\n"); + break; + default: + netdev_err(netdev, "PTM Error: Unknown error (%#x)\n", ptm_stat); + break; + } +} + +static int igc_phc_get_syncdevicetime(ktime_t *device, + struct system_counterval_t *system, + void *ctx) +{ + u32 stat, t2_curr_h, t2_curr_l, ctrl; + struct igc_adapter *adapter = ctx; + struct igc_hw *hw = &adapter->hw; + int err, count = 100; + ktime_t t1, t2_curr; + + /* Get a snapshot of system clocks to use as historic value. */ + ktime_get_snapshot(&adapter->snapshot); + + do { + /* Doing this in a loop because in the event of a + * badly timed (ha!) system clock adjustment, we may + * get PTM errors from the PCI root, but these errors + * are transitory. Repeating the process returns valid + * data eventually. + */ + + /* To "manually" start the PTM cycle we need to clear and + * then set again the TRIG bit. + */ + ctrl = rd32(IGC_PTM_CTRL); + ctrl &= ~IGC_PTM_CTRL_TRIG; + wr32(IGC_PTM_CTRL, ctrl); + ctrl |= IGC_PTM_CTRL_TRIG; + wr32(IGC_PTM_CTRL, ctrl); + + /* The cycle only starts "for real" when software notifies + * that it has read the registers, this is done by setting + * VALID bit. + */ + wr32(IGC_PTM_STAT, IGC_PTM_STAT_VALID); + + err = readx_poll_timeout(rd32, IGC_PTM_STAT, stat, + stat, IGC_PTM_STAT_SLEEP, + IGC_PTM_STAT_TIMEOUT); + if (err < 0) { + netdev_err(adapter->netdev, "Timeout reading IGC_PTM_STAT register\n"); + return err; + } + + if ((stat & IGC_PTM_STAT_VALID) == IGC_PTM_STAT_VALID) + break; + + if (stat & ~IGC_PTM_STAT_VALID) { + /* An error occurred, log it. */ + igc_ptm_log_error(adapter, stat); + /* The STAT register is write-1-to-clear (W1C), + * so write the previous error status to clear it. + */ + wr32(IGC_PTM_STAT, stat); + continue; + } + } while (--count); + + if (!count) { + netdev_err(adapter->netdev, "Exceeded number of tries for PTM cycle\n"); + return -ETIMEDOUT; + } + + t1 = ktime_set(rd32(IGC_PTM_T1_TIM0_H), rd32(IGC_PTM_T1_TIM0_L)); + + t2_curr_l = rd32(IGC_PTM_CURR_T2_L); + t2_curr_h = rd32(IGC_PTM_CURR_T2_H); + + /* FIXME: When the register that tells the endianness of the + * PTM registers are implemented, check them here and add the + * appropriate conversion. + */ + t2_curr_h = swab32(t2_curr_h); + + t2_curr = ((s64)t2_curr_h << 32 | t2_curr_l); + + *device = t1; + *system = igc_device_tstamp_to_system(t2_curr); + + return 0; +} + +static int igc_ptp_getcrosststamp(struct ptp_clock_info *ptp, + struct system_device_crosststamp *cts) +{ + struct igc_adapter *adapter = container_of(ptp, struct igc_adapter, + ptp_caps); + + return get_device_system_crosststamp(igc_phc_get_syncdevicetime, + adapter, &adapter->snapshot, cts); +} + /** * igc_ptp_init - Initialize PTP functionality * @adapter: Board private structure @@ -597,9 +917,17 @@ void igc_ptp_init(struct igc_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct igc_hw *hw = &adapter->hw; + int i; switch (hw->mac.type) { case igc_i225: + for (i = 0; i < IGC_N_SDP; i++) { + struct ptp_pin_desc *ppd = &adapter->sdp_config[i]; + + snprintf(ppd->name, sizeof(ppd->name), "SDP%d", i); + ppd->index = i; + ppd->func = PTP_PF_NONE; + } snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr); adapter->ptp_caps.owner = THIS_MODULE; adapter->ptp_caps.max_adj = 62499999; @@ -608,6 +936,17 @@ void igc_ptp_init(struct igc_adapter *adapter) adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225; adapter->ptp_caps.settime64 = igc_ptp_settime_i225; adapter->ptp_caps.enable = igc_ptp_feature_enable_i225; + adapter->ptp_caps.pps = 1; + adapter->ptp_caps.pin_config = adapter->sdp_config; + adapter->ptp_caps.n_ext_ts = IGC_N_EXTTS; + adapter->ptp_caps.n_per_out = IGC_N_PEROUT; + adapter->ptp_caps.n_pins = IGC_N_SDP; + adapter->ptp_caps.verify = igc_ptp_verify_pin; + + if (!igc_is_crosststamp_supported(adapter)) + break; + + adapter->ptp_caps.getcrosststamp = igc_ptp_getcrosststamp; break; default: adapter->ptp_clock = NULL; @@ -620,20 +959,49 @@ void igc_ptp_init(struct igc_adapter *adapter) adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF; - igc_ptp_reset(adapter); + adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real()); + adapter->ptp_reset_start = ktime_get(); adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps, &adapter->pdev->dev); if (IS_ERR(adapter->ptp_clock)) { adapter->ptp_clock = NULL; - dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n"); + netdev_err(netdev, "ptp_clock_register failed\n"); } else if (adapter->ptp_clock) { - dev_info(&adapter->pdev->dev, "added PHC on %s\n", - adapter->netdev->name); + netdev_info(netdev, "PHC added\n"); adapter->ptp_flags |= IGC_PTP_ENABLED; } } +static void igc_ptp_time_save(struct igc_adapter *adapter) +{ + igc_ptp_read(adapter, &adapter->prev_ptp_time); + adapter->ptp_reset_start = ktime_get(); +} + +static void igc_ptp_time_restore(struct igc_adapter *adapter) +{ + struct timespec64 ts = adapter->prev_ptp_time; + ktime_t delta; + + delta = ktime_sub(ktime_get(), adapter->ptp_reset_start); + + timespec64_add_ns(&ts, ktime_to_ns(delta)); + + igc_ptp_write_i225(adapter, &ts); +} + +static void igc_ptm_stop(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 ctrl; + + ctrl = rd32(IGC_PTM_CTRL); + ctrl &= ~IGC_PTM_CTRL_EN; + + wr32(IGC_PTM_CTRL, ctrl); +} + /** * igc_ptp_suspend - Disable PTP work items and prepare for suspend * @adapter: Board private structure @@ -647,10 +1015,13 @@ void igc_ptp_suspend(struct igc_adapter *adapter) return; cancel_work_sync(&adapter->ptp_tx_work); - if (adapter->ptp_tx_skb) { - dev_kfree_skb_any(adapter->ptp_tx_skb); - adapter->ptp_tx_skb = NULL; - clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + dev_kfree_skb_any(adapter->ptp_tx_skb); + adapter->ptp_tx_skb = NULL; + clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + + if (pci_device_is_present(adapter->pdev)) { + igc_ptp_time_save(adapter); + igc_ptm_stop(adapter); } } @@ -666,8 +1037,7 @@ void igc_ptp_stop(struct igc_adapter *adapter) if (adapter->ptp_clock) { ptp_clock_unregister(adapter->ptp_clock); - dev_info(&adapter->pdev->dev, "removed PHC on %s\n", - adapter->netdev->name); + netdev_info(adapter->netdev, "PHC removed\n"); adapter->ptp_flags &= ~IGC_PTP_ENABLED; } } @@ -681,7 +1051,9 @@ void igc_ptp_stop(struct igc_adapter *adapter) void igc_ptp_reset(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; + u32 cycle_ctrl, ctrl; unsigned long flags; + u32 timadj; /* reset the tstamp_config */ igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config); @@ -690,10 +1062,38 @@ void igc_ptp_reset(struct igc_adapter *adapter) switch (adapter->hw.mac.type) { case igc_i225: + timadj = rd32(IGC_TIMADJ); + timadj |= IGC_TIMADJ_ADJUST_METH; + wr32(IGC_TIMADJ, timadj); + wr32(IGC_TSAUXC, 0x0); wr32(IGC_TSSDP, 0x0); - wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS); + wr32(IGC_TSIM, + IGC_TSICR_INTERRUPTS | + (adapter->pps_sys_wrap_on ? IGC_TSICR_SYS_WRAP : 0)); wr32(IGC_IMS, IGC_IMS_TS); + + if (!igc_is_crosststamp_supported(adapter)) + break; + + wr32(IGC_PCIE_DIG_DELAY, IGC_PCIE_DIG_DELAY_DEFAULT); + wr32(IGC_PCIE_PHY_DELAY, IGC_PCIE_PHY_DELAY_DEFAULT); + + cycle_ctrl = IGC_PTM_CYCLE_CTRL_CYC_TIME(IGC_PTM_CYC_TIME_DEFAULT); + + wr32(IGC_PTM_CYCLE_CTRL, cycle_ctrl); + + ctrl = IGC_PTM_CTRL_EN | + IGC_PTM_CTRL_START_NOW | + IGC_PTM_CTRL_SHRT_CYC(IGC_PTM_SHORT_CYC_DEFAULT) | + IGC_PTM_CTRL_PTM_TO(IGC_PTM_TIMEOUT_DEFAULT) | + IGC_PTM_CTRL_TRIG; + + wr32(IGC_PTM_CTRL, ctrl); + + /* Force the first cycle to run. */ + wr32(IGC_PTM_STAT, IGC_PTM_STAT_VALID); + break; default: /* No work to do. */ @@ -702,9 +1102,7 @@ void igc_ptp_reset(struct igc_adapter *adapter) /* Re-initialize the timer. */ if (hw->mac.type == igc_i225) { - struct timespec64 ts64 = ktime_to_timespec64(ktime_get_real()); - - igc_ptp_write_i225(adapter, &ts64); + igc_ptp_time_restore(adapter); } else { timecounter_init(&adapter->tc, &adapter->cc, ktime_to_ns(ktime_get_real())); diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index c9029b549b90..c0d8214148d1 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -10,9 +10,10 @@ #define IGC_EECD 0x00010 /* EEPROM/Flash Control - RW */ #define IGC_CTRL_EXT 0x00018 /* Extended Device Control - RW */ #define IGC_MDIC 0x00020 /* MDI Control - RW */ -#define IGC_MDICNFG 0x00E04 /* MDC/MDIO Configuration - RW */ #define IGC_CONNSW 0x00034 /* Copper/Fiber switch control - RW */ +#define IGC_VET 0x00038 /* VLAN Ether Type - RW */ #define IGC_I225_PHPM 0x00E14 /* I225 PHY Power Management */ +#define IGC_GPHY_VERSION 0x0001E /* I225 gPHY Firmware Version */ /* Internal Packet Buffer Size Registers */ #define IGC_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ @@ -30,10 +31,6 @@ #define IGC_FCRTL 0x02160 /* FC Receive Threshold Low - RW */ #define IGC_FCRTH 0x02168 /* FC Receive Threshold High - RW */ #define IGC_FCRTV 0x02460 /* FC Refresh Timer Value - RW */ -#define IGC_FCSTS 0x02464 /* FC Status - RO */ - -/* PCIe Register Description */ -#define IGC_GCR 0x05B00 /* PCIe control- RW */ /* Semaphore registers */ #define IGC_SW_FW_SYNC 0x05B5C /* SW-FW Synchronization - RW */ @@ -44,6 +41,7 @@ #define IGC_FACTPS 0x05B30 /* Interrupt Register Description */ +#define IGC_EICR 0x01580 /* Ext. Interrupt Cause read - W0 */ #define IGC_EICS 0x01520 /* Ext. Interrupt Cause Set - W0 */ #define IGC_EIMS 0x01524 /* Ext. Interrupt Mask Set/Read - RW */ #define IGC_EIMC 0x01528 /* Ext. Interrupt Mask Clear - WO */ @@ -61,31 +59,14 @@ #define IGC_IVAR_MISC 0x01740 /* IVAR for "other" causes - RW */ #define IGC_GPIE 0x01514 /* General Purpose Intr Enable - RW */ -/* Interrupt Cause */ -#define IGC_ICRXPTC 0x04104 /* Rx Packet Timer Expire Count */ -#define IGC_ICRXATC 0x04108 /* Rx Absolute Timer Expire Count */ -#define IGC_ICTXPTC 0x0410C /* Tx Packet Timer Expire Count */ -#define IGC_ICTXATC 0x04110 /* Tx Absolute Timer Expire Count */ -#define IGC_ICTXQEC 0x04118 /* Tx Queue Empty Count */ -#define IGC_ICTXQMTC 0x0411C /* Tx Queue Min Threshold Count */ -#define IGC_ICRXDMTC 0x04120 /* Rx Descriptor Min Threshold Count */ -#define IGC_ICRXOC 0x04124 /* Receiver Overrun Count */ - -#define IGC_CBTMPC 0x0402C /* Circuit Breaker TX Packet Count */ -#define IGC_HTDPMC 0x0403C /* Host Transmit Discarded Packets */ -#define IGC_CBRMPC 0x040FC /* Circuit Breaker RX Packet Count */ -#define IGC_RPTHC 0x04104 /* Rx Packets To Host */ -#define IGC_HGPTC 0x04118 /* Host Good Packets TX Count */ -#define IGC_HTCBDPC 0x04124 /* Host TX Circ.Breaker Drop Count */ - -/* MSI-X Table Register Descriptions */ -#define IGC_PBACL 0x05B68 /* MSIx PBA Clear - R/W 1 to clear */ - /* RSS registers */ #define IGC_MRQC 0x05818 /* Multiple Receive Control - RW */ /* Filtering Registers */ #define IGC_ETQF(_n) (0x05CB0 + (4 * (_n))) /* EType Queue Fltr */ +#define IGC_FHFT(_n) (0x09000 + (256 * (_n))) /* Flexible Host Filter */ +#define IGC_FHFT_EXT(_n) (0x09A00 + (256 * (_n))) /* Flexible Host Filter Extended */ +#define IGC_FHFTSL 0x05804 /* Flex Filter indirect table select */ /* ETQF register bit definitions */ #define IGC_ETQF_FILTER_ENABLE BIT(26) @@ -94,6 +75,19 @@ #define IGC_ETQF_QUEUE_MASK 0x00070000 #define IGC_ETQF_ETYPE_MASK 0x0000FFFF +/* FHFT register bit definitions */ +#define IGC_FHFT_LENGTH_MASK GENMASK(7, 0) +#define IGC_FHFT_QUEUE_SHIFT 8 +#define IGC_FHFT_QUEUE_MASK GENMASK(10, 8) +#define IGC_FHFT_PRIO_SHIFT 16 +#define IGC_FHFT_PRIO_MASK GENMASK(18, 16) +#define IGC_FHFT_IMM_INT BIT(24) +#define IGC_FHFT_DROP BIT(25) + +/* FHFTSL register bit definitions */ +#define IGC_FHFTSL_FTSL_SHIFT 0 +#define IGC_FHFTSL_FTSL_MASK GENMASK(1, 0) + /* Redirection Table - RW Array */ #define IGC_RETA(_i) (0x05C00 + ((_i) * 4)) /* RSS Random Key - RW Array */ @@ -114,10 +108,11 @@ #define IGC_RLPML 0x05004 /* Rx Long Packet Max Length */ #define IGC_RFCTL 0x05008 /* Receive Filter Control*/ #define IGC_MTA 0x05200 /* Multicast Table Array - RW Array */ +#define IGC_RA 0x05400 /* Receive Address - RW Array */ #define IGC_UTA 0x0A000 /* Unicast Table Array - RW */ #define IGC_RAL(_n) (0x05400 + ((_n) * 0x08)) #define IGC_RAH(_n) (0x05404 + ((_n) * 0x08)) -#define IGC_VLAPQF 0x055B0 /* VLAN Priority Queue Filter VLAPQF */ +#define IGC_VLANPQF 0x055B0 /* VLAN Priority Queue Filter - RW */ /* Transmit Register Descriptions */ #define IGC_TCTL 0x00400 /* Tx Control - RW */ @@ -133,13 +128,9 @@ #define IGC_MMDAC 13 /* MMD Access Control */ #define IGC_MMDAAD 14 /* MMD Access Address/Data */ -/* Good transmitted packets counter registers */ -#define IGC_PQGPTC(_n) (0x010014 + (0x100 * (_n))) - /* Statistics Register Descriptions */ #define IGC_CRCERRS 0x04000 /* CRC Error Count - R/clr */ #define IGC_ALGNERRC 0x04004 /* Alignment Error Count - R/clr */ -#define IGC_SYMERRS 0x04008 /* Symbol Error Count - R/clr */ #define IGC_RXERRC 0x0400C /* Receive Error Count - R/clr */ #define IGC_MPC 0x04010 /* Missed Packet Count - R/clr */ #define IGC_SCC 0x04014 /* Single Collision Count - R/clr */ @@ -147,10 +138,10 @@ #define IGC_MCC 0x0401C /* Multiple Collision Count - R/clr */ #define IGC_LATECOL 0x04020 /* Late Collision Count - R/clr */ #define IGC_COLC 0x04028 /* Collision Count - R/clr */ +#define IGC_RERC 0x0402C /* Receive Error Count - R/clr */ #define IGC_DC 0x04030 /* Defer Count - R/clr */ #define IGC_TNCRS 0x04034 /* Tx-No CRS - R/clr */ -#define IGC_SEC 0x04038 /* Sequence Error Count - R/clr */ -#define IGC_CEXTERR 0x0403C /* Carrier Extension Error Count - R/clr */ +#define IGC_HTDPMC 0x0403C /* Host Transmit Discarded by MAC - R/clr */ #define IGC_RLEC 0x04040 /* Receive Length Error Count - R/clr */ #define IGC_XONRXC 0x04048 /* XON Rx Count - R/clr */ #define IGC_XONTXC 0x0404C /* XON Tx Count - R/clr */ @@ -194,13 +185,10 @@ #define IGC_MPTC 0x040F0 /* Multicast Packets Tx Count - R/clr */ #define IGC_BPTC 0x040F4 /* Broadcast Packets Tx Count - R/clr */ #define IGC_TSCTC 0x040F8 /* TCP Segmentation Context Tx - R/clr */ -#define IGC_TSCTFC 0x040FC /* TCP Segmentation Context Tx Fail - R/clr */ #define IGC_IAC 0x04100 /* Interrupt Assertion Count */ -#define IGC_ICTXPTC 0x0410C /* Interrupt Cause Tx Pkt Timer Expire Count */ -#define IGC_ICTXATC 0x04110 /* Interrupt Cause Tx Abs Timer Expire Count */ -#define IGC_ICTXQEC 0x04118 /* Interrupt Cause Tx Queue Empty Count */ -#define IGC_ICTXQMTC 0x0411C /* Interrupt Cause Tx Queue Min Thresh Count */ #define IGC_RPTHC 0x04104 /* Rx Packets To Host */ +#define IGC_TLPIC 0x04148 /* EEE Tx LPI Count */ +#define IGC_RLPIC 0x0414C /* EEE Rx LPI Count */ #define IGC_HGPTC 0x04118 /* Host Good Packets Tx Count */ #define IGC_RXDMTC 0x04120 /* Rx Descriptor Minimum Threshold Count */ #define IGC_HGORCL 0x04128 /* Host Good Octets Received Count Low */ @@ -208,7 +196,6 @@ #define IGC_HGOTCL 0x04130 /* Host Good Octets Transmit Count Low */ #define IGC_HGOTCH 0x04134 /* Host Good Octets Transmit Count High */ #define IGC_LENERRS 0x04138 /* Length Errors Count */ -#define IGC_HRMPC 0x0A018 /* Header Redirection Missed Packet Count */ /* Time sync registers */ #define IGC_TSICR 0x0B66C /* Time Sync Interrupt Cause */ @@ -218,13 +205,36 @@ #define IGC_TSYNCTXCTL 0x0B614 /* Tx Time Sync Control register - RW */ #define IGC_TSYNCRXCFG 0x05F50 /* Time Sync Rx Configuration - RW */ #define IGC_TSSDP 0x0003C /* Time Sync SDP Configuration Register - RW */ +#define IGC_TRGTTIML0 0x0B644 /* Target Time Register 0 Low - RW */ +#define IGC_TRGTTIMH0 0x0B648 /* Target Time Register 0 High - RW */ +#define IGC_TRGTTIML1 0x0B64C /* Target Time Register 1 Low - RW */ +#define IGC_TRGTTIMH1 0x0B650 /* Target Time Register 1 High - RW */ +#define IGC_FREQOUT0 0x0B654 /* Frequency Out 0 Control Register - RW */ +#define IGC_FREQOUT1 0x0B658 /* Frequency Out 1 Control Register - RW */ +#define IGC_AUXSTMPL0 0x0B65C /* Auxiliary Time Stamp 0 Register Low - RO */ +#define IGC_AUXSTMPH0 0x0B660 /* Auxiliary Time Stamp 0 Register High - RO */ +#define IGC_AUXSTMPL1 0x0B664 /* Auxiliary Time Stamp 1 Register Low - RO */ +#define IGC_AUXSTMPH1 0x0B668 /* Auxiliary Time Stamp 1 Register High - RO */ #define IGC_IMIR(_i) (0x05A80 + ((_i) * 4)) /* Immediate Interrupt */ #define IGC_IMIREXT(_i) (0x05AA0 + ((_i) * 4)) /* Immediate INTR Ext*/ #define IGC_FTQF(_n) (0x059E0 + (4 * (_n))) /* 5-tuple Queue Fltr */ -#define IGC_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ +/* Transmit Scheduling Registers */ +#define IGC_TQAVCTRL 0x3570 +#define IGC_TXQCTL(_n) (0x3344 + 0x4 * (_n)) +#define IGC_BASET_L 0x3314 +#define IGC_BASET_H 0x3318 +#define IGC_QBVCYCLET 0x331C +#define IGC_QBVCYCLET_S 0x3320 + +#define IGC_STQT(_n) (0x3324 + 0x4 * (_n)) +#define IGC_ENDQT(_n) (0x3334 + 0x4 * (_n)) +#define IGC_DTXMXPKTSZ 0x355C + +#define IGC_TQAVCC(_n) (0x3004 + ((_n) * 0x40)) +#define IGC_TQAVHC(_n) (0x300C + ((_n) * 0x40)) /* System Time Registers */ #define IGC_SYSTIML 0x0B600 /* System time register Low - RO */ @@ -232,11 +242,32 @@ #define IGC_SYSTIMR 0x0B6F8 /* System time register Residue */ #define IGC_TIMINCA 0x0B608 /* Increment attributes register - RW */ -#define IGC_RXSTMPL 0x0B624 /* Rx timestamp Low - RO */ -#define IGC_RXSTMPH 0x0B628 /* Rx timestamp High - RO */ #define IGC_TXSTMPL 0x0B618 /* Tx timestamp value Low - RO */ #define IGC_TXSTMPH 0x0B61C /* Tx timestamp value High - RO */ +#define IGC_TIMADJ 0x0B60C /* Time Adjustment Offset Register */ + +/* PCIe Registers */ +#define IGC_PTM_CTRL 0x12540 /* PTM Control */ +#define IGC_PTM_STAT 0x12544 /* PTM Status */ +#define IGC_PTM_CYCLE_CTRL 0x1254C /* PTM Cycle Control */ + +/* PTM Time registers */ +#define IGC_PTM_T1_TIM0_L 0x12558 /* T1 on Timer 0 Low */ +#define IGC_PTM_T1_TIM0_H 0x1255C /* T1 on Timer 0 High */ + +#define IGC_PTM_CURR_T2_L 0x1258C /* Current T2 Low */ +#define IGC_PTM_CURR_T2_H 0x12590 /* Current T2 High */ +#define IGC_PTM_PREV_T2_L 0x12584 /* Previous T2 Low */ +#define IGC_PTM_PREV_T2_H 0x12588 /* Previous T2 High */ +#define IGC_PTM_PREV_T4M1 0x12578 /* T4 Minus T1 on previous PTM Cycle */ +#define IGC_PTM_CURR_T4M1 0x1257C /* T4 Minus T1 on this PTM Cycle */ +#define IGC_PTM_PREV_T3M2 0x12580 /* T3 Minus T2 on previous PTM Cycle */ +#define IGC_PTM_TDELAY 0x12594 /* PTM PCIe Link Delay */ + +#define IGC_PCIE_DIG_DELAY 0x12550 /* PCIe Digital Delay */ +#define IGC_PCIE_PHY_DELAY 0x12554 /* PCIe PHY Delay */ + /* Management registers */ #define IGC_MANC 0x05820 /* Management Control - RW */ @@ -248,10 +279,22 @@ #define IGC_WUFC 0x05808 /* Wakeup Filter Control - RW */ #define IGC_WUS 0x05810 /* Wakeup Status - R/W1C */ #define IGC_WUPL 0x05900 /* Wakeup Packet Length - RW */ +#define IGC_WUFC_EXT 0x0580C /* Wakeup Filter Control Register Extended - RW */ /* Wake Up packet memory */ #define IGC_WUPM_REG(_i) (0x05A00 + ((_i) * 4)) +/* Energy Efficient Ethernet "EEE" registers */ +#define IGC_EEER 0x0E30 /* Energy Efficient Ethernet "EEE"*/ +#define IGC_IPCNFG 0x0E38 /* Internal PHY Configuration */ +#define IGC_EEE_SU 0x0E34 /* EEE Setup */ + +/* LTR registers */ +#define IGC_LTRC 0x01A0 /* Latency Tolerance Reporting Control */ +#define IGC_DMACR 0x02508 /* DMA Coalescing Control Register */ +#define IGC_LTRMINV 0x5BB0 /* LTR Minimum Value */ +#define IGC_LTRMAXV 0x5BB4 /* LTR Maximum Value */ + /* forward declaration */ struct igc_hw; u32 igc_rd32(struct igc_hw *hw, u32 reg); @@ -273,4 +316,6 @@ do { \ #define array_rd32(reg, offset) (igc_rd32(hw, (reg) + ((offset) << 2))) +#define IGC_REMOVED(h) unlikely(!(h)) + #endif diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c new file mode 100644 index 000000000000..0fce22de2ab8 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_tsn.c @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 Intel Corporation */ + +#include "igc.h" +#include "igc_tsn.h" + +static bool is_any_launchtime(struct igc_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + if (ring->launchtime_enable) + return true; + } + + return false; +} + +static bool is_cbs_enabled(struct igc_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + if (ring->cbs_enable) + return true; + } + + return false; +} + +static unsigned int igc_tsn_new_flags(struct igc_adapter *adapter) +{ + unsigned int new_flags = adapter->flags & ~IGC_FLAG_TSN_ANY_ENABLED; + + if (adapter->base_time) + new_flags |= IGC_FLAG_TSN_QBV_ENABLED; + + if (is_any_launchtime(adapter)) + new_flags |= IGC_FLAG_TSN_QBV_ENABLED; + + if (is_cbs_enabled(adapter)) + new_flags |= IGC_FLAG_TSN_QAV_ENABLED; + + return new_flags; +} + +/* Returns the TSN specific registers to their default values after + * the adapter is reset. + */ +static int igc_tsn_disable_offload(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 tqavctrl; + int i; + + wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); + wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_DEFAULT); + + tqavctrl = rd32(IGC_TQAVCTRL); + tqavctrl &= ~(IGC_TQAVCTRL_TRANSMIT_MODE_TSN | + IGC_TQAVCTRL_ENHANCED_QAV); + wr32(IGC_TQAVCTRL, tqavctrl); + + for (i = 0; i < adapter->num_tx_queues; i++) { + wr32(IGC_TXQCTL(i), 0); + wr32(IGC_STQT(i), 0); + wr32(IGC_ENDQT(i), NSEC_PER_SEC); + } + + wr32(IGC_QBVCYCLET_S, 0); + wr32(IGC_QBVCYCLET, NSEC_PER_SEC); + + adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED; + + return 0; +} + +static int igc_tsn_enable_offload(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 tqavctrl, baset_l, baset_h; + u32 sec, nsec, cycle; + ktime_t base_time, systim; + int i; + + cycle = adapter->cycle_time; + base_time = adapter->base_time; + + wr32(IGC_TSAUXC, 0); + wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_TSN); + wr32(IGC_TXPBS, IGC_TXPBSIZE_TSN); + + tqavctrl = rd32(IGC_TQAVCTRL); + tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV; + wr32(IGC_TQAVCTRL, tqavctrl); + + wr32(IGC_QBVCYCLET_S, cycle); + wr32(IGC_QBVCYCLET, cycle); + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + u32 txqctl = 0; + u16 cbs_value; + u32 tqavcc; + + wr32(IGC_STQT(i), ring->start_time); + wr32(IGC_ENDQT(i), ring->end_time); + + if (adapter->base_time) { + /* If we have a base_time we are in "taprio" + * mode and we need to be strict about the + * cycles: only transmit a packet if it can be + * completed during that cycle. + */ + txqctl |= IGC_TXQCTL_STRICT_CYCLE | + IGC_TXQCTL_STRICT_END; + } + + if (ring->launchtime_enable) + txqctl |= IGC_TXQCTL_QUEUE_MODE_LAUNCHT; + + /* Skip configuring CBS for Q2 and Q3 */ + if (i > 1) + goto skip_cbs; + + if (ring->cbs_enable) { + if (i == 0) + txqctl |= IGC_TXQCTL_QAV_SEL_CBS0; + else + txqctl |= IGC_TXQCTL_QAV_SEL_CBS1; + + /* According to i225 datasheet section 7.5.2.7, we + * should set the 'idleSlope' field from TQAVCC + * register following the equation: + * + * value = link-speed 0x7736 * BW * 0.2 + * ---------- * ----------------- (E1) + * 100Mbps 2.5 + * + * Note that 'link-speed' is in Mbps. + * + * 'BW' is the percentage bandwidth out of full + * link speed which can be found with the + * following equation. Note that idleSlope here + * is the parameter from this function + * which is in kbps. + * + * BW = idleSlope + * ----------------- (E2) + * link-speed * 1000 + * + * That said, we can come up with a generic + * equation to calculate the value we should set + * it TQAVCC register by replacing 'BW' in E1 by E2. + * The resulting equation is: + * + * value = link-speed * 0x7736 * idleSlope * 0.2 + * ------------------------------------- (E3) + * 100 * 2.5 * link-speed * 1000 + * + * 'link-speed' is present in both sides of the + * fraction so it is canceled out. The final + * equation is the following: + * + * value = idleSlope * 61036 + * ----------------- (E4) + * 2500000 + * + * NOTE: For i225, given the above, we can see + * that idleslope is represented in + * 40.959433 kbps units by the value at + * the TQAVCC register (2.5Gbps / 61036), + * which reduces the granularity for + * idleslope increments. + * + * In i225 controller, the sendSlope and loCredit + * parameters from CBS are not configurable + * by software so we don't do any + * 'controller configuration' in respect to + * these parameters. + */ + cbs_value = DIV_ROUND_UP_ULL(ring->idleslope + * 61036ULL, 2500000); + + tqavcc = rd32(IGC_TQAVCC(i)); + tqavcc &= ~IGC_TQAVCC_IDLESLOPE_MASK; + tqavcc |= cbs_value | IGC_TQAVCC_KEEP_CREDITS; + wr32(IGC_TQAVCC(i), tqavcc); + + wr32(IGC_TQAVHC(i), + 0x80000000 + ring->hicredit * 0x7735); + } else { + /* Disable any CBS for the queue */ + txqctl &= ~(IGC_TXQCTL_QAV_SEL_MASK); + + /* Set idleSlope to zero. */ + tqavcc = rd32(IGC_TQAVCC(i)); + tqavcc &= ~(IGC_TQAVCC_IDLESLOPE_MASK | + IGC_TQAVCC_KEEP_CREDITS); + wr32(IGC_TQAVCC(i), tqavcc); + + /* Set hiCredit to zero. */ + wr32(IGC_TQAVHC(i), 0); + } +skip_cbs: + wr32(IGC_TXQCTL(i), txqctl); + } + + nsec = rd32(IGC_SYSTIML); + sec = rd32(IGC_SYSTIMH); + + systim = ktime_set(sec, nsec); + + if (ktime_compare(systim, base_time) > 0) { + s64 n; + + n = div64_s64(ktime_sub_ns(systim, base_time), cycle); + base_time = ktime_add_ns(base_time, (n + 1) * cycle); + } + + baset_h = div_s64_rem(base_time, NSEC_PER_SEC, &baset_l); + + wr32(IGC_BASET_H, baset_h); + wr32(IGC_BASET_L, baset_l); + + return 0; +} + +int igc_tsn_reset(struct igc_adapter *adapter) +{ + unsigned int new_flags; + int err = 0; + + new_flags = igc_tsn_new_flags(adapter); + + if (!(new_flags & IGC_FLAG_TSN_ANY_ENABLED)) + return igc_tsn_disable_offload(adapter); + + err = igc_tsn_enable_offload(adapter); + if (err < 0) + return err; + + adapter->flags = new_flags; + + return err; +} + +int igc_tsn_offload_apply(struct igc_adapter *adapter) +{ + int err; + + if (netif_running(adapter->netdev)) { + schedule_work(&adapter->reset_task); + return 0; + } + + err = igc_tsn_enable_offload(adapter); + if (err < 0) + return err; + + adapter->flags = igc_tsn_new_flags(adapter); + return 0; +} diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.h b/drivers/net/ethernet/intel/igc/igc_tsn.h new file mode 100644 index 000000000000..1512307f5a52 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_tsn.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020 Intel Corporation */ + +#ifndef _IGC_TSN_H_ +#define _IGC_TSN_H_ + +int igc_tsn_offload_apply(struct igc_adapter *adapter); +int igc_tsn_reset(struct igc_adapter *adapter); + +#endif /* _IGC_BASE_H */ diff --git a/drivers/net/ethernet/intel/igc/igc_xdp.c b/drivers/net/ethernet/intel/igc/igc_xdp.c new file mode 100644 index 000000000000..aeeb34e64610 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_xdp.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Intel Corporation. */ + +#include <linux/if_vlan.h> +#include <net/xdp_sock_drv.h> + +#include "igc.h" +#include "igc_xdp.h" + +int igc_xdp_set_prog(struct igc_adapter *adapter, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct net_device *dev = adapter->netdev; + bool if_running = netif_running(dev); + struct bpf_prog *old_prog; + + if (dev->mtu > ETH_DATA_LEN) { + /* For now, the driver doesn't support XDP functionality with + * jumbo frames so we return error. + */ + NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported"); + return -EOPNOTSUPP; + } + + if (if_running) + igc_close(dev); + + old_prog = xchg(&adapter->xdp_prog, prog); + if (old_prog) + bpf_prog_put(old_prog); + + if (if_running) + igc_open(dev); + + return 0; +} + +static int igc_xdp_enable_pool(struct igc_adapter *adapter, + struct xsk_buff_pool *pool, u16 queue_id) +{ + struct net_device *ndev = adapter->netdev; + struct device *dev = &adapter->pdev->dev; + struct igc_ring *rx_ring, *tx_ring; + struct napi_struct *napi; + bool needs_reset; + u32 frame_size; + int err; + + if (queue_id >= adapter->num_rx_queues || + queue_id >= adapter->num_tx_queues) + return -EINVAL; + + frame_size = xsk_pool_get_rx_frame_size(pool); + if (frame_size < ETH_FRAME_LEN + VLAN_HLEN * 2) { + /* When XDP is enabled, the driver doesn't support frames that + * span over multiple buffers. To avoid that, we check if xsk + * frame size is big enough to fit the max ethernet frame size + * + vlan double tagging. + */ + return -EOPNOTSUPP; + } + + err = xsk_pool_dma_map(pool, dev, IGC_RX_DMA_ATTR); + if (err) { + netdev_err(ndev, "Failed to map xsk pool\n"); + return err; + } + + needs_reset = netif_running(adapter->netdev) && igc_xdp_is_enabled(adapter); + + rx_ring = adapter->rx_ring[queue_id]; + tx_ring = adapter->tx_ring[queue_id]; + /* Rx and Tx rings share the same napi context. */ + napi = &rx_ring->q_vector->napi; + + if (needs_reset) { + igc_disable_rx_ring(rx_ring); + igc_disable_tx_ring(tx_ring); + napi_disable(napi); + } + + set_bit(IGC_RING_FLAG_AF_XDP_ZC, &rx_ring->flags); + set_bit(IGC_RING_FLAG_AF_XDP_ZC, &tx_ring->flags); + + if (needs_reset) { + napi_enable(napi); + igc_enable_rx_ring(rx_ring); + igc_enable_tx_ring(tx_ring); + + err = igc_xsk_wakeup(ndev, queue_id, XDP_WAKEUP_RX); + if (err) { + xsk_pool_dma_unmap(pool, IGC_RX_DMA_ATTR); + return err; + } + } + + return 0; +} + +static int igc_xdp_disable_pool(struct igc_adapter *adapter, u16 queue_id) +{ + struct igc_ring *rx_ring, *tx_ring; + struct xsk_buff_pool *pool; + struct napi_struct *napi; + bool needs_reset; + + if (queue_id >= adapter->num_rx_queues || + queue_id >= adapter->num_tx_queues) + return -EINVAL; + + pool = xsk_get_pool_from_qid(adapter->netdev, queue_id); + if (!pool) + return -EINVAL; + + needs_reset = netif_running(adapter->netdev) && igc_xdp_is_enabled(adapter); + + rx_ring = adapter->rx_ring[queue_id]; + tx_ring = adapter->tx_ring[queue_id]; + /* Rx and Tx rings share the same napi context. */ + napi = &rx_ring->q_vector->napi; + + if (needs_reset) { + igc_disable_rx_ring(rx_ring); + igc_disable_tx_ring(tx_ring); + napi_disable(napi); + } + + xsk_pool_dma_unmap(pool, IGC_RX_DMA_ATTR); + clear_bit(IGC_RING_FLAG_AF_XDP_ZC, &rx_ring->flags); + clear_bit(IGC_RING_FLAG_AF_XDP_ZC, &tx_ring->flags); + + if (needs_reset) { + napi_enable(napi); + igc_enable_rx_ring(rx_ring); + igc_enable_tx_ring(tx_ring); + } + + return 0; +} + +int igc_xdp_setup_pool(struct igc_adapter *adapter, struct xsk_buff_pool *pool, + u16 queue_id) +{ + return pool ? igc_xdp_enable_pool(adapter, pool, queue_id) : + igc_xdp_disable_pool(adapter, queue_id); +} diff --git a/drivers/net/ethernet/intel/igc/igc_xdp.h b/drivers/net/ethernet/intel/igc/igc_xdp.h new file mode 100644 index 000000000000..a74e5487d199 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_xdp.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020, Intel Corporation. */ + +#ifndef _IGC_XDP_H_ +#define _IGC_XDP_H_ + +int igc_xdp_set_prog(struct igc_adapter *adapter, struct bpf_prog *prog, + struct netlink_ext_ack *extack); +int igc_xdp_setup_pool(struct igc_adapter *adapter, struct xsk_buff_pool *pool, + u16 queue_id); + +static inline bool igc_xdp_is_enabled(struct igc_adapter *adapter) +{ + return !!adapter->xdp_prog; +} + +#endif /* _IGC_XDP_H_ */ |