diff options
Diffstat (limited to 'include/net')
54 files changed, 1485 insertions, 636 deletions
diff --git a/include/net/6lowpan.h b/include/net/6lowpan.h index da3a77d25fcb..da84cf920b78 100644 --- a/include/net/6lowpan.h +++ b/include/net/6lowpan.h @@ -58,6 +58,9 @@ #include <net/ipv6.h> #include <net/net_namespace.h> +/* special link-layer handling */ +#include <net/mac802154.h> + #define EUI64_ADDR_LEN 8 #define LOWPAN_NHC_MAX_ID_LEN 1 @@ -93,7 +96,7 @@ static inline bool lowpan_is_iphc(u8 dispatch) } #define LOWPAN_PRIV_SIZE(llpriv_size) \ - (sizeof(struct lowpan_priv) + llpriv_size) + (sizeof(struct lowpan_dev) + llpriv_size) enum lowpan_lltypes { LOWPAN_LLTYPE_BTLE, @@ -129,7 +132,7 @@ lowpan_iphc_ctx_is_compression(const struct lowpan_iphc_ctx *ctx) return test_bit(LOWPAN_IPHC_CTX_FLAG_COMPRESSION, &ctx->flags); } -struct lowpan_priv { +struct lowpan_dev { enum lowpan_lltypes lltype; struct dentry *iface_debugfs; struct lowpan_iphc_ctx_table ctx; @@ -139,11 +142,23 @@ struct lowpan_priv { }; static inline -struct lowpan_priv *lowpan_priv(const struct net_device *dev) +struct lowpan_dev *lowpan_dev(const struct net_device *dev) { return netdev_priv(dev); } +/* private device info */ +struct lowpan_802154_dev { + struct net_device *wdev; /* wpan device ptr */ + u16 fragment_tag; +}; + +static inline struct +lowpan_802154_dev *lowpan_802154_dev(const struct net_device *dev) +{ + return (struct lowpan_802154_dev *)lowpan_dev(dev)->priv; +} + struct lowpan_802154_cb { u16 d_tag; unsigned int d_size; @@ -157,6 +172,22 @@ struct lowpan_802154_cb *lowpan_802154_cb(const struct sk_buff *skb) return (struct lowpan_802154_cb *)skb->cb; } +static inline void lowpan_iphc_uncompress_eui64_lladdr(struct in6_addr *ipaddr, + const void *lladdr) +{ + /* fe:80::XXXX:XXXX:XXXX:XXXX + * \_________________/ + * hwaddr + */ + ipaddr->s6_addr[0] = 0xFE; + ipaddr->s6_addr[1] = 0x80; + memcpy(&ipaddr->s6_addr[8], lladdr, EUI64_ADDR_LEN); + /* second bit-flip (Universe/Local) + * is done according RFC2464 + */ + ipaddr->s6_addr[8] ^= 0x02; +} + #ifdef DEBUG /* print data in line */ static inline void raw_dump_inline(const char *caller, char *msg, diff --git a/include/net/act_api.h b/include/net/act_api.h index 2a19fe111c78..03e322b30218 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -135,6 +135,7 @@ void tcf_hashinfo_destroy(const struct tc_action_ops *ops, static inline void tc_action_net_exit(struct tc_action_net *tn) { tcf_hashinfo_destroy(tn->ops, tn->hinfo); + kfree(tn->hinfo); } int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, diff --git a/include/net/af_rxrpc.h b/include/net/af_rxrpc.h index e797d45a5ae6..ac1bc3c49fbd 100644 --- a/include/net/af_rxrpc.h +++ b/include/net/af_rxrpc.h @@ -12,6 +12,7 @@ #ifndef _NET_RXRPC_H #define _NET_RXRPC_H +#include <linux/skbuff.h> #include <linux/rxrpc.h> struct rxrpc_call; @@ -19,11 +20,12 @@ struct rxrpc_call; /* * the mark applied to socket buffers that may be intercepted */ -enum { +enum rxrpc_skb_mark { RXRPC_SKB_MARK_DATA, /* data message */ RXRPC_SKB_MARK_FINAL_ACK, /* final ACK received message */ RXRPC_SKB_MARK_BUSY, /* server busy message */ RXRPC_SKB_MARK_REMOTE_ABORT, /* remote abort message */ + RXRPC_SKB_MARK_LOCAL_ABORT, /* local abort message */ RXRPC_SKB_MARK_NET_ERROR, /* network error message */ RXRPC_SKB_MARK_LOCAL_ERROR, /* local error message */ RXRPC_SKB_MARK_NEW_CALL, /* local error message */ diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index 5d38d980b89d..eefcf3e96421 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -61,6 +61,8 @@ #define HCI_RS232 4 #define HCI_PCI 5 #define HCI_SDIO 6 +#define HCI_SPI 7 +#define HCI_I2C 8 /* HCI controller types */ #define HCI_BREDR 0x00 diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index b39277eb251f..183916e168f1 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -68,26 +68,6 @@ struct wiphy; */ /** - * enum ieee80211_band - supported frequency bands - * - * The bands are assigned this way because the supported - * bitrates differ in these bands. - * - * @IEEE80211_BAND_2GHZ: 2.4GHz ISM band - * @IEEE80211_BAND_5GHZ: around 5GHz band (4.9-5.7) - * @IEEE80211_BAND_60GHZ: around 60 GHz band (58.32 - 64.80 GHz) - * @IEEE80211_NUM_BANDS: number of defined bands - */ -enum ieee80211_band { - IEEE80211_BAND_2GHZ = NL80211_BAND_2GHZ, - IEEE80211_BAND_5GHZ = NL80211_BAND_5GHZ, - IEEE80211_BAND_60GHZ = NL80211_BAND_60GHZ, - - /* keep last */ - IEEE80211_NUM_BANDS -}; - -/** * enum ieee80211_channel_flags - channel flags * * Channel flags set by the regulatory control code. @@ -167,7 +147,7 @@ enum ieee80211_channel_flags { * @dfs_cac_ms: DFS CAC time in milliseconds, this is valid for DFS channels. */ struct ieee80211_channel { - enum ieee80211_band band; + enum nl80211_band band; u16 center_freq; u16 hw_value; u32 flags; @@ -324,7 +304,7 @@ struct ieee80211_sta_vht_cap { struct ieee80211_supported_band { struct ieee80211_channel *channels; struct ieee80211_rate *bitrates; - enum ieee80211_band band; + enum nl80211_band band; int n_channels; int n_bitrates; struct ieee80211_sta_ht_cap ht_cap; @@ -1370,7 +1350,7 @@ struct mesh_setup { bool user_mpm; u8 dtim_period; u16 beacon_interval; - int mcast_rate[IEEE80211_NUM_BANDS]; + int mcast_rate[NUM_NL80211_BANDS]; u32 basic_rates; }; @@ -1468,7 +1448,7 @@ struct cfg80211_scan_request { size_t ie_len; u32 flags; - u32 rates[IEEE80211_NUM_BANDS]; + u32 rates[NUM_NL80211_BANDS]; struct wireless_dev *wdev; @@ -1750,7 +1730,12 @@ enum cfg80211_assoc_req_flags { * @ie_len: Length of ie buffer in octets * @use_mfp: Use management frame protection (IEEE 802.11w) in this association * @crypto: crypto settings - * @prev_bssid: previous BSSID, if not %NULL use reassociate frame + * @prev_bssid: previous BSSID, if not %NULL use reassociate frame. This is used + * to indicate a request to reassociate within the ESS instead of a request + * do the initial association with the ESS. When included, this is set to + * the BSSID of the current association, i.e., to the value that is + * included in the Current AP address field of the Reassociation Request + * frame. * @flags: See &enum cfg80211_assoc_req_flags * @ht_capa: HT Capabilities over-rides. Values set in ht_capa_mask * will be used in ht_capa. Un-supported values will be ignored. @@ -1855,7 +1840,7 @@ struct cfg80211_ibss_params { bool privacy; bool control_port; bool userspace_handles_dfs; - int mcast_rate[IEEE80211_NUM_BANDS]; + int mcast_rate[NUM_NL80211_BANDS]; struct ieee80211_ht_cap ht_capa; struct ieee80211_ht_cap ht_capa_mask; }; @@ -1867,7 +1852,7 @@ struct cfg80211_ibss_params { * @delta: value of RSSI level adjustment. */ struct cfg80211_bss_select_adjust { - enum ieee80211_band band; + enum nl80211_band band; s8 delta; }; @@ -1882,7 +1867,7 @@ struct cfg80211_bss_select_adjust { struct cfg80211_bss_selection { enum nl80211_bss_select_attr behaviour; union { - enum ieee80211_band band_pref; + enum nl80211_band band_pref; struct cfg80211_bss_select_adjust adjust; } param; }; @@ -1925,7 +1910,12 @@ struct cfg80211_bss_selection { * @pbss: if set, connect to a PCP instead of AP. Valid for DMG * networks. * @bss_select: criteria to be used for BSS selection. - * @prev_bssid: previous BSSID, if not %NULL use reassociate frame + * @prev_bssid: previous BSSID, if not %NULL use reassociate frame. This is used + * to indicate a request to reassociate within the ESS instead of a request + * do the initial association with the ESS. When included, this is set to + * the BSSID of the current association, i.e., to the value that is + * included in the Current AP address field of the Reassociation Request + * frame. */ struct cfg80211_connect_params { struct ieee80211_channel *channel; @@ -1980,7 +1970,7 @@ struct cfg80211_bitrate_mask { u8 ht_mcs[IEEE80211_HT_MCS_MASK_LEN]; u16 vht_mcs[NL80211_VHT_NSS_MAX]; enum nl80211_txrate_gi gi; - } control[IEEE80211_NUM_BANDS]; + } control[NUM_NL80211_BANDS]; }; /** * struct cfg80211_pmksa - PMK Security Association @@ -2377,7 +2367,17 @@ struct cfg80211_qos_map { * @connect: Connect to the ESS with the specified parameters. When connected, * call cfg80211_connect_result() with status code %WLAN_STATUS_SUCCESS. * If the connection fails for some reason, call cfg80211_connect_result() - * with the status from the AP. + * with the status from the AP. The driver is allowed to roam to other + * BSSes within the ESS when the other BSS matches the connect parameters. + * When such roaming is initiated by the driver, the driver is expected to + * verify that the target matches the configured security parameters and + * to use Reassociation Request frame instead of Association Request frame. + * The connect function can also be used to request the driver to perform + * a specific roam when connected to an ESS. In that case, the prev_bssid + * parameter is set to the BSSID of the currently associated BSS as an + * indication of requesting reassociation. In both the driver-initiated and + * new connect() call initiated roaming cases, the result of roaming is + * indicated with a call to cfg80211_roamed() or cfg80211_roamed_bss(). * (invoked with the wireless_dev mutex held) * @disconnect: Disconnect from the BSS/ESS. * (invoked with the wireless_dev mutex held) @@ -2657,7 +2657,7 @@ struct cfg80211_ops { int (*leave_ibss)(struct wiphy *wiphy, struct net_device *dev); int (*set_mcast_rate)(struct wiphy *wiphy, struct net_device *dev, - int rate[IEEE80211_NUM_BANDS]); + int rate[NUM_NL80211_BANDS]); int (*set_wiphy_params)(struct wiphy *wiphy, u32 changed); @@ -3303,7 +3303,7 @@ struct wiphy { * help determine whether you own this wiphy or not. */ const void *privid; - struct ieee80211_supported_band *bands[IEEE80211_NUM_BANDS]; + struct ieee80211_supported_band *bands[NUM_NL80211_BANDS]; /* Lets us get back the wiphy on the callback */ void (*reg_notifier)(struct wiphy *wiphy, @@ -3638,7 +3638,7 @@ static inline void *wdev_priv(struct wireless_dev *wdev) * @band: band, necessary due to channel number overlap * Return: The corresponding frequency (in MHz), or 0 if the conversion failed. */ -int ieee80211_channel_to_frequency(int chan, enum ieee80211_band band); +int ieee80211_channel_to_frequency(int chan, enum nl80211_band band); /** * ieee80211_frequency_to_channel - convert frequency to channel number @@ -5069,7 +5069,7 @@ void cfg80211_ch_switch_started_notify(struct net_device *dev, * Returns %true if the conversion was successful, %false otherwise. */ bool ieee80211_operating_class_to_band(u8 operating_class, - enum ieee80211_band *band); + enum nl80211_band *band); /** * ieee80211_chandef_to_operating_class - convert chandef to operation class diff --git a/include/net/cls_cgroup.h b/include/net/cls_cgroup.h index c0a92e2c286d..74c9693d4941 100644 --- a/include/net/cls_cgroup.h +++ b/include/net/cls_cgroup.h @@ -17,6 +17,7 @@ #include <linux/hardirq.h> #include <linux/rcupdate.h> #include <net/sock.h> +#include <net/inet_sock.h> #ifdef CONFIG_CGROUP_NET_CLASSID struct cgroup_cls_state { @@ -63,11 +64,13 @@ static inline u32 task_get_classid(const struct sk_buff *skb) * softirqs always disables bh. */ if (in_serving_softirq()) { + struct sock *sk = skb_to_full_sk(skb); + /* If there is an sock_cgroup_classid we'll use that. */ - if (!skb->sk) + if (!sk || !sk_fullsock(sk)) return 0; - classid = sock_cgroup_classid(&skb->sk->sk_cgrp_data); + classid = sock_cgroup_classid(&sk->sk_cgrp_data); } return classid; diff --git a/include/net/codel.h b/include/net/codel.h index d168aca115cc..a6e428f80135 100644 --- a/include/net/codel.h +++ b/include/net/codel.h @@ -87,27 +87,6 @@ static inline codel_time_t codel_get_time(void) ((s32)((a) - (b)) >= 0)) #define codel_time_before_eq(a, b) codel_time_after_eq(b, a) -/* Qdiscs using codel plugin must use codel_skb_cb in their own cb[] */ -struct codel_skb_cb { - codel_time_t enqueue_time; -}; - -static struct codel_skb_cb *get_codel_cb(const struct sk_buff *skb) -{ - qdisc_cb_private_validate(skb, sizeof(struct codel_skb_cb)); - return (struct codel_skb_cb *)qdisc_skb_cb(skb)->data; -} - -static codel_time_t codel_get_enqueue_time(const struct sk_buff *skb) -{ - return get_codel_cb(skb)->enqueue_time; -} - -static void codel_set_enqueue_time(struct sk_buff *skb) -{ - get_codel_cb(skb)->enqueue_time = codel_get_time(); -} - static inline u32 codel_time_to_us(codel_time_t val) { u64 valns = ((u64)val << CODEL_SHIFT); @@ -176,198 +155,10 @@ struct codel_stats { #define CODEL_DISABLED_THRESHOLD INT_MAX -static void codel_params_init(struct codel_params *params, - const struct Qdisc *sch) -{ - params->interval = MS2TIME(100); - params->target = MS2TIME(5); - params->mtu = psched_mtu(qdisc_dev(sch)); - params->ce_threshold = CODEL_DISABLED_THRESHOLD; - params->ecn = false; -} - -static void codel_vars_init(struct codel_vars *vars) -{ - memset(vars, 0, sizeof(*vars)); -} - -static void codel_stats_init(struct codel_stats *stats) -{ - stats->maxpacket = 0; -} - -/* - * http://en.wikipedia.org/wiki/Methods_of_computing_square_roots#Iterative_methods_for_reciprocal_square_roots - * new_invsqrt = (invsqrt / 2) * (3 - count * invsqrt^2) - * - * Here, invsqrt is a fixed point number (< 1.0), 32bit mantissa, aka Q0.32 - */ -static void codel_Newton_step(struct codel_vars *vars) -{ - u32 invsqrt = ((u32)vars->rec_inv_sqrt) << REC_INV_SQRT_SHIFT; - u32 invsqrt2 = ((u64)invsqrt * invsqrt) >> 32; - u64 val = (3LL << 32) - ((u64)vars->count * invsqrt2); - - val >>= 2; /* avoid overflow in following multiply */ - val = (val * invsqrt) >> (32 - 2 + 1); - - vars->rec_inv_sqrt = val >> REC_INV_SQRT_SHIFT; -} - -/* - * CoDel control_law is t + interval/sqrt(count) - * We maintain in rec_inv_sqrt the reciprocal value of sqrt(count) to avoid - * both sqrt() and divide operation. - */ -static codel_time_t codel_control_law(codel_time_t t, - codel_time_t interval, - u32 rec_inv_sqrt) -{ - return t + reciprocal_scale(interval, rec_inv_sqrt << REC_INV_SQRT_SHIFT); -} - -static bool codel_should_drop(const struct sk_buff *skb, - struct Qdisc *sch, - struct codel_vars *vars, - struct codel_params *params, - struct codel_stats *stats, - codel_time_t now) -{ - bool ok_to_drop; - - if (!skb) { - vars->first_above_time = 0; - return false; - } - - vars->ldelay = now - codel_get_enqueue_time(skb); - sch->qstats.backlog -= qdisc_pkt_len(skb); - - if (unlikely(qdisc_pkt_len(skb) > stats->maxpacket)) - stats->maxpacket = qdisc_pkt_len(skb); - - if (codel_time_before(vars->ldelay, params->target) || - sch->qstats.backlog <= params->mtu) { - /* went below - stay below for at least interval */ - vars->first_above_time = 0; - return false; - } - ok_to_drop = false; - if (vars->first_above_time == 0) { - /* just went above from below. If we stay above - * for at least interval we'll say it's ok to drop - */ - vars->first_above_time = now + params->interval; - } else if (codel_time_after(now, vars->first_above_time)) { - ok_to_drop = true; - } - return ok_to_drop; -} - +typedef u32 (*codel_skb_len_t)(const struct sk_buff *skb); +typedef codel_time_t (*codel_skb_time_t)(const struct sk_buff *skb); +typedef void (*codel_skb_drop_t)(struct sk_buff *skb, void *ctx); typedef struct sk_buff * (*codel_skb_dequeue_t)(struct codel_vars *vars, - struct Qdisc *sch); + void *ctx); -static struct sk_buff *codel_dequeue(struct Qdisc *sch, - struct codel_params *params, - struct codel_vars *vars, - struct codel_stats *stats, - codel_skb_dequeue_t dequeue_func) -{ - struct sk_buff *skb = dequeue_func(vars, sch); - codel_time_t now; - bool drop; - - if (!skb) { - vars->dropping = false; - return skb; - } - now = codel_get_time(); - drop = codel_should_drop(skb, sch, vars, params, stats, now); - if (vars->dropping) { - if (!drop) { - /* sojourn time below target - leave dropping state */ - vars->dropping = false; - } else if (codel_time_after_eq(now, vars->drop_next)) { - /* It's time for the next drop. Drop the current - * packet and dequeue the next. The dequeue might - * take us out of dropping state. - * If not, schedule the next drop. - * A large backlog might result in drop rates so high - * that the next drop should happen now, - * hence the while loop. - */ - while (vars->dropping && - codel_time_after_eq(now, vars->drop_next)) { - vars->count++; /* dont care of possible wrap - * since there is no more divide - */ - codel_Newton_step(vars); - if (params->ecn && INET_ECN_set_ce(skb)) { - stats->ecn_mark++; - vars->drop_next = - codel_control_law(vars->drop_next, - params->interval, - vars->rec_inv_sqrt); - goto end; - } - stats->drop_len += qdisc_pkt_len(skb); - qdisc_drop(skb, sch); - stats->drop_count++; - skb = dequeue_func(vars, sch); - if (!codel_should_drop(skb, sch, - vars, params, stats, now)) { - /* leave dropping state */ - vars->dropping = false; - } else { - /* and schedule the next drop */ - vars->drop_next = - codel_control_law(vars->drop_next, - params->interval, - vars->rec_inv_sqrt); - } - } - } - } else if (drop) { - u32 delta; - - if (params->ecn && INET_ECN_set_ce(skb)) { - stats->ecn_mark++; - } else { - stats->drop_len += qdisc_pkt_len(skb); - qdisc_drop(skb, sch); - stats->drop_count++; - - skb = dequeue_func(vars, sch); - drop = codel_should_drop(skb, sch, vars, params, - stats, now); - } - vars->dropping = true; - /* if min went above target close to when we last went below it - * assume that the drop rate that controlled the queue on the - * last cycle is a good starting point to control it now. - */ - delta = vars->count - vars->lastcount; - if (delta > 1 && - codel_time_before(now - vars->drop_next, - 16 * params->interval)) { - vars->count = delta; - /* we dont care if rec_inv_sqrt approximation - * is not very precise : - * Next Newton steps will correct it quadratically. - */ - codel_Newton_step(vars); - } else { - vars->count = 1; - vars->rec_inv_sqrt = ~0U >> REC_INV_SQRT_SHIFT; - } - vars->lastcount = vars->count; - vars->drop_next = codel_control_law(now, params->interval, - vars->rec_inv_sqrt); - } -end: - if (skb && codel_time_after(vars->ldelay, params->ce_threshold) && - INET_ECN_set_ce(skb)) - stats->ce_mark++; - return skb; -} #endif diff --git a/include/net/codel_impl.h b/include/net/codel_impl.h new file mode 100644 index 000000000000..d289b91dcd65 --- /dev/null +++ b/include/net/codel_impl.h @@ -0,0 +1,255 @@ +#ifndef __NET_SCHED_CODEL_IMPL_H +#define __NET_SCHED_CODEL_IMPL_H + +/* + * Codel - The Controlled-Delay Active Queue Management algorithm + * + * Copyright (C) 2011-2012 Kathleen Nichols <nichols@pollere.com> + * Copyright (C) 2011-2012 Van Jacobson <van@pollere.net> + * Copyright (C) 2012 Michael D. Taht <dave.taht@bufferbloat.net> + * Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The names of the authors may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General + * Public License ("GPL") version 2, in which case the provisions of the + * GPL apply INSTEAD OF those given above. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + * + */ + +/* Controlling Queue Delay (CoDel) algorithm + * ========================================= + * Source : Kathleen Nichols and Van Jacobson + * http://queue.acm.org/detail.cfm?id=2209336 + * + * Implemented on linux by Dave Taht and Eric Dumazet + */ + +static void codel_params_init(struct codel_params *params) +{ + params->interval = MS2TIME(100); + params->target = MS2TIME(5); + params->ce_threshold = CODEL_DISABLED_THRESHOLD; + params->ecn = false; +} + +static void codel_vars_init(struct codel_vars *vars) +{ + memset(vars, 0, sizeof(*vars)); +} + +static void codel_stats_init(struct codel_stats *stats) +{ + stats->maxpacket = 0; +} + +/* + * http://en.wikipedia.org/wiki/Methods_of_computing_square_roots#Iterative_methods_for_reciprocal_square_roots + * new_invsqrt = (invsqrt / 2) * (3 - count * invsqrt^2) + * + * Here, invsqrt is a fixed point number (< 1.0), 32bit mantissa, aka Q0.32 + */ +static void codel_Newton_step(struct codel_vars *vars) +{ + u32 invsqrt = ((u32)vars->rec_inv_sqrt) << REC_INV_SQRT_SHIFT; + u32 invsqrt2 = ((u64)invsqrt * invsqrt) >> 32; + u64 val = (3LL << 32) - ((u64)vars->count * invsqrt2); + + val >>= 2; /* avoid overflow in following multiply */ + val = (val * invsqrt) >> (32 - 2 + 1); + + vars->rec_inv_sqrt = val >> REC_INV_SQRT_SHIFT; +} + +/* + * CoDel control_law is t + interval/sqrt(count) + * We maintain in rec_inv_sqrt the reciprocal value of sqrt(count) to avoid + * both sqrt() and divide operation. + */ +static codel_time_t codel_control_law(codel_time_t t, + codel_time_t interval, + u32 rec_inv_sqrt) +{ + return t + reciprocal_scale(interval, rec_inv_sqrt << REC_INV_SQRT_SHIFT); +} + +static bool codel_should_drop(const struct sk_buff *skb, + void *ctx, + struct codel_vars *vars, + struct codel_params *params, + struct codel_stats *stats, + codel_skb_len_t skb_len_func, + codel_skb_time_t skb_time_func, + u32 *backlog, + codel_time_t now) +{ + bool ok_to_drop; + u32 skb_len; + + if (!skb) { + vars->first_above_time = 0; + return false; + } + + skb_len = skb_len_func(skb); + vars->ldelay = now - skb_time_func(skb); + + if (unlikely(skb_len > stats->maxpacket)) + stats->maxpacket = skb_len; + + if (codel_time_before(vars->ldelay, params->target) || + *backlog <= params->mtu) { + /* went below - stay below for at least interval */ + vars->first_above_time = 0; + return false; + } + ok_to_drop = false; + if (vars->first_above_time == 0) { + /* just went above from below. If we stay above + * for at least interval we'll say it's ok to drop + */ + vars->first_above_time = now + params->interval; + } else if (codel_time_after(now, vars->first_above_time)) { + ok_to_drop = true; + } + return ok_to_drop; +} + +static struct sk_buff *codel_dequeue(void *ctx, + u32 *backlog, + struct codel_params *params, + struct codel_vars *vars, + struct codel_stats *stats, + codel_skb_len_t skb_len_func, + codel_skb_time_t skb_time_func, + codel_skb_drop_t drop_func, + codel_skb_dequeue_t dequeue_func) +{ + struct sk_buff *skb = dequeue_func(vars, ctx); + codel_time_t now; + bool drop; + + if (!skb) { + vars->dropping = false; + return skb; + } + now = codel_get_time(); + drop = codel_should_drop(skb, ctx, vars, params, stats, + skb_len_func, skb_time_func, backlog, now); + if (vars->dropping) { + if (!drop) { + /* sojourn time below target - leave dropping state */ + vars->dropping = false; + } else if (codel_time_after_eq(now, vars->drop_next)) { + /* It's time for the next drop. Drop the current + * packet and dequeue the next. The dequeue might + * take us out of dropping state. + * If not, schedule the next drop. + * A large backlog might result in drop rates so high + * that the next drop should happen now, + * hence the while loop. + */ + while (vars->dropping && + codel_time_after_eq(now, vars->drop_next)) { + vars->count++; /* dont care of possible wrap + * since there is no more divide + */ + codel_Newton_step(vars); + if (params->ecn && INET_ECN_set_ce(skb)) { + stats->ecn_mark++; + vars->drop_next = + codel_control_law(vars->drop_next, + params->interval, + vars->rec_inv_sqrt); + goto end; + } + stats->drop_len += skb_len_func(skb); + drop_func(skb, ctx); + stats->drop_count++; + skb = dequeue_func(vars, ctx); + if (!codel_should_drop(skb, ctx, + vars, params, stats, + skb_len_func, + skb_time_func, + backlog, now)) { + /* leave dropping state */ + vars->dropping = false; + } else { + /* and schedule the next drop */ + vars->drop_next = + codel_control_law(vars->drop_next, + params->interval, + vars->rec_inv_sqrt); + } + } + } + } else if (drop) { + u32 delta; + + if (params->ecn && INET_ECN_set_ce(skb)) { + stats->ecn_mark++; + } else { + stats->drop_len += skb_len_func(skb); + drop_func(skb, ctx); + stats->drop_count++; + + skb = dequeue_func(vars, ctx); + drop = codel_should_drop(skb, ctx, vars, params, + stats, skb_len_func, + skb_time_func, backlog, now); + } + vars->dropping = true; + /* if min went above target close to when we last went below it + * assume that the drop rate that controlled the queue on the + * last cycle is a good starting point to control it now. + */ + delta = vars->count - vars->lastcount; + if (delta > 1 && + codel_time_before(now - vars->drop_next, + 16 * params->interval)) { + vars->count = delta; + /* we dont care if rec_inv_sqrt approximation + * is not very precise : + * Next Newton steps will correct it quadratically. + */ + codel_Newton_step(vars); + } else { + vars->count = 1; + vars->rec_inv_sqrt = ~0U >> REC_INV_SQRT_SHIFT; + } + vars->lastcount = vars->count; + vars->drop_next = codel_control_law(now, params->interval, + vars->rec_inv_sqrt); + } +end: + if (skb && codel_time_after(vars->ldelay, params->ce_threshold) && + INET_ECN_set_ce(skb)) + stats->ce_mark++; + return skb; +} + +#endif diff --git a/include/net/codel_qdisc.h b/include/net/codel_qdisc.h new file mode 100644 index 000000000000..8144d9cd2908 --- /dev/null +++ b/include/net/codel_qdisc.h @@ -0,0 +1,73 @@ +#ifndef __NET_SCHED_CODEL_QDISC_H +#define __NET_SCHED_CODEL_QDISC_H + +/* + * Codel - The Controlled-Delay Active Queue Management algorithm + * + * Copyright (C) 2011-2012 Kathleen Nichols <nichols@pollere.com> + * Copyright (C) 2011-2012 Van Jacobson <van@pollere.net> + * Copyright (C) 2012 Michael D. Taht <dave.taht@bufferbloat.net> + * Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The names of the authors may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General + * Public License ("GPL") version 2, in which case the provisions of the + * GPL apply INSTEAD OF those given above. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + * + */ + +/* Controlling Queue Delay (CoDel) algorithm + * ========================================= + * Source : Kathleen Nichols and Van Jacobson + * http://queue.acm.org/detail.cfm?id=2209336 + * + * Implemented on linux by Dave Taht and Eric Dumazet + */ + +/* Qdiscs using codel plugin must use codel_skb_cb in their own cb[] */ +struct codel_skb_cb { + codel_time_t enqueue_time; +}; + +static struct codel_skb_cb *get_codel_cb(const struct sk_buff *skb) +{ + qdisc_cb_private_validate(skb, sizeof(struct codel_skb_cb)); + return (struct codel_skb_cb *)qdisc_skb_cb(skb)->data; +} + +static codel_time_t codel_get_enqueue_time(const struct sk_buff *skb) +{ + return get_codel_cb(skb)->enqueue_time; +} + +static void codel_set_enqueue_time(struct sk_buff *skb) +{ + get_codel_cb(skb)->enqueue_time = codel_get_time(); +} + +#endif diff --git a/include/net/devlink.h b/include/net/devlink.h index c37d257891d6..1d45b61cb320 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -24,6 +24,7 @@ struct devlink_ops; struct devlink { struct list_head list; struct list_head port_list; + struct list_head sb_list; const struct devlink_ops *ops; struct device *dev; possible_net_t _net; @@ -42,6 +43,12 @@ struct devlink_port { u32 split_group; }; +struct devlink_sb_pool_info { + enum devlink_sb_pool_type pool_type; + u32 size; + enum devlink_sb_threshold_type threshold_type; +}; + struct devlink_ops { size_t priv_size; int (*port_type_set)(struct devlink_port *devlink_port, @@ -49,6 +56,40 @@ struct devlink_ops { int (*port_split)(struct devlink *devlink, unsigned int port_index, unsigned int count); int (*port_unsplit)(struct devlink *devlink, unsigned int port_index); + int (*sb_pool_get)(struct devlink *devlink, unsigned int sb_index, + u16 pool_index, + struct devlink_sb_pool_info *pool_info); + int (*sb_pool_set)(struct devlink *devlink, unsigned int sb_index, + u16 pool_index, u32 size, + enum devlink_sb_threshold_type threshold_type); + int (*sb_port_pool_get)(struct devlink_port *devlink_port, + unsigned int sb_index, u16 pool_index, + u32 *p_threshold); + int (*sb_port_pool_set)(struct devlink_port *devlink_port, + unsigned int sb_index, u16 pool_index, + u32 threshold); + int (*sb_tc_pool_bind_get)(struct devlink_port *devlink_port, + unsigned int sb_index, + u16 tc_index, + enum devlink_sb_pool_type pool_type, + u16 *p_pool_index, u32 *p_threshold); + int (*sb_tc_pool_bind_set)(struct devlink_port *devlink_port, + unsigned int sb_index, + u16 tc_index, + enum devlink_sb_pool_type pool_type, + u16 pool_index, u32 threshold); + int (*sb_occ_snapshot)(struct devlink *devlink, + unsigned int sb_index); + int (*sb_occ_max_clear)(struct devlink *devlink, + unsigned int sb_index); + int (*sb_occ_port_pool_get)(struct devlink_port *devlink_port, + unsigned int sb_index, u16 pool_index, + u32 *p_cur, u32 *p_max); + int (*sb_occ_tc_port_bind_get)(struct devlink_port *devlink_port, + unsigned int sb_index, + u16 tc_index, + enum devlink_sb_pool_type pool_type, + u32 *p_cur, u32 *p_max); }; static inline void *devlink_priv(struct devlink *devlink) @@ -82,6 +123,11 @@ void devlink_port_type_ib_set(struct devlink_port *devlink_port, void devlink_port_type_clear(struct devlink_port *devlink_port); void devlink_port_split_set(struct devlink_port *devlink_port, u32 split_group); +int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, + u32 size, u16 ingress_pools_count, + u16 egress_pools_count, u16 ingress_tc_count, + u16 egress_tc_count); +void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index); #else @@ -135,6 +181,21 @@ static inline void devlink_port_split_set(struct devlink_port *devlink_port, { } +static inline int devlink_sb_register(struct devlink *devlink, + unsigned int sb_index, u32 size, + u16 ingress_pools_count, + u16 egress_pools_count, + u16 ingress_tc_count, + u16 egress_tc_count) +{ + return 0; +} + +static inline void devlink_sb_unregister(struct devlink *devlink, + unsigned int sb_index) +{ +} + #endif #endif /* _NET_DEVLINK_H_ */ diff --git a/include/net/dsa.h b/include/net/dsa.h index 18d1be3ad62d..17c3d37b6779 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -16,7 +16,6 @@ #include <linux/timer.h> #include <linux/workqueue.h> #include <linux/of.h> -#include <linux/of_gpio.h> #include <linux/phy.h> #include <linux/phy_fixed.h> #include <linux/ethtool.h> @@ -65,13 +64,6 @@ struct dsa_chip_data { * NULL if there is only one switch chip. */ s8 *rtable; - - /* - * A switch may have a GPIO line tied to its reset pin. Parse - * this from the device tree, and use it before performing - * switch soft reset. - */ - struct gpio_desc *reset; }; struct dsa_platform_data { @@ -111,6 +103,11 @@ struct dsa_switch_tree { enum dsa_tag_protocol tag_protocol; /* + * Original copy of the master netdev ethtool_ops + */ + struct ethtool_ops master_ethtool_ops; + + /* * The switch and port to which the CPU is attached. */ s8 cpu_switch; @@ -123,6 +120,8 @@ struct dsa_switch_tree { }; struct dsa_switch { + struct device *dev; + /* * Parent switch tree, and switch index. */ @@ -130,25 +129,21 @@ struct dsa_switch { int index; /* - * Tagging protocol understood by this switch + * Give the switch driver somewhere to hang its private data + * structure. */ - enum dsa_tag_protocol tag_protocol; + void *priv; /* * Configuration data for this switch. */ - struct dsa_chip_data *pd; + struct dsa_chip_data *cd; /* * The used switch driver. */ struct dsa_switch_driver *drv; - /* - * Reference to host device to use. - */ - struct device *master_dev; - #ifdef CONFIG_NET_DSA_HWMON /* * Hardware monitoring information @@ -161,7 +156,7 @@ struct dsa_switch { * Slave mii_bus and devices for the individual ports. */ u32 dsa_port_mask; - u32 phys_port_mask; + u32 enabled_port_mask; u32 phys_mii_mask; struct mii_bus *slave_mii_bus; struct net_device *ports[DSA_MAX_PORTS]; @@ -179,7 +174,7 @@ static inline bool dsa_is_dsa_port(struct dsa_switch *ds, int p) static inline bool dsa_is_port_initialized(struct dsa_switch *ds, int p) { - return ds->phys_port_mask & (1 << p) && ds->ports[p]; + return ds->enabled_port_mask & (1 << p) && ds->ports[p]; } static inline u8 dsa_upstream_port(struct dsa_switch *ds) @@ -195,7 +190,7 @@ static inline u8 dsa_upstream_port(struct dsa_switch *ds) if (dst->cpu_switch == ds->index) return dst->cpu_port; else - return ds->pd->rtable[dst->cpu_switch]; + return ds->cd->rtable[dst->cpu_switch]; } struct switchdev_trans; @@ -207,12 +202,13 @@ struct dsa_switch_driver { struct list_head list; enum dsa_tag_protocol tag_protocol; - int priv_size; /* * Probing and setup. */ - char *(*probe)(struct device *host_dev, int sw_addr); + const char *(*probe)(struct device *dsa_dev, + struct device *host_dev, int sw_addr, + void **priv); int (*setup)(struct dsa_switch *ds); int (*set_addr)(struct dsa_switch *ds, u8 *addr); u32 (*get_phy_flags)(struct dsa_switch *ds, int port); @@ -341,7 +337,7 @@ struct mii_bus *dsa_host_dev_to_mii_bus(struct device *dev); static inline void *ds_to_priv(struct dsa_switch *ds) { - return (void *)(ds + 1); + return ds->priv; } static inline bool dsa_uses_tagged_protocol(struct dsa_switch_tree *dst) diff --git a/include/net/dst.h b/include/net/dst.h index 5c98443c1c9e..6835d224d47b 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -85,12 +85,11 @@ struct dst_entry { #endif #ifdef CONFIG_64BIT - struct lwtunnel_state *lwtstate; /* * Align __refcnt to a 64 bytes alignment * (L1_CACHE_SIZE would be too much) */ - long __pad_to_align_refcnt[1]; + long __pad_to_align_refcnt[2]; #endif /* * __refcnt wants to be on a different cache line from @@ -99,9 +98,7 @@ struct dst_entry { atomic_t __refcnt; /* client references */ int __use; unsigned long lastuse; -#ifndef CONFIG_64BIT struct lwtunnel_state *lwtstate; -#endif union { struct dst_entry *next; struct rtable __rcu *rt_next; diff --git a/include/net/fq.h b/include/net/fq.h new file mode 100644 index 000000000000..268b49049c37 --- /dev/null +++ b/include/net/fq.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016 Qualcomm Atheros, Inc + * + * GPL v2 + * + * Based on net/sched/sch_fq_codel.c + */ +#ifndef __NET_SCHED_FQ_H +#define __NET_SCHED_FQ_H + +struct fq_tin; + +/** + * struct fq_flow - per traffic flow queue + * + * @tin: owner of this flow. Used to manage collisions, i.e. when a packet + * hashes to an index which points to a flow that is already owned by a + * different tin the packet is destined to. In such case the implementer + * must provide a fallback flow + * @flowchain: can be linked to fq_tin's new_flows or old_flows. Used for DRR++ + * (deficit round robin) based round robin queuing similar to the one + * found in net/sched/sch_fq_codel.c + * @backlogchain: can be linked to other fq_flow and fq. Used to keep track of + * fat flows and efficient head-dropping if packet limit is reached + * @queue: sk_buff queue to hold packets + * @backlog: number of bytes pending in the queue. The number of packets can be + * found in @queue.qlen + * @deficit: used for DRR++ + */ +struct fq_flow { + struct fq_tin *tin; + struct list_head flowchain; + struct list_head backlogchain; + struct sk_buff_head queue; + u32 backlog; + int deficit; +}; + +/** + * struct fq_tin - a logical container of fq_flows + * + * Used to group fq_flows into a logical aggregate. DRR++ scheme is used to + * pull interleaved packets out of the associated flows. + * + * @new_flows: linked list of fq_flow + * @old_flows: linked list of fq_flow + */ +struct fq_tin { + struct list_head new_flows; + struct list_head old_flows; + u32 backlog_bytes; + u32 backlog_packets; + u32 overlimit; + u32 collisions; + u32 flows; + u32 tx_bytes; + u32 tx_packets; +}; + +/** + * struct fq - main container for fair queuing purposes + * + * @backlogs: linked to fq_flows. Used to maintain fat flows for efficient + * head-dropping when @backlog reaches @limit + * @limit: max number of packets that can be queued across all flows + * @backlog: number of packets queued across all flows + */ +struct fq { + struct fq_flow *flows; + struct list_head backlogs; + spinlock_t lock; + u32 flows_cnt; + u32 perturbation; + u32 limit; + u32 quantum; + u32 backlog; + u32 overlimit; + u32 collisions; +}; + +typedef struct sk_buff *fq_tin_dequeue_t(struct fq *, + struct fq_tin *, + struct fq_flow *flow); + +typedef void fq_skb_free_t(struct fq *, + struct fq_tin *, + struct fq_flow *, + struct sk_buff *); + +typedef struct fq_flow *fq_flow_get_default_t(struct fq *, + struct fq_tin *, + int idx, + struct sk_buff *); + +#endif diff --git a/include/net/fq_impl.h b/include/net/fq_impl.h new file mode 100644 index 000000000000..163f3ed0f05a --- /dev/null +++ b/include/net/fq_impl.h @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2016 Qualcomm Atheros, Inc + * + * GPL v2 + * + * Based on net/sched/sch_fq_codel.c + */ +#ifndef __NET_SCHED_FQ_IMPL_H +#define __NET_SCHED_FQ_IMPL_H + +#include <net/fq.h> + +/* functions that are embedded into includer */ + +static struct sk_buff *fq_flow_dequeue(struct fq *fq, + struct fq_flow *flow) +{ + struct fq_tin *tin = flow->tin; + struct fq_flow *i; + struct sk_buff *skb; + + lockdep_assert_held(&fq->lock); + + skb = __skb_dequeue(&flow->queue); + if (!skb) + return NULL; + + tin->backlog_bytes -= skb->len; + tin->backlog_packets--; + flow->backlog -= skb->len; + fq->backlog--; + + if (flow->backlog == 0) { + list_del_init(&flow->backlogchain); + } else { + i = flow; + + list_for_each_entry_continue(i, &fq->backlogs, backlogchain) + if (i->backlog < flow->backlog) + break; + + list_move_tail(&flow->backlogchain, + &i->backlogchain); + } + + return skb; +} + +static struct sk_buff *fq_tin_dequeue(struct fq *fq, + struct fq_tin *tin, + fq_tin_dequeue_t dequeue_func) +{ + struct fq_flow *flow; + struct list_head *head; + struct sk_buff *skb; + + lockdep_assert_held(&fq->lock); + +begin: + head = &tin->new_flows; + if (list_empty(head)) { + head = &tin->old_flows; + if (list_empty(head)) + return NULL; + } + + flow = list_first_entry(head, struct fq_flow, flowchain); + + if (flow->deficit <= 0) { + flow->deficit += fq->quantum; + list_move_tail(&flow->flowchain, + &tin->old_flows); + goto begin; + } + + skb = dequeue_func(fq, tin, flow); + if (!skb) { + /* force a pass through old_flows to prevent starvation */ + if ((head == &tin->new_flows) && + !list_empty(&tin->old_flows)) { + list_move_tail(&flow->flowchain, &tin->old_flows); + } else { + list_del_init(&flow->flowchain); + flow->tin = NULL; + } + goto begin; + } + + flow->deficit -= skb->len; + tin->tx_bytes += skb->len; + tin->tx_packets++; + + return skb; +} + +static struct fq_flow *fq_flow_classify(struct fq *fq, + struct fq_tin *tin, + struct sk_buff *skb, + fq_flow_get_default_t get_default_func) +{ + struct fq_flow *flow; + u32 hash; + u32 idx; + + lockdep_assert_held(&fq->lock); + + hash = skb_get_hash_perturb(skb, fq->perturbation); + idx = reciprocal_scale(hash, fq->flows_cnt); + flow = &fq->flows[idx]; + + if (flow->tin && flow->tin != tin) { + flow = get_default_func(fq, tin, idx, skb); + tin->collisions++; + fq->collisions++; + } + + if (!flow->tin) + tin->flows++; + + return flow; +} + +static void fq_recalc_backlog(struct fq *fq, + struct fq_tin *tin, + struct fq_flow *flow) +{ + struct fq_flow *i; + + if (list_empty(&flow->backlogchain)) + list_add_tail(&flow->backlogchain, &fq->backlogs); + + i = flow; + list_for_each_entry_continue_reverse(i, &fq->backlogs, + backlogchain) + if (i->backlog > flow->backlog) + break; + + list_move(&flow->backlogchain, &i->backlogchain); +} + +static void fq_tin_enqueue(struct fq *fq, + struct fq_tin *tin, + struct sk_buff *skb, + fq_skb_free_t free_func, + fq_flow_get_default_t get_default_func) +{ + struct fq_flow *flow; + + lockdep_assert_held(&fq->lock); + + flow = fq_flow_classify(fq, tin, skb, get_default_func); + + flow->tin = tin; + flow->backlog += skb->len; + tin->backlog_bytes += skb->len; + tin->backlog_packets++; + fq->backlog++; + + fq_recalc_backlog(fq, tin, flow); + + if (list_empty(&flow->flowchain)) { + flow->deficit = fq->quantum; + list_add_tail(&flow->flowchain, + &tin->new_flows); + } + + __skb_queue_tail(&flow->queue, skb); + + if (fq->backlog > fq->limit) { + flow = list_first_entry_or_null(&fq->backlogs, + struct fq_flow, + backlogchain); + if (!flow) + return; + + skb = fq_flow_dequeue(fq, flow); + if (!skb) + return; + + free_func(fq, flow->tin, flow, skb); + + flow->tin->overlimit++; + fq->overlimit++; + } +} + +static void fq_flow_reset(struct fq *fq, + struct fq_flow *flow, + fq_skb_free_t free_func) +{ + struct sk_buff *skb; + + while ((skb = fq_flow_dequeue(fq, flow))) + free_func(fq, flow->tin, flow, skb); + + if (!list_empty(&flow->flowchain)) + list_del_init(&flow->flowchain); + + if (!list_empty(&flow->backlogchain)) + list_del_init(&flow->backlogchain); + + flow->tin = NULL; + + WARN_ON_ONCE(flow->backlog); +} + +static void fq_tin_reset(struct fq *fq, + struct fq_tin *tin, + fq_skb_free_t free_func) +{ + struct list_head *head; + struct fq_flow *flow; + + for (;;) { + head = &tin->new_flows; + if (list_empty(head)) { + head = &tin->old_flows; + if (list_empty(head)) + break; + } + + flow = list_first_entry(head, struct fq_flow, flowchain); + fq_flow_reset(fq, flow, free_func); + } + + WARN_ON_ONCE(tin->backlog_bytes); + WARN_ON_ONCE(tin->backlog_packets); +} + +static void fq_flow_init(struct fq_flow *flow) +{ + INIT_LIST_HEAD(&flow->flowchain); + INIT_LIST_HEAD(&flow->backlogchain); + __skb_queue_head_init(&flow->queue); +} + +static void fq_tin_init(struct fq_tin *tin) +{ + INIT_LIST_HEAD(&tin->new_flows); + INIT_LIST_HEAD(&tin->old_flows); +} + +static int fq_init(struct fq *fq, int flows_cnt) +{ + int i; + + memset(fq, 0, sizeof(fq[0])); + INIT_LIST_HEAD(&fq->backlogs); + spin_lock_init(&fq->lock); + fq->flows_cnt = max_t(u32, flows_cnt, 1); + fq->perturbation = prandom_u32(); + fq->quantum = 300; + fq->limit = 8192; + + fq->flows = kcalloc(fq->flows_cnt, sizeof(fq->flows[0]), GFP_KERNEL); + if (!fq->flows) + return -ENOMEM; + + for (i = 0; i < fq->flows_cnt; i++) + fq_flow_init(&fq->flows[i]); + + return 0; +} + +static void fq_reset(struct fq *fq, + fq_skb_free_t free_func) +{ + int i; + + for (i = 0; i < fq->flows_cnt; i++) + fq_flow_reset(fq, &fq->flows[i], free_func); + + kfree(fq->flows); + fq->flows = NULL; +} + +#endif diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h index cbafa3768d48..610cd397890e 100644 --- a/include/net/gen_stats.h +++ b/include/net/gen_stats.h @@ -19,17 +19,19 @@ struct gnet_dump { /* Backward compatibility */ int compat_tc_stats; int compat_xstats; + int padattr; void * xstats; int xstats_len; struct tc_stats tc_stats; }; int gnet_stats_start_copy(struct sk_buff *skb, int type, spinlock_t *lock, - struct gnet_dump *d); + struct gnet_dump *d, int padattr); int gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int tc_stats_type, int xstats_type, - spinlock_t *lock, struct gnet_dump *d); + spinlock_t *lock, struct gnet_dump *d, + int padattr); int gnet_stats_copy_basic(struct gnet_dump *d, struct gnet_stats_basic_cpu __percpu *cpu, diff --git a/include/net/geneve.h b/include/net/geneve.h index e6c23dc765f7..cb544a530146 100644 --- a/include/net/geneve.h +++ b/include/net/geneve.h @@ -62,13 +62,11 @@ struct genevehdr { struct geneve_opt options[]; }; -#if IS_ENABLED(CONFIG_GENEVE) -void geneve_get_rx_port(struct net_device *netdev); -#else static inline void geneve_get_rx_port(struct net_device *netdev) { + ASSERT_RTNL(); + call_netdevice_notifiers(NETDEV_OFFLOAD_PUSH_GENEVE, netdev); } -#endif #ifdef CONFIG_INET struct net_device *geneve_dev_create_fb(struct net *net, const char *name, diff --git a/include/net/gre.h b/include/net/gre.h index 97eafdc47eea..a14093c70eab 100644 --- a/include/net/gre.h +++ b/include/net/gre.h @@ -25,4 +25,108 @@ int gre_del_protocol(const struct gre_protocol *proto, u8 version); struct net_device *gretap_fb_dev_create(struct net *net, const char *name, u8 name_assign_type); +int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi, + bool *csum_err); + +static inline int gre_calc_hlen(__be16 o_flags) +{ + int addend = 4; + + if (o_flags & TUNNEL_CSUM) + addend += 4; + if (o_flags & TUNNEL_KEY) + addend += 4; + if (o_flags & TUNNEL_SEQ) + addend += 4; + return addend; +} + +static inline __be16 gre_flags_to_tnl_flags(__be16 flags) +{ + __be16 tflags = 0; + + if (flags & GRE_CSUM) + tflags |= TUNNEL_CSUM; + if (flags & GRE_ROUTING) + tflags |= TUNNEL_ROUTING; + if (flags & GRE_KEY) + tflags |= TUNNEL_KEY; + if (flags & GRE_SEQ) + tflags |= TUNNEL_SEQ; + if (flags & GRE_STRICT) + tflags |= TUNNEL_STRICT; + if (flags & GRE_REC) + tflags |= TUNNEL_REC; + if (flags & GRE_VERSION) + tflags |= TUNNEL_VERSION; + + return tflags; +} + +static inline __be16 gre_tnl_flags_to_gre_flags(__be16 tflags) +{ + __be16 flags = 0; + + if (tflags & TUNNEL_CSUM) + flags |= GRE_CSUM; + if (tflags & TUNNEL_ROUTING) + flags |= GRE_ROUTING; + if (tflags & TUNNEL_KEY) + flags |= GRE_KEY; + if (tflags & TUNNEL_SEQ) + flags |= GRE_SEQ; + if (tflags & TUNNEL_STRICT) + flags |= GRE_STRICT; + if (tflags & TUNNEL_REC) + flags |= GRE_REC; + if (tflags & TUNNEL_VERSION) + flags |= GRE_VERSION; + + return flags; +} + +static inline __sum16 gre_checksum(struct sk_buff *skb) +{ + __wsum csum; + + if (skb->ip_summed == CHECKSUM_PARTIAL) + csum = lco_csum(skb); + else + csum = skb_checksum(skb, 0, skb->len, 0); + return csum_fold(csum); +} + +static inline void gre_build_header(struct sk_buff *skb, int hdr_len, + __be16 flags, __be16 proto, + __be32 key, __be32 seq) +{ + struct gre_base_hdr *greh; + + skb_push(skb, hdr_len); + + skb_reset_transport_header(skb); + greh = (struct gre_base_hdr *)skb->data; + greh->flags = gre_tnl_flags_to_gre_flags(flags); + greh->protocol = proto; + + if (flags & (TUNNEL_KEY | TUNNEL_CSUM | TUNNEL_SEQ)) { + __be32 *ptr = (__be32 *)(((u8 *)greh) + hdr_len - 4); + + if (flags & TUNNEL_SEQ) { + *ptr = seq; + ptr--; + } + if (flags & TUNNEL_KEY) { + *ptr = key; + ptr--; + } + if (flags & TUNNEL_CSUM && + !(skb_shinfo(skb)->gso_type & + (SKB_GSO_GRE | SKB_GSO_GRE_CSUM))) { + *ptr = 0; + *(__sum16 *)ptr = gre_checksum(skb); + } + } +} + #endif diff --git a/include/net/gtp.h b/include/net/gtp.h new file mode 100644 index 000000000000..894a37b87d63 --- /dev/null +++ b/include/net/gtp.h @@ -0,0 +1,34 @@ +#ifndef _GTP_H_ +#define _GTP_H + +/* General GTP protocol related definitions. */ + +#define GTP0_PORT 3386 +#define GTP1U_PORT 2152 + +#define GTP_TPDU 255 + +struct gtp0_header { /* According to GSM TS 09.60. */ + __u8 flags; + __u8 type; + __be16 length; + __be16 seq; + __be16 flow; + __u8 number; + __u8 spare[3]; + __be64 tid; +} __attribute__ ((packed)); + +struct gtp1_header { /* According to 3GPP TS 29.060. */ + __u8 flags; + __u8 type; + __be16 length; + __be32 tid; +} __attribute__ ((packed)); + +#define GTP1_F_NPDU 0x01 +#define GTP1_F_SEQ 0x02 +#define GTP1_F_EXTHDR 0x04 +#define GTP1_F_MASK 0x07 + +#endif diff --git a/include/net/icmp.h b/include/net/icmp.h index 970028e13382..3ef2743a8eec 100644 --- a/include/net/icmp.h +++ b/include/net/icmp.h @@ -30,9 +30,9 @@ struct icmp_err { extern const struct icmp_err icmp_err_convert[]; #define ICMP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.icmp_statistics, field) -#define ICMP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.icmp_statistics, field) +#define __ICMP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.icmp_statistics, field) #define ICMPMSGOUT_INC_STATS(net, field) SNMP_INC_STATS_ATOMIC_LONG((net)->mib.icmpmsg_statistics, field+256) -#define ICMPMSGIN_INC_STATS_BH(net, field) SNMP_INC_STATS_ATOMIC_LONG((net)->mib.icmpmsg_statistics, field) +#define ICMPMSGIN_INC_STATS(net, field) SNMP_INC_STATS_ATOMIC_LONG((net)->mib.icmpmsg_statistics, field) struct dst_entry; struct net_proto_family; diff --git a/include/net/ip.h b/include/net/ip.h index 93725e546758..37165fba3741 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -36,6 +36,7 @@ struct sock; struct inet_skb_parm { + int iif; struct ip_options opt; /* Compiled IP options */ unsigned char flags; @@ -187,17 +188,15 @@ void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb, unsigned int len); #define IP_INC_STATS(net, field) SNMP_INC_STATS64((net)->mib.ip_statistics, field) -#define IP_INC_STATS_BH(net, field) SNMP_INC_STATS64_BH((net)->mib.ip_statistics, field) +#define __IP_INC_STATS(net, field) __SNMP_INC_STATS64((net)->mib.ip_statistics, field) #define IP_ADD_STATS(net, field, val) SNMP_ADD_STATS64((net)->mib.ip_statistics, field, val) -#define IP_ADD_STATS_BH(net, field, val) SNMP_ADD_STATS64_BH((net)->mib.ip_statistics, field, val) +#define __IP_ADD_STATS(net, field, val) __SNMP_ADD_STATS64((net)->mib.ip_statistics, field, val) #define IP_UPD_PO_STATS(net, field, val) SNMP_UPD_PO_STATS64((net)->mib.ip_statistics, field, val) -#define IP_UPD_PO_STATS_BH(net, field, val) SNMP_UPD_PO_STATS64_BH((net)->mib.ip_statistics, field, val) +#define __IP_UPD_PO_STATS(net, field, val) __SNMP_UPD_PO_STATS64((net)->mib.ip_statistics, field, val) #define NET_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.net_statistics, field) -#define NET_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.net_statistics, field) -#define NET_INC_STATS_USER(net, field) SNMP_INC_STATS_USER((net)->mib.net_statistics, field) +#define __NET_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.net_statistics, field) #define NET_ADD_STATS(net, field, adnd) SNMP_ADD_STATS((net)->mib.net_statistics, field, adnd) -#define NET_ADD_STATS_BH(net, field, adnd) SNMP_ADD_STATS_BH((net)->mib.net_statistics, field, adnd) -#define NET_ADD_STATS_USER(net, field, adnd) SNMP_ADD_STATS_USER((net)->mib.net_statistics, field, adnd) +#define __NET_ADD_STATS(net, field, adnd) __SNMP_ADD_STATS((net)->mib.net_statistics, field, adnd) u64 snmp_get_cpu_field(void __percpu *mib, int cpu, int offct); unsigned long snmp_fold_field(void __percpu *mib, int offt); diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 295d291269e2..54c779416eec 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -101,6 +101,9 @@ void fib6_force_start_gc(struct net *net); struct rt6_info *addrconf_dst_alloc(struct inet6_dev *idev, const struct in6_addr *addr, bool anycast); +struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, + int flags); + /* * support functions for ND * diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 499a707765ea..fb9e0153f4f2 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -42,6 +42,7 @@ struct ip6_tnl { struct __ip6_tnl_parm parms; /* tunnel configuration parameters */ struct flowi fl; /* flowi template for xmit */ struct dst_cache dst_cache; /* cached dst */ + struct gro_cells gro_cells; int err_count; unsigned long err_time; @@ -49,8 +50,10 @@ struct ip6_tnl { /* These fields used only by GRE */ __u32 i_seqno; /* The last seen seqno */ __u32 o_seqno; /* The last output seqno */ - int hlen; /* Precalculated GRE header length */ + int hlen; /* tun_hlen + encap_hlen */ + int tun_hlen; /* Precalculated header length */ int mlink; + }; /* Tunnel encapsulation limit destination sub-option */ @@ -63,13 +66,19 @@ struct ipv6_tlv_tnl_enc_lim { int ip6_tnl_rcv_ctl(struct ip6_tnl *t, const struct in6_addr *laddr, const struct in6_addr *raddr); +int ip6_tnl_rcv(struct ip6_tnl *tunnel, struct sk_buff *skb, + const struct tnl_ptk_info *tpi, struct metadata_dst *tun_dst, + bool log_ecn_error); int ip6_tnl_xmit_ctl(struct ip6_tnl *t, const struct in6_addr *laddr, const struct in6_addr *raddr); +int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, + struct flowi6 *fl6, int encap_limit, __u32 *pmtu, __u8 proto); __u16 ip6_tnl_parse_tlv_enc_lim(struct sk_buff *skb, __u8 *raw); __u32 ip6_tnl_get_cap(struct ip6_tnl *t, const struct in6_addr *laddr, const struct in6_addr *raddr); struct net *ip6_tnl_get_link_net(const struct net_device *dev); int ip6_tnl_get_iflink(const struct net_device *dev); +int ip6_tnl_change_mtu(struct net_device *dev, int new_mtu); #ifdef CONFIG_INET static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 16435d8b1f93..d916b4315903 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -105,24 +105,23 @@ struct ip_tunnel { struct net_device *dev; struct net *net; /* netns for packet i/o */ - int err_count; /* Number of arrived ICMP errors */ unsigned long err_time; /* Time when the last ICMP error * arrived */ + int err_count; /* Number of arrived ICMP errors */ /* These four fields used only by GRE */ u32 i_seqno; /* The last seen seqno */ u32 o_seqno; /* The last output seqno */ int tun_hlen; /* Precalculated header length */ - int mlink; struct dst_cache dst_cache; struct ip_tunnel_parm parms; + int mlink; int encap_hlen; /* Encap header length (FOU,GUE) */ - struct ip_tunnel_encap encap; - int hlen; /* tun_hlen + encap_hlen */ + struct ip_tunnel_encap encap; /* for SIT */ #ifdef CONFIG_IPV6_SIT_6RD @@ -161,6 +160,7 @@ struct tnl_ptk_info { #define PACKET_RCVD 0 #define PACKET_REJECT 1 +#define PACKET_NEXT 2 #define IP_TNL_HASH_BITS 7 #define IP_TNL_HASH_SIZE (1 << IP_TNL_HASH_BITS) @@ -310,7 +310,7 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md, gfp_t flags); -struct sk_buff *iptunnel_handle_offloads(struct sk_buff *skb, int gso_type_mask); +int iptunnel_handle_offloads(struct sk_buff *skb, int gso_type_mask); static inline int iptunnel_pull_offloads(struct sk_buff *skb) { diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index a6cc576fd467..af4c10ebb241 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -731,6 +731,12 @@ struct ip_vs_pe { u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, u32 initval, bool inverse); int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); + /* create connections for real-server outgoing packets */ + struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, + struct ip_vs_dest *dest, + struct sk_buff *skb, + const struct ip_vs_iphdr *iph, + __be16 dport, __be16 cport); }; /* The application module object (a.k.a. app incarnation) */ @@ -874,6 +880,7 @@ struct netns_ipvs { /* Service counters */ atomic_t ftpsvc_counter; atomic_t nullsvc_counter; + atomic_t conn_out_counter; #ifdef CONFIG_SYSCTL /* 1/rate drop and drop-entry variables */ @@ -1147,6 +1154,12 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) */ const char *ip_vs_proto_name(unsigned int proto); void ip_vs_init_hash_table(struct list_head *table, int rows); +struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, + struct ip_vs_dest *dest, + struct sk_buff *skb, + const struct ip_vs_iphdr *iph, + __be16 dport, + __be16 cport); #define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t))) #define IP_VS_APP_TYPE_FTP 1 @@ -1378,6 +1391,10 @@ ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, const union nf_inet_addr *daddr, __be16 dport); +struct ip_vs_dest * +ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, + const union nf_inet_addr *daddr, __be16 dport); + int ip_vs_use_count_inc(void); void ip_vs_use_count_dec(void); int ip_vs_register_nl_ioctl(void); diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 55ee1eb7d026..11a045281948 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -121,21 +121,21 @@ struct frag_hdr { extern int sysctl_mld_max_msf; extern int sysctl_mld_qrv; -#define _DEVINC(net, statname, modifier, idev, field) \ +#define _DEVINC(net, statname, mod, idev, field) \ ({ \ struct inet6_dev *_idev = (idev); \ if (likely(_idev != NULL)) \ - SNMP_INC_STATS##modifier((_idev)->stats.statname, (field)); \ - SNMP_INC_STATS##modifier((net)->mib.statname##_statistics, (field));\ + mod##SNMP_INC_STATS64((_idev)->stats.statname, (field));\ + mod##SNMP_INC_STATS64((net)->mib.statname##_statistics, (field));\ }) /* per device counters are atomic_long_t */ -#define _DEVINCATOMIC(net, statname, modifier, idev, field) \ +#define _DEVINCATOMIC(net, statname, mod, idev, field) \ ({ \ struct inet6_dev *_idev = (idev); \ if (likely(_idev != NULL)) \ SNMP_INC_STATS_ATOMIC_LONG((_idev)->stats.statname##dev, (field)); \ - SNMP_INC_STATS##modifier((net)->mib.statname##_statistics, (field));\ + mod##SNMP_INC_STATS((net)->mib.statname##_statistics, (field));\ }) /* per device and per net counters are atomic_long_t */ @@ -147,46 +147,44 @@ extern int sysctl_mld_qrv; SNMP_INC_STATS_ATOMIC_LONG((net)->mib.statname##_statistics, (field));\ }) -#define _DEVADD(net, statname, modifier, idev, field, val) \ +#define _DEVADD(net, statname, mod, idev, field, val) \ ({ \ struct inet6_dev *_idev = (idev); \ if (likely(_idev != NULL)) \ - SNMP_ADD_STATS##modifier((_idev)->stats.statname, (field), (val)); \ - SNMP_ADD_STATS##modifier((net)->mib.statname##_statistics, (field), (val));\ + mod##SNMP_ADD_STATS((_idev)->stats.statname, (field), (val)); \ + mod##SNMP_ADD_STATS((net)->mib.statname##_statistics, (field), (val));\ }) -#define _DEVUPD(net, statname, modifier, idev, field, val) \ +#define _DEVUPD(net, statname, mod, idev, field, val) \ ({ \ struct inet6_dev *_idev = (idev); \ if (likely(_idev != NULL)) \ - SNMP_UPD_PO_STATS##modifier((_idev)->stats.statname, field, (val)); \ - SNMP_UPD_PO_STATS##modifier((net)->mib.statname##_statistics, field, (val));\ + mod##SNMP_UPD_PO_STATS((_idev)->stats.statname, field, (val)); \ + mod##SNMP_UPD_PO_STATS((net)->mib.statname##_statistics, field, (val));\ }) /* MIBs */ #define IP6_INC_STATS(net, idev,field) \ - _DEVINC(net, ipv6, 64, idev, field) -#define IP6_INC_STATS_BH(net, idev,field) \ - _DEVINC(net, ipv6, 64_BH, idev, field) + _DEVINC(net, ipv6, , idev, field) +#define __IP6_INC_STATS(net, idev,field) \ + _DEVINC(net, ipv6, __, idev, field) #define IP6_ADD_STATS(net, idev,field,val) \ - _DEVADD(net, ipv6, 64, idev, field, val) -#define IP6_ADD_STATS_BH(net, idev,field,val) \ - _DEVADD(net, ipv6, 64_BH, idev, field, val) + _DEVADD(net, ipv6, , idev, field, val) +#define __IP6_ADD_STATS(net, idev,field,val) \ + _DEVADD(net, ipv6, __, idev, field, val) #define IP6_UPD_PO_STATS(net, idev,field,val) \ - _DEVUPD(net, ipv6, 64, idev, field, val) -#define IP6_UPD_PO_STATS_BH(net, idev,field,val) \ - _DEVUPD(net, ipv6, 64_BH, idev, field, val) + _DEVUPD(net, ipv6, , idev, field, val) +#define __IP6_UPD_PO_STATS(net, idev,field,val) \ + _DEVUPD(net, ipv6, __, idev, field, val) #define ICMP6_INC_STATS(net, idev, field) \ _DEVINCATOMIC(net, icmpv6, , idev, field) -#define ICMP6_INC_STATS_BH(net, idev, field) \ - _DEVINCATOMIC(net, icmpv6, _BH, idev, field) +#define __ICMP6_INC_STATS(net, idev, field) \ + _DEVINCATOMIC(net, icmpv6, __, idev, field) #define ICMP6MSGOUT_INC_STATS(net, idev, field) \ _DEVINC_ATOMIC_ATOMIC(net, icmpv6msg, idev, field +256) -#define ICMP6MSGOUT_INC_STATS_BH(net, idev, field) \ - _DEVINC_ATOMIC_ATOMIC(net, icmpv6msg, idev, field +256) -#define ICMP6MSGIN_INC_STATS_BH(net, idev, field) \ +#define ICMP6MSGIN_INC_STATS(net, idev, field) \ _DEVINC_ATOMIC_ATOMIC(net, icmpv6msg, idev, field) struct ip6_ra_chain { @@ -253,6 +251,13 @@ struct ipv6_fl_socklist { struct rcu_head rcu; }; +struct ipcm6_cookie { + __s16 hlimit; + __s16 tclass; + __s8 dontfrag; + struct ipv6_txoptions *opt; +}; + static inline struct ipv6_txoptions *txopt_get(const struct ipv6_pinfo *np) { struct ipv6_txoptions *opt; @@ -865,9 +870,9 @@ int ip6_find_1stfragopt(struct sk_buff *skb, u8 **nexthdr); int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb), - void *from, int length, int transhdrlen, int hlimit, - int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6, - struct rt6_info *rt, unsigned int flags, int dontfrag, + void *from, int length, int transhdrlen, + struct ipcm6_cookie *ipc6, struct flowi6 *fl6, + struct rt6_info *rt, unsigned int flags, const struct sockcm_cookie *sockc); int ip6_push_pending_frames(struct sock *sk); @@ -883,9 +888,8 @@ struct sk_buff *ip6_make_skb(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb), void *from, int length, int transhdrlen, - int hlimit, int tclass, struct ipv6_txoptions *opt, - struct flowi6 *fl6, struct rt6_info *rt, - unsigned int flags, int dontfrag, + struct ipcm6_cookie *ipc6, struct flowi6 *fl6, + struct rt6_info *rt, unsigned int flags, const struct sockcm_cookie *sockc); static inline struct sk_buff *ip6_finish_skb(struct sock *sk) @@ -961,6 +965,8 @@ int compat_ipv6_getsockopt(struct sock *sk, int level, int optname, int ip6_datagram_connect(struct sock *sk, struct sockaddr *addr, int addr_len); int ip6_datagram_connect_v6_only(struct sock *sk, struct sockaddr *addr, int addr_len); +int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr); +void ip6_datagram_release_cb(struct sock *sk); int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len); diff --git a/include/net/l3mdev.h b/include/net/l3mdev.h index c43a9c73de5e..374388dc01c8 100644 --- a/include/net/l3mdev.h +++ b/include/net/l3mdev.h @@ -25,6 +25,8 @@ struct l3mdev_ops { u32 (*l3mdev_fib_table)(const struct net_device *dev); + struct sk_buff * (*l3mdev_l3_rcv)(struct net_device *dev, + struct sk_buff *skb, u16 proto); /* IPv4 ops */ struct rtable * (*l3mdev_get_rtable)(const struct net_device *dev, @@ -130,51 +132,36 @@ static inline bool netif_index_is_l3_master(struct net *net, int ifindex) return rc; } -static inline int l3mdev_get_saddr(struct net *net, int ifindex, - struct flowi4 *fl4) -{ - struct net_device *dev; - int rc = 0; +int l3mdev_get_saddr(struct net *net, int ifindex, struct flowi4 *fl4); - if (ifindex) { +struct dst_entry *l3mdev_get_rt6_dst(struct net *net, const struct flowi6 *fl6); - rcu_read_lock(); +static inline +struct sk_buff *l3mdev_l3_rcv(struct sk_buff *skb, u16 proto) +{ + struct net_device *master = NULL; - dev = dev_get_by_index_rcu(net, ifindex); - if (dev && netif_is_l3_master(dev) && - dev->l3mdev_ops->l3mdev_get_saddr) { - rc = dev->l3mdev_ops->l3mdev_get_saddr(dev, fl4); - } + if (netif_is_l3_slave(skb->dev)) + master = netdev_master_upper_dev_get_rcu(skb->dev); + else if (netif_is_l3_master(skb->dev)) + master = skb->dev; - rcu_read_unlock(); - } + if (master && master->l3mdev_ops->l3mdev_l3_rcv) + skb = master->l3mdev_ops->l3mdev_l3_rcv(master, skb, proto); - return rc; + return skb; } -static inline struct dst_entry *l3mdev_get_rt6_dst(const struct net_device *dev, - const struct flowi6 *fl6) +static inline +struct sk_buff *l3mdev_ip_rcv(struct sk_buff *skb) { - if (netif_is_l3_master(dev) && dev->l3mdev_ops->l3mdev_get_rt6_dst) - return dev->l3mdev_ops->l3mdev_get_rt6_dst(dev, fl6); - - return NULL; + return l3mdev_l3_rcv(skb, AF_INET); } static inline -struct dst_entry *l3mdev_rt6_dst_by_oif(struct net *net, - const struct flowi6 *fl6) +struct sk_buff *l3mdev_ip6_rcv(struct sk_buff *skb) { - struct dst_entry *dst = NULL; - struct net_device *dev; - - dev = dev_get_by_index(net, fl6->flowi6_oif); - if (dev) { - dst = l3mdev_get_rt6_dst(dev, fl6); - dev_put(dev); - } - - return dst; + return l3mdev_l3_rcv(skb, AF_INET6); } #else @@ -233,16 +220,21 @@ static inline int l3mdev_get_saddr(struct net *net, int ifindex, } static inline -struct dst_entry *l3mdev_get_rt6_dst(const struct net_device *dev, - const struct flowi6 *fl6) +struct dst_entry *l3mdev_get_rt6_dst(struct net *net, const struct flowi6 *fl6) { return NULL; } + static inline -struct dst_entry *l3mdev_rt6_dst_by_oif(struct net *net, - const struct flowi6 *fl6) +struct sk_buff *l3mdev_ip_rcv(struct sk_buff *skb) { - return NULL; + return skb; +} + +static inline +struct sk_buff *l3mdev_ip6_rcv(struct sk_buff *skb) +{ + return skb; } #endif diff --git a/include/net/mac80211.h b/include/net/mac80211.h index a3ee76559791..07ef9378df2b 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -549,7 +549,7 @@ struct ieee80211_bss_conf { u8 sync_dtim_count; u32 basic_rates; struct ieee80211_rate *beacon_rate; - int mcast_rate[IEEE80211_NUM_BANDS]; + int mcast_rate[NUM_NL80211_BANDS]; u16 ht_operation_mode; s32 cqm_rssi_thold; u32 cqm_rssi_hyst; @@ -938,8 +938,8 @@ struct ieee80211_tx_info { * @common_ie_len: length of the common_ies */ struct ieee80211_scan_ies { - const u8 *ies[IEEE80211_NUM_BANDS]; - size_t len[IEEE80211_NUM_BANDS]; + const u8 *ies[NUM_NL80211_BANDS]; + size_t len[NUM_NL80211_BANDS]; const u8 *common_ies; size_t common_ie_len; }; @@ -1007,6 +1007,8 @@ ieee80211_tx_info_clear_status(struct ieee80211_tx_info *info) * flag indicates that the PN was verified for replay protection. * Note that this flag is also currently only supported when a frame * is also decrypted (ie. @RX_FLAG_DECRYPTED must be set) + * @RX_FLAG_DUP_VALIDATED: The driver should set this flag if it did + * de-duplication by itself. * @RX_FLAG_FAILED_FCS_CRC: Set this flag if the FCS check failed on * the frame. * @RX_FLAG_FAILED_PLCP_CRC: Set this flag if the PCLP check failed on @@ -1752,7 +1754,7 @@ struct ieee80211_sta_rates { * @txq: per-TID data TX queues (if driver uses the TXQ abstraction) */ struct ieee80211_sta { - u32 supp_rates[IEEE80211_NUM_BANDS]; + u32 supp_rates[NUM_NL80211_BANDS]; u8 addr[ETH_ALEN]; u16 aid; struct ieee80211_sta_ht_cap ht_cap; @@ -4402,7 +4404,7 @@ __le16 ieee80211_ctstoself_duration(struct ieee80211_hw *hw, */ __le16 ieee80211_generic_frame_duration(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - enum ieee80211_band band, + enum nl80211_band band, size_t frame_len, struct ieee80211_rate *rate); @@ -5355,7 +5357,7 @@ struct rate_control_ops { }; static inline int rate_supported(struct ieee80211_sta *sta, - enum ieee80211_band band, + enum nl80211_band band, int index) { return (sta == NULL || sta->supp_rates[band] & BIT(index)); diff --git a/include/net/mac802154.h b/include/net/mac802154.h index 6cd7a70706a9..e465c8551ac3 100644 --- a/include/net/mac802154.h +++ b/include/net/mac802154.h @@ -288,6 +288,16 @@ static inline void ieee802154_le16_to_be16(void *be16_dst, const void *le16_src) } /** + * ieee802154_be16_to_le16 - copies and convert be16 to le16 + * @le16_dst: le16 destination pointer + * @be16_src: be16 source pointer + */ +static inline void ieee802154_be16_to_le16(void *le16_dst, const void *be16_src) +{ + put_unaligned_le16(get_unaligned_be16(be16_src), le16_dst); +} + +/** * ieee802154_alloc_hw - Allocate a new hardware device * * This must be called once for each hardware device. The returned pointer diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index fde4068eec0b..dd78bea227c8 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -289,8 +289,6 @@ struct kernel_param; int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp); extern unsigned int nf_conntrack_htable_size; extern unsigned int nf_conntrack_max; -extern unsigned int nf_conntrack_hash_rnd; -void init_nf_conntrack_hash_rnd(void); struct nf_conn *nf_ct_tmpl_alloc(struct net *net, const struct nf_conntrack_zone *zone, diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 62e17d1319ff..3e2f3328945c 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -81,6 +81,7 @@ print_tuple(struct seq_file *s, const struct nf_conntrack_tuple *tuple, #define CONNTRACK_LOCKS 1024 +extern struct hlist_nulls_head *nf_conntrack_hash; extern spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS]; void nf_conntrack_lock(spinlock_t *lock); diff --git a/include/net/netfilter/nf_conntrack_ecache.h b/include/net/netfilter/nf_conntrack_ecache.h index 57c880378443..fa36447371c6 100644 --- a/include/net/netfilter/nf_conntrack_ecache.h +++ b/include/net/netfilter/nf_conntrack_ecache.h @@ -73,6 +73,8 @@ void nf_conntrack_unregister_notifier(struct net *net, struct nf_ct_event_notifier *nb); void nf_ct_deliver_cached_events(struct nf_conn *ct); +int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct, + u32 portid, int report); static inline void nf_conntrack_event_cache(enum ip_conntrack_events event, struct nf_conn *ct) @@ -91,69 +93,25 @@ nf_conntrack_event_cache(enum ip_conntrack_events event, struct nf_conn *ct) } static inline int -nf_conntrack_eventmask_report(unsigned int eventmask, - struct nf_conn *ct, - u32 portid, - int report) -{ - int ret = 0; - struct net *net = nf_ct_net(ct); - struct nf_ct_event_notifier *notify; - struct nf_conntrack_ecache *e; - - rcu_read_lock(); - notify = rcu_dereference(net->ct.nf_conntrack_event_cb); - if (notify == NULL) - goto out_unlock; - - e = nf_ct_ecache_find(ct); - if (e == NULL) - goto out_unlock; - - if (nf_ct_is_confirmed(ct) && !nf_ct_is_dying(ct)) { - struct nf_ct_event item = { - .ct = ct, - .portid = e->portid ? e->portid : portid, - .report = report - }; - /* This is a resent of a destroy event? If so, skip missed */ - unsigned long missed = e->portid ? 0 : e->missed; - - if (!((eventmask | missed) & e->ctmask)) - goto out_unlock; - - ret = notify->fcn(eventmask | missed, &item); - if (unlikely(ret < 0 || missed)) { - spin_lock_bh(&ct->lock); - if (ret < 0) { - /* This is a destroy event that has been - * triggered by a process, we store the PORTID - * to include it in the retransmission. */ - if (eventmask & (1 << IPCT_DESTROY) && - e->portid == 0 && portid != 0) - e->portid = portid; - else - e->missed |= eventmask; - } else - e->missed &= ~missed; - spin_unlock_bh(&ct->lock); - } - } -out_unlock: - rcu_read_unlock(); - return ret; -} - -static inline int nf_conntrack_event_report(enum ip_conntrack_events event, struct nf_conn *ct, u32 portid, int report) { + const struct net *net = nf_ct_net(ct); + + if (!rcu_access_pointer(net->ct.nf_conntrack_event_cb)) + return 0; + return nf_conntrack_eventmask_report(1 << event, ct, portid, report); } static inline int nf_conntrack_event(enum ip_conntrack_events event, struct nf_conn *ct) { + const struct net *net = nf_ct_net(ct); + + if (!rcu_access_pointer(net->ct.nf_conntrack_event_cb)) + return 0; + return nf_conntrack_eventmask_report(1 << event, ct, 0, 0); } @@ -172,43 +130,9 @@ int nf_ct_expect_register_notifier(struct net *net, void nf_ct_expect_unregister_notifier(struct net *net, struct nf_exp_event_notifier *nb); -static inline void -nf_ct_expect_event_report(enum ip_conntrack_expect_events event, - struct nf_conntrack_expect *exp, - u32 portid, - int report) -{ - struct net *net = nf_ct_exp_net(exp); - struct nf_exp_event_notifier *notify; - struct nf_conntrack_ecache *e; - - rcu_read_lock(); - notify = rcu_dereference(net->ct.nf_expect_event_cb); - if (notify == NULL) - goto out_unlock; - - e = nf_ct_ecache_find(exp->master); - if (e == NULL) - goto out_unlock; - - if (e->expmask & (1 << event)) { - struct nf_exp_event item = { - .exp = exp, - .portid = portid, - .report = report - }; - notify->fcn(1 << event, &item); - } -out_unlock: - rcu_read_unlock(); -} - -static inline void -nf_ct_expect_event(enum ip_conntrack_expect_events event, - struct nf_conntrack_expect *exp) -{ - nf_ct_expect_event_report(event, exp, 0, 0); -} +void nf_ct_expect_event_report(enum ip_conntrack_expect_events event, + struct nf_conntrack_expect *exp, + u32 portid, int report); int nf_conntrack_ecache_pernet_init(struct net *net); void nf_conntrack_ecache_pernet_fini(struct net *net); @@ -245,8 +169,6 @@ static inline int nf_conntrack_event_report(enum ip_conntrack_events event, u32 portid, int report) { return 0; } static inline void nf_ct_deliver_cached_events(const struct nf_conn *ct) {} -static inline void nf_ct_expect_event(enum ip_conntrack_expect_events event, - struct nf_conntrack_expect *exp) {} static inline void nf_ct_expect_event_report(enum ip_conntrack_expect_events e, struct nf_conntrack_expect *exp, u32 portid, diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index dce56f09ac9a..5ed33ea4718e 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -10,6 +10,7 @@ extern unsigned int nf_ct_expect_hsize; extern unsigned int nf_ct_expect_max; +extern struct hlist_head *nf_ct_expect_hash; struct nf_conntrack_expect { /* Conntrack expectation list member */ diff --git a/include/net/netfilter/nf_conntrack_l4proto.h b/include/net/netfilter/nf_conntrack_l4proto.h index 956d8a6ac069..1a5fb36f165f 100644 --- a/include/net/netfilter/nf_conntrack_l4proto.h +++ b/include/net/netfilter/nf_conntrack_l4proto.h @@ -23,6 +23,9 @@ struct nf_conntrack_l4proto { /* L4 Protocol number. */ u_int8_t l4proto; + /* Resolve clashes on insertion races. */ + bool allow_clash; + /* Try to fill in the third arg: dataoff is offset past network protocol hdr. Return true if possible. */ bool (*pkt_to_tuple)(const struct sk_buff *skb, unsigned int dataoff, diff --git a/include/net/netfilter/nf_conntrack_labels.h b/include/net/netfilter/nf_conntrack_labels.h index 7e2b1d025f50..c5f8fc736b3d 100644 --- a/include/net/netfilter/nf_conntrack_labels.h +++ b/include/net/netfilter/nf_conntrack_labels.h @@ -45,7 +45,6 @@ static inline struct nf_conn_labels *nf_ct_labels_ext_add(struct nf_conn *ct) #endif } -bool nf_connlabel_match(const struct nf_conn *ct, u16 bit); int nf_connlabel_set(struct nf_conn *ct, u16 bit); int nf_connlabels_replace(struct nf_conn *ct, @@ -54,11 +53,11 @@ int nf_connlabels_replace(struct nf_conn *ct, #ifdef CONFIG_NF_CONNTRACK_LABELS int nf_conntrack_labels_init(void); void nf_conntrack_labels_fini(void); -int nf_connlabels_get(struct net *net, unsigned int n_bits); +int nf_connlabels_get(struct net *net, unsigned int bit); void nf_connlabels_put(struct net *net); #else static inline int nf_conntrack_labels_init(void) { return 0; } static inline void nf_conntrack_labels_fini(void) {} -static inline int nf_connlabels_get(struct net *net, unsigned int n_bits) { return 0; } +static inline int nf_connlabels_get(struct net *net, unsigned int bit) { return 0; } static inline void nf_connlabels_put(struct net *net) {} #endif diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index f6b1daf2e698..092235458691 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -303,7 +303,7 @@ void nft_unregister_set(struct nft_set_ops *ops); struct nft_set { struct list_head list; struct list_head bindings; - char name[IFNAMSIZ]; + char name[NFT_SET_MAXNAMELEN]; u32 ktype; u32 dtype; u32 size; diff --git a/include/net/netlink.h b/include/net/netlink.h index 0e3172751755..e589cb3dccee 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -102,10 +102,12 @@ * nla_put_s8(skb, type, value) add s8 attribute to skb * nla_put_s16(skb, type, value) add s16 attribute to skb * nla_put_s32(skb, type, value) add s32 attribute to skb - * nla_put_s64(skb, type, value) add s64 attribute to skb + * nla_put_s64(skb, type, value, + * padattr) add s64 attribute to skb * nla_put_string(skb, type, str) add string attribute to skb * nla_put_flag(skb, type) add flag attribute to skb - * nla_put_msecs(skb, type, jiffies) add msecs attribute to skb + * nla_put_msecs(skb, type, jiffies, + * padattr) add msecs attribute to skb * nla_put_in_addr(skb, type, addr) add IPv4 address attribute to skb * nla_put_in6_addr(skb, type, addr) add IPv6 address attribute to skb * @@ -244,13 +246,21 @@ int nla_memcpy(void *dest, const struct nlattr *src, int count); int nla_memcmp(const struct nlattr *nla, const void *data, size_t size); int nla_strcmp(const struct nlattr *nla, const char *str); struct nlattr *__nla_reserve(struct sk_buff *skb, int attrtype, int attrlen); +struct nlattr *__nla_reserve_64bit(struct sk_buff *skb, int attrtype, + int attrlen, int padattr); void *__nla_reserve_nohdr(struct sk_buff *skb, int attrlen); struct nlattr *nla_reserve(struct sk_buff *skb, int attrtype, int attrlen); +struct nlattr *nla_reserve_64bit(struct sk_buff *skb, int attrtype, + int attrlen, int padattr); void *nla_reserve_nohdr(struct sk_buff *skb, int attrlen); void __nla_put(struct sk_buff *skb, int attrtype, int attrlen, const void *data); +void __nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen, + const void *data, int padattr); void __nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data); int nla_put(struct sk_buff *skb, int attrtype, int attrlen, const void *data); +int nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen, + const void *data, int padattr); int nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data); int nla_append(struct sk_buff *skb, int attrlen, const void *data); @@ -848,36 +858,56 @@ static inline int nla_put_u64(struct sk_buff *skb, int attrtype, u64 value) } /** - * nla_put_be64 - Add a __be64 netlink attribute to a socket buffer + * nla_put_u64_64bit - Add a u64 netlink attribute to a skb and align it * @skb: socket buffer to add attribute to * @attrtype: attribute type * @value: numeric value + * @padattr: attribute type for the padding */ -static inline int nla_put_be64(struct sk_buff *skb, int attrtype, __be64 value) +static inline int nla_put_u64_64bit(struct sk_buff *skb, int attrtype, + u64 value, int padattr) { - return nla_put(skb, attrtype, sizeof(__be64), &value); + return nla_put_64bit(skb, attrtype, sizeof(u64), &value, padattr); } /** - * nla_put_net64 - Add 64-bit network byte order netlink attribute to a socket buffer + * nla_put_be64 - Add a __be64 netlink attribute to a socket buffer and align it * @skb: socket buffer to add attribute to * @attrtype: attribute type * @value: numeric value + * @padattr: attribute type for the padding */ -static inline int nla_put_net64(struct sk_buff *skb, int attrtype, __be64 value) +static inline int nla_put_be64(struct sk_buff *skb, int attrtype, __be64 value, + int padattr) { - return nla_put_be64(skb, attrtype | NLA_F_NET_BYTEORDER, value); + return nla_put_64bit(skb, attrtype, sizeof(__be64), &value, padattr); } /** - * nla_put_le64 - Add a __le64 netlink attribute to a socket buffer + * nla_put_net64 - Add 64-bit network byte order nlattr to a skb and align it * @skb: socket buffer to add attribute to * @attrtype: attribute type * @value: numeric value + * @padattr: attribute type for the padding */ -static inline int nla_put_le64(struct sk_buff *skb, int attrtype, __le64 value) +static inline int nla_put_net64(struct sk_buff *skb, int attrtype, __be64 value, + int padattr) { - return nla_put(skb, attrtype, sizeof(__le64), &value); + return nla_put_be64(skb, attrtype | NLA_F_NET_BYTEORDER, value, + padattr); +} + +/** + * nla_put_le64 - Add a __le64 netlink attribute to a socket buffer and align it + * @skb: socket buffer to add attribute to + * @attrtype: attribute type + * @value: numeric value + * @padattr: attribute type for the padding + */ +static inline int nla_put_le64(struct sk_buff *skb, int attrtype, __le64 value, + int padattr) +{ + return nla_put_64bit(skb, attrtype, sizeof(__le64), &value, padattr); } /** @@ -914,14 +944,16 @@ static inline int nla_put_s32(struct sk_buff *skb, int attrtype, s32 value) } /** - * nla_put_s64 - Add a s64 netlink attribute to a socket buffer + * nla_put_s64 - Add a s64 netlink attribute to a socket buffer and align it * @skb: socket buffer to add attribute to * @attrtype: attribute type * @value: numeric value + * @padattr: attribute type for the padding */ -static inline int nla_put_s64(struct sk_buff *skb, int attrtype, s64 value) +static inline int nla_put_s64(struct sk_buff *skb, int attrtype, s64 value, + int padattr) { - return nla_put(skb, attrtype, sizeof(s64), &value); + return nla_put_64bit(skb, attrtype, sizeof(s64), &value, padattr); } /** @@ -947,16 +979,18 @@ static inline int nla_put_flag(struct sk_buff *skb, int attrtype) } /** - * nla_put_msecs - Add a msecs netlink attribute to a socket buffer + * nla_put_msecs - Add a msecs netlink attribute to a skb and align it * @skb: socket buffer to add attribute to * @attrtype: attribute type * @njiffies: number of jiffies to convert to msecs + * @padattr: attribute type for the padding */ static inline int nla_put_msecs(struct sk_buff *skb, int attrtype, - unsigned long njiffies) + unsigned long njiffies, int padattr) { u64 tmp = jiffies_to_msecs(njiffies); - return nla_put(skb, attrtype, sizeof(u64), &tmp); + + return nla_put_64bit(skb, attrtype, sizeof(u64), &tmp, padattr); } /** @@ -1231,6 +1265,61 @@ static inline int nla_validate_nested(const struct nlattr *start, int maxtype, } /** + * nla_need_padding_for_64bit - test 64-bit alignment of the next attribute + * @skb: socket buffer the message is stored in + * + * Return true if padding is needed to align the next attribute (nla_data()) to + * a 64-bit aligned area. + */ +static inline bool nla_need_padding_for_64bit(struct sk_buff *skb) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + /* The nlattr header is 4 bytes in size, that's why we test + * if the skb->data _is_ aligned. A NOP attribute, plus + * nlattr header for next attribute, will make nla_data() + * 8-byte aligned. + */ + if (IS_ALIGNED((unsigned long)skb_tail_pointer(skb), 8)) + return true; +#endif + return false; +} + +/** + * nla_align_64bit - 64-bit align the nla_data() of next attribute + * @skb: socket buffer the message is stored in + * @padattr: attribute type for the padding + * + * Conditionally emit a padding netlink attribute in order to make + * the next attribute we emit have a 64-bit aligned nla_data() area. + * This will only be done in architectures which do not have + * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS defined. + * + * Returns zero on success or a negative error code. + */ +static inline int nla_align_64bit(struct sk_buff *skb, int padattr) +{ + if (nla_need_padding_for_64bit(skb) && + !nla_reserve(skb, padattr, 0)) + return -EMSGSIZE; + + return 0; +} + +/** + * nla_total_size_64bit - total length of attribute including padding + * @payload: length of payload + */ +static inline int nla_total_size_64bit(int payload) +{ + return NLA_ALIGN(nla_attr_size(payload)) +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + + NLA_ALIGN(nla_attr_size(0)) +#endif + ; +} + +/** * nla_for_each_attr - iterate over a stream of attributes * @pos: loop counter, set to current attribute * @head: head of attribute stream diff --git a/include/net/netns/conntrack.h b/include/net/netns/conntrack.h index 723b61c82b3f..38b1a80517f0 100644 --- a/include/net/netns/conntrack.h +++ b/include/net/netns/conntrack.h @@ -84,7 +84,6 @@ struct netns_ct { struct ctl_table_header *event_sysctl_header; struct ctl_table_header *helper_sysctl_header; #endif - char *slabname; unsigned int sysctl_log_invalid; /* Log invalid packets */ int sysctl_events; int sysctl_acct; @@ -93,11 +92,6 @@ struct netns_ct { int sysctl_tstamp; int sysctl_checksum; - unsigned int htable_size; - seqcount_t generation; - struct kmem_cache *nf_conntrack_cachep; - struct hlist_nulls_head *hash; - struct hlist_head *expect_hash; struct ct_pcpu __percpu *pcpu_lists; struct ip_conntrack_stat __percpu *stat; struct nf_ct_event_notifier __rcu *nf_conntrack_event_cb; @@ -107,9 +101,5 @@ struct netns_ct { unsigned int labels_used; u8 label_words; #endif -#ifdef CONFIG_NF_NAT_NEEDED - struct hlist_head *nat_bysource; - unsigned int nat_htable_size; -#endif }; #endif diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index a69cde3ce460..d061ffeb1e71 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -133,6 +133,9 @@ struct netns_ipv4 { struct fib_rules_ops *mr_rules_ops; #endif #endif +#ifdef CONFIG_IP_ROUTE_MULTIPATH + int sysctl_fib_multipath_use_neigh; +#endif atomic_t rt_genid; }; #endif diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h index 730d82ad6ee5..24cd3949a9a4 100644 --- a/include/net/netns/xfrm.h +++ b/include/net/netns/xfrm.h @@ -80,6 +80,7 @@ struct netns_xfrm { struct flow_cache flow_cache_global; atomic_t flow_cache_genid; struct list_head flow_cache_gc_list; + atomic_t flow_cache_gc_count; spinlock_t flow_cache_gc_lock; struct work_struct flow_cache_gc_work; struct work_struct flow_cache_flush_work; diff --git a/include/net/nl802154.h b/include/net/nl802154.h index 32cb3e591e07..fcab4de49951 100644 --- a/include/net/nl802154.h +++ b/include/net/nl802154.h @@ -138,6 +138,8 @@ enum nl802154_attrs { NL802154_ATTR_SEC_KEY, #endif /* CONFIG_IEEE802154_NL802154_EXPERIMENTAL */ + NL802154_ATTR_PAD, + __NL802154_ATTR_AFTER_LAST, NL802154_ATTR_MAX = __NL802154_ATTR_AFTER_LAST - 1 }; @@ -295,6 +297,7 @@ enum nl802154_dev_addr_attrs { NL802154_DEV_ADDR_ATTR_MODE, NL802154_DEV_ADDR_ATTR_SHORT, NL802154_DEV_ADDR_ATTR_EXTENDED, + NL802154_DEV_ADDR_ATTR_PAD, /* keep last */ __NL802154_DEV_ADDR_ATTR_AFTER_LAST, @@ -320,6 +323,7 @@ enum nl802154_key_id_attrs { NL802154_KEY_ID_ATTR_IMPLICIT, NL802154_KEY_ID_ATTR_SOURCE_SHORT, NL802154_KEY_ID_ATTR_SOURCE_EXTENDED, + NL802154_KEY_ID_ATTR_PAD, /* keep last */ __NL802154_KEY_ID_ATTR_AFTER_LAST, @@ -402,6 +406,7 @@ enum nl802154_dev { NL802154_DEV_ATTR_EXTENDED_ADDR, NL802154_DEV_ATTR_SECLEVEL_EXEMPT, NL802154_DEV_ATTR_KEY_MODE, + NL802154_DEV_ATTR_PAD, /* keep last */ __NL802154_DEV_ATTR_AFTER_LAST, @@ -414,6 +419,7 @@ enum nl802154_devkey { NL802154_DEVKEY_ATTR_FRAME_COUNTER, NL802154_DEVKEY_ATTR_EXTENDED_ADDR, NL802154_DEVKEY_ATTR_ID, + NL802154_DEVKEY_ATTR_PAD, /* keep last */ __NL802154_DEVKEY_ATTR_AFTER_LAST, diff --git a/include/net/route.h b/include/net/route.h index f4b11eee1754..ad777d79af94 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -209,6 +209,9 @@ unsigned int inet_addr_type_dev_table(struct net *net, void ip_rt_multicast_event(struct in_device *); int ip_rt_ioctl(struct net *, unsigned int cmd, void __user *arg); void ip_rt_get_source(u8 *src, struct sk_buff *skb, struct rtable *rt); +struct rtable *rt_dst_alloc(struct net_device *dev, + unsigned int flags, u16 type, + bool nopolicy, bool noxfrm, bool will_cache); struct in_ifaddr; void fib_add_ifaddr(struct in_ifaddr *); diff --git a/include/net/rtnetlink.h b/include/net/rtnetlink.h index 2f87c1ba13de..006a7b81d758 100644 --- a/include/net/rtnetlink.h +++ b/include/net/rtnetlink.h @@ -47,6 +47,9 @@ static inline int rtnl_msg_family(const struct nlmsghdr *nlh) * @get_num_rx_queues: Function to determine number of receive queues * to create when creating a new device. * @get_link_net: Function to get the i/o netns of the device + * @get_linkxstats_size: Function to calculate the required room for + * dumping device-specific extended link stats + * @fill_linkxstats: Function to dump device-specific extended link stats */ struct rtnl_link_ops { struct list_head list; @@ -95,6 +98,10 @@ struct rtnl_link_ops { const struct net_device *dev, const struct net_device *slave_dev); struct net *(*get_link_net)(const struct net_device *dev); + size_t (*get_linkxstats_size)(const struct net_device *dev); + int (*fill_linkxstats)(struct sk_buff *skb, + const struct net_device *dev, + int *prividx); }; int __rtnl_link_register(struct rtnl_link_ops *ops); diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h index 65521cfdcade..b392ac8382f2 100644 --- a/include/net/sctp/sctp.h +++ b/include/net/sctp/sctp.h @@ -116,6 +116,22 @@ extern struct percpu_counter sctp_sockets_allocated; int sctp_asconf_mgmt(struct sctp_sock *, struct sctp_sockaddr_entry *); struct sk_buff *sctp_skb_recv_datagram(struct sock *, int, int, int *); +int sctp_transport_walk_start(struct rhashtable_iter *iter); +void sctp_transport_walk_stop(struct rhashtable_iter *iter); +struct sctp_transport *sctp_transport_get_next(struct net *net, + struct rhashtable_iter *iter); +struct sctp_transport *sctp_transport_get_idx(struct net *net, + struct rhashtable_iter *iter, int pos); +int sctp_transport_lookup_process(int (*cb)(struct sctp_transport *, void *), + struct net *net, + const union sctp_addr *laddr, + const union sctp_addr *paddr, void *p); +int sctp_for_each_transport(int (*cb)(struct sctp_transport *, void *), + struct net *net, int pos, void *p); +int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), void *p); +int sctp_get_sctp_info(struct sock *sk, struct sctp_association *asoc, + struct sctp_info *info); + /* * sctp/primitive.c */ @@ -189,10 +205,9 @@ extern int sysctl_sctp_wmem[3]; */ /* SCTP SNMP MIB stats handlers */ -#define SCTP_INC_STATS(net, field) SNMP_INC_STATS((net)->sctp.sctp_statistics, field) -#define SCTP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->sctp.sctp_statistics, field) -#define SCTP_INC_STATS_USER(net, field) SNMP_INC_STATS_USER((net)->sctp.sctp_statistics, field) -#define SCTP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->sctp.sctp_statistics, field) +#define SCTP_INC_STATS(net, field) SNMP_INC_STATS((net)->sctp.sctp_statistics, field) +#define __SCTP_INC_STATS(net, field) __SNMP_INC_STATS((net)->sctp.sctp_statistics, field) +#define SCTP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->sctp.sctp_statistics, field) /* sctp mib definitions */ enum { @@ -359,21 +374,6 @@ int sctp_do_peeloff(struct sock *sk, sctp_assoc_t id, struct socket **sockp); #define sctp_skb_for_each(pos, head, tmp) \ skb_queue_walk_safe(head, pos, tmp) -/* A helper to append an entire skb list (list) to another (head). */ -static inline void sctp_skb_list_tail(struct sk_buff_head *list, - struct sk_buff_head *head) -{ - unsigned long flags; - - spin_lock_irqsave(&head->lock, flags); - spin_lock(&list->lock); - - skb_queue_splice_tail_init(list, head); - - spin_unlock(&list->lock); - spin_unlock_irqrestore(&head->lock, flags); -} - /** * sctp_list_dequeue - remove from the head of the queue * @list: list to dequeue from @@ -386,11 +386,9 @@ static inline struct list_head *sctp_list_dequeue(struct list_head *list) { struct list_head *result = NULL; - if (list->next != list) { + if (!list_empty(list)) { result = list->next; - list->next = result->next; - list->next->prev = list; - INIT_LIST_HEAD(result); + list_del_init(result); } return result; } diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 6df1ce7a411c..16b013a6191c 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -210,14 +210,15 @@ struct sctp_sock { int user_frag; __u32 autoclose; - __u8 nodelay; - __u8 disable_fragments; - __u8 v4mapped; - __u8 frag_interleave; __u32 adaptation_ind; __u32 pd_point; - __u8 recvrcvinfo; - __u8 recvnxtinfo; + __u16 nodelay:1, + disable_fragments:1, + v4mapped:1, + frag_interleave:1, + recvrcvinfo:1, + recvnxtinfo:1, + data_ready_signalled:1; atomic_t pd_mode; /* Receive to here while partial delivery is in effect. */ @@ -847,6 +848,11 @@ struct sctp_transport { */ ktime_t last_time_heard; + /* When was the last time that we sent a chunk using this + * transport? We use this to check for idle transports + */ + unsigned long last_time_sent; + /* Last time(in jiffies) when cwnd is reduced due to the congestion * indication based on ECNE chunk. */ @@ -952,7 +958,8 @@ void sctp_transport_route(struct sctp_transport *, union sctp_addr *, struct sctp_sock *); void sctp_transport_pmtu(struct sctp_transport *, struct sock *sk); void sctp_transport_free(struct sctp_transport *); -void sctp_transport_reset_timers(struct sctp_transport *); +void sctp_transport_reset_t3_rtx(struct sctp_transport *); +void sctp_transport_reset_hb_timer(struct sctp_transport *); int sctp_transport_hold(struct sctp_transport *); void sctp_transport_put(struct sctp_transport *); void sctp_transport_update_rto(struct sctp_transport *, __u32); diff --git a/include/net/snmp.h b/include/net/snmp.h index 35512ac6dcfb..c9228ad7ee91 100644 --- a/include/net/snmp.h +++ b/include/net/snmp.h @@ -123,12 +123,9 @@ struct linux_xfrm_mib { #define DECLARE_SNMP_STAT(type, name) \ extern __typeof__(type) __percpu *name -#define SNMP_INC_STATS_BH(mib, field) \ +#define __SNMP_INC_STATS(mib, field) \ __this_cpu_inc(mib->mibs[field]) -#define SNMP_INC_STATS_USER(mib, field) \ - this_cpu_inc(mib->mibs[field]) - #define SNMP_INC_STATS_ATOMIC_LONG(mib, field) \ atomic_long_inc(&mib->mibs[field]) @@ -138,12 +135,9 @@ struct linux_xfrm_mib { #define SNMP_DEC_STATS(mib, field) \ this_cpu_dec(mib->mibs[field]) -#define SNMP_ADD_STATS_BH(mib, field, addend) \ +#define __SNMP_ADD_STATS(mib, field, addend) \ __this_cpu_add(mib->mibs[field], addend) -#define SNMP_ADD_STATS_USER(mib, field, addend) \ - this_cpu_add(mib->mibs[field], addend) - #define SNMP_ADD_STATS(mib, field, addend) \ this_cpu_add(mib->mibs[field], addend) #define SNMP_UPD_PO_STATS(mib, basefield, addend) \ @@ -152,7 +146,7 @@ struct linux_xfrm_mib { this_cpu_inc(ptr[basefield##PKTS]); \ this_cpu_add(ptr[basefield##OCTETS], addend); \ } while (0) -#define SNMP_UPD_PO_STATS_BH(mib, basefield, addend) \ +#define __SNMP_UPD_PO_STATS(mib, basefield, addend) \ do { \ __typeof__((mib->mibs) + 0) ptr = mib->mibs; \ __this_cpu_inc(ptr[basefield##PKTS]); \ @@ -162,7 +156,7 @@ struct linux_xfrm_mib { #if BITS_PER_LONG==32 -#define SNMP_ADD_STATS64_BH(mib, field, addend) \ +#define __SNMP_ADD_STATS64(mib, field, addend) \ do { \ __typeof__(*mib) *ptr = raw_cpu_ptr(mib); \ u64_stats_update_begin(&ptr->syncp); \ @@ -170,20 +164,16 @@ struct linux_xfrm_mib { u64_stats_update_end(&ptr->syncp); \ } while (0) -#define SNMP_ADD_STATS64_USER(mib, field, addend) \ +#define SNMP_ADD_STATS64(mib, field, addend) \ do { \ local_bh_disable(); \ - SNMP_ADD_STATS64_BH(mib, field, addend); \ - local_bh_enable(); \ + __SNMP_ADD_STATS64(mib, field, addend); \ + local_bh_enable(); \ } while (0) -#define SNMP_ADD_STATS64(mib, field, addend) \ - SNMP_ADD_STATS64_USER(mib, field, addend) - -#define SNMP_INC_STATS64_BH(mib, field) SNMP_ADD_STATS64_BH(mib, field, 1) -#define SNMP_INC_STATS64_USER(mib, field) SNMP_ADD_STATS64_USER(mib, field, 1) +#define __SNMP_INC_STATS64(mib, field) SNMP_ADD_STATS64(mib, field, 1) #define SNMP_INC_STATS64(mib, field) SNMP_ADD_STATS64(mib, field, 1) -#define SNMP_UPD_PO_STATS64_BH(mib, basefield, addend) \ +#define __SNMP_UPD_PO_STATS64(mib, basefield, addend) \ do { \ __typeof__(*mib) *ptr; \ ptr = raw_cpu_ptr((mib)); \ @@ -195,19 +185,17 @@ struct linux_xfrm_mib { #define SNMP_UPD_PO_STATS64(mib, basefield, addend) \ do { \ local_bh_disable(); \ - SNMP_UPD_PO_STATS64_BH(mib, basefield, addend); \ - local_bh_enable(); \ + __SNMP_UPD_PO_STATS64(mib, basefield, addend); \ + local_bh_enable(); \ } while (0) #else -#define SNMP_INC_STATS64_BH(mib, field) SNMP_INC_STATS_BH(mib, field) -#define SNMP_INC_STATS64_USER(mib, field) SNMP_INC_STATS_USER(mib, field) +#define __SNMP_INC_STATS64(mib, field) __SNMP_INC_STATS(mib, field) #define SNMP_INC_STATS64(mib, field) SNMP_INC_STATS(mib, field) #define SNMP_DEC_STATS64(mib, field) SNMP_DEC_STATS(mib, field) -#define SNMP_ADD_STATS64_BH(mib, field, addend) SNMP_ADD_STATS_BH(mib, field, addend) -#define SNMP_ADD_STATS64_USER(mib, field, addend) SNMP_ADD_STATS_USER(mib, field, addend) +#define __SNMP_ADD_STATS64(mib, field, addend) __SNMP_ADD_STATS(mib, field, addend) #define SNMP_ADD_STATS64(mib, field, addend) SNMP_ADD_STATS(mib, field, addend) #define SNMP_UPD_PO_STATS64(mib, basefield, addend) SNMP_UPD_PO_STATS(mib, basefield, addend) -#define SNMP_UPD_PO_STATS64_BH(mib, basefield, addend) SNMP_UPD_PO_STATS_BH(mib, basefield, addend) +#define __SNMP_UPD_PO_STATS64(mib, basefield, addend) __SNMP_UPD_PO_STATS(mib, basefield, addend) #endif #endif diff --git a/include/net/sock.h b/include/net/sock.h index 81d6fecec0a2..c9c8b19df27c 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -569,7 +569,7 @@ static inline bool __sk_del_node_init(struct sock *sk) modifications. */ -static inline void sock_hold(struct sock *sk) +static __always_inline void sock_hold(struct sock *sk) { atomic_inc(&sk->sk_refcnt); } @@ -577,7 +577,7 @@ static inline void sock_hold(struct sock *sk) /* Ungrab socket in the context, which assumes that socket refcnt cannot hit zero, f.e. it is true in context of any socketcall. */ -static inline void __sock_put(struct sock *sk) +static __always_inline void __sock_put(struct sock *sk) { atomic_dec(&sk->sk_refcnt); } @@ -630,12 +630,20 @@ static inline void sk_add_node(struct sock *sk, struct hlist_head *list) static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list) { sock_hold(sk); - hlist_add_head_rcu(&sk->sk_node, list); + if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && + sk->sk_family == AF_INET6) + hlist_add_tail_rcu(&sk->sk_node, list); + else + hlist_add_head_rcu(&sk->sk_node, list); } static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list) { - hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list); + if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && + sk->sk_family == AF_INET6) + hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list); + else + hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list); } static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list) @@ -918,6 +926,17 @@ void sk_stream_kill_queues(struct sock *sk); void sk_set_memalloc(struct sock *sk); void sk_clear_memalloc(struct sock *sk); +void __sk_flush_backlog(struct sock *sk); + +static inline bool sk_flush_backlog(struct sock *sk) +{ + if (unlikely(READ_ONCE(sk->sk_backlog.tail))) { + __sk_flush_backlog(sk); + return true; + } + return false; +} + int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb); struct request_sock_ops; @@ -1316,21 +1335,6 @@ static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb) __kfree_skb(skb); } -/* Used by processes to "lock" a socket state, so that - * interrupts and bottom half handlers won't change it - * from under us. It essentially blocks any incoming - * packets, so that we won't get any new data or any - * packets that change the state of the socket. - * - * While locked, BH processing will add new packets to - * the backlog queue. This queue is processed by the - * owner of the socket lock right before it is released. - * - * Since ~2.3.5 it is also exclusive sleep lock serializing - * accesses from user process context. - */ -#define sock_owned_by_user(sk) ((sk)->sk_lock.owned) - static inline void sock_release_ownership(struct sock *sk) { if (sk->sk_lock.owned) { @@ -1403,6 +1407,40 @@ static inline void unlock_sock_fast(struct sock *sk, bool slow) spin_unlock_bh(&sk->sk_lock.slock); } +/* Used by processes to "lock" a socket state, so that + * interrupts and bottom half handlers won't change it + * from under us. It essentially blocks any incoming + * packets, so that we won't get any new data or any + * packets that change the state of the socket. + * + * While locked, BH processing will add new packets to + * the backlog queue. This queue is processed by the + * owner of the socket lock right before it is released. + * + * Since ~2.3.5 it is also exclusive sleep lock serializing + * accesses from user process context. + */ + +static inline void sock_owned_by_me(const struct sock *sk) +{ +#ifdef CONFIG_LOCKDEP + WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks); +#endif +} + +static inline bool sock_owned_by_user(const struct sock *sk) +{ + sock_owned_by_me(sk); + return sk->sk_lock.owned; +} + +/* no reclassification while locks are held */ +static inline bool sock_allow_reclassification(const struct sock *csk) +{ + struct sock *sk = (struct sock *)csk; + + return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock); +} struct sock *sk_alloc(struct net *net, int family, gfp_t priority, struct proto *prot, int kern); @@ -1412,6 +1450,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority); struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, gfp_t priority); +void __sock_wfree(struct sk_buff *skb); void sock_wfree(struct sk_buff *skb); void skb_orphan_partial(struct sk_buff *skb); void sock_rfree(struct sk_buff *skb); @@ -1918,11 +1957,19 @@ static inline unsigned long sock_wspace(struct sock *sk) */ static inline void sk_set_bit(int nr, struct sock *sk) { + if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) && + !sock_flag(sk, SOCK_FASYNC)) + return; + set_bit(nr, &sk->sk_wq_raw->flags); } static inline void sk_clear_bit(int nr, struct sock *sk) { + if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) && + !sock_flag(sk, SOCK_FASYNC)) + return; + clear_bit(nr, &sk->sk_wq_raw->flags); } diff --git a/include/net/switchdev.h b/include/net/switchdev.h index d451122e8404..51d77b2ce2b2 100644 --- a/include/net/switchdev.h +++ b/include/net/switchdev.h @@ -54,6 +54,8 @@ struct switchdev_attr { struct net_device *orig_dev; enum switchdev_attr_id id; u32 flags; + void *complete_priv; + void (*complete)(struct net_device *dev, int err, void *priv); union { struct netdev_phys_item_id ppid; /* PORT_PARENT_ID */ u8 stp_state; /* PORT_STP_STATE */ @@ -75,6 +77,8 @@ struct switchdev_obj { struct net_device *orig_dev; enum switchdev_obj_id id; u32 flags; + void *complete_priv; + void (*complete)(struct net_device *dev, int err, void *priv); }; /* SWITCHDEV_OBJ_ID_PORT_VLAN */ diff --git a/include/net/tc_act/tc_mirred.h b/include/net/tc_act/tc_mirred.h index dae96bae1c19..e891835eb74e 100644 --- a/include/net/tc_act/tc_mirred.h +++ b/include/net/tc_act/tc_mirred.h @@ -2,6 +2,7 @@ #define __NET_TC_MIR_H #include <net/act_api.h> +#include <linux/tc_act/tc_mirred.h> struct tcf_mirred { struct tcf_common common; @@ -14,4 +15,18 @@ struct tcf_mirred { #define to_mirred(a) \ container_of(a->priv, struct tcf_mirred, common) +static inline bool is_tcf_mirred_redirect(const struct tc_action *a) +{ +#ifdef CONFIG_NET_CLS_ACT + if (a->ops && a->ops->type == TCA_ACT_MIRRED) + return to_mirred(a)->tcfm_eaction == TCA_EGRESS_REDIR; +#endif + return false; +} + +static inline int tcf_mirred_ifindex(const struct tc_action *a) +{ + return to_mirred(a)->tcfm_ifindex; +} + #endif /* __NET_TC_MIR_H */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 74d3ed5eb219..0bcc70f4e1fb 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -332,9 +332,8 @@ bool tcp_check_oom(struct sock *sk, int shift); extern struct proto tcp_prot; #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) -#define TCP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.tcp_statistics, field) +#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) -#define TCP_ADD_STATS_USER(net, field, val) SNMP_ADD_STATS_USER((net)->mib.tcp_statistics, field, val) #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) void tcp_tasklet_init(void); @@ -452,10 +451,15 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); int tcp_connect(struct sock *sk); +enum tcp_synack_type { + TCP_SYNACK_NORMAL, + TCP_SYNACK_FASTOPEN, + TCP_SYNACK_COOKIE, +}; struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, struct request_sock *req, struct tcp_fastopen_cookie *foc, - bool attach_req); + enum tcp_synack_type synack_type); int tcp_disconnect(struct sock *sk, int flags); void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); @@ -533,8 +537,8 @@ __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, int nonagle); bool tcp_may_send_now(struct sock *sk); -int __tcp_retransmit_skb(struct sock *, struct sk_buff *); -int tcp_retransmit_skb(struct sock *, struct sk_buff *); +int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); +int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); void tcp_retransmit_timer(struct sock *sk); void tcp_xmit_retransmit_queue(struct sock *); void tcp_simple_retransmit(struct sock *); @@ -552,6 +556,8 @@ void tcp_send_ack(struct sock *sk); void tcp_send_delayed_ack(struct sock *sk); void tcp_send_loss_probe(struct sock *sk); bool tcp_schedule_loss_probe(struct sock *sk); +void tcp_skb_collapse_tstamp(struct sk_buff *skb, + const struct sk_buff *next_skb); /* tcp_input.c */ void tcp_resume_early_retransmit(struct sock *sk); @@ -755,14 +761,20 @@ struct tcp_skb_cb { __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ - unused:7; + eor:1, /* Is skb MSG_EOR marked? */ + unused:6; __u32 ack_seq; /* Sequence number ACK'd */ union { - struct inet_skb_parm h4; + struct { + /* There is space for up to 20 bytes */ + } tx; /* only used for outgoing skbs */ + union { + struct inet_skb_parm h4; #if IS_ENABLED(CONFIG_IPV6) - struct inet6_skb_parm h6; + struct inet6_skb_parm h6; #endif - } header; /* For incoming frames */ + } header; /* For incoming skbs */ + }; }; #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) @@ -774,7 +786,9 @@ struct tcp_skb_cb { */ static inline int tcp_v6_iif(const struct sk_buff *skb) { - return TCP_SKB_CB(skb)->header.h6.iif; + bool l3_slave = skb_l3mdev_slave(TCP_SKB_CB(skb)->header.h6.flags); + + return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; } #endif @@ -802,6 +816,11 @@ static inline int tcp_skb_mss(const struct sk_buff *skb) return TCP_SKB_CB(skb)->tcp_gso_size; } +static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) +{ + return likely(!TCP_SKB_CB(skb)->eor); +} + /* Events passed to congestion control interface */ enum tcp_ca_event { CA_EVENT_TX_START, /* first transmit when no packets in flight */ @@ -837,6 +856,11 @@ enum tcp_ca_ack_event_flags { union tcp_cc_info; +struct ack_sample { + u32 pkts_acked; + s32 rtt_us; +}; + struct tcp_congestion_ops { struct list_head list; u32 key; @@ -860,7 +884,7 @@ struct tcp_congestion_ops { /* new value of cwnd after loss (optional) */ u32 (*undo_cwnd)(struct sock *sk); /* hook for packet ack accounting (optional) */ - void (*pkts_acked)(struct sock *sk, u32 num_acked, s32 rtt_us); + void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); /* get info for inet_diag (optional) */ size_t (*get_info)(struct sock *sk, u32 ext, int *attr, union tcp_cc_info *info); @@ -1291,10 +1315,10 @@ bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, static inline void tcp_mib_init(struct net *net) { /* See RFC 2012 */ - TCP_ADD_STATS_USER(net, TCP_MIB_RTOALGORITHM, 1); - TCP_ADD_STATS_USER(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); - TCP_ADD_STATS_USER(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); - TCP_ADD_STATS_USER(net, TCP_MIB_MAXCONN, -1); + TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); + TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); + TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); + TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); } /* from STCP */ @@ -1728,7 +1752,7 @@ struct tcp_request_sock_ops { int (*send_synack)(const struct sock *sk, struct dst_entry *dst, struct flowi *fl, struct request_sock *req, struct tcp_fastopen_cookie *foc, - bool attach_req); + enum tcp_synack_type synack_type); }; #ifdef CONFIG_SYN_COOKIES @@ -1737,7 +1761,7 @@ static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, __u16 *mss) { tcp_synq_overflow(sk); - NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); + __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); return ops->cookie_init_seq(skb, mss); } #else @@ -1846,7 +1870,7 @@ static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) static inline void tcp_listendrop(const struct sock *sk) { atomic_inc(&((struct sock *)sk)->sk_drops); - NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); + __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); } #endif /* _TCP_H */ diff --git a/include/net/transp_v6.h b/include/net/transp_v6.h index 2b1c3450ab20..276f9760ab56 100644 --- a/include/net/transp_v6.h +++ b/include/net/transp_v6.h @@ -41,8 +41,7 @@ void ip6_datagram_recv_specific_ctl(struct sock *sk, struct msghdr *msg, struct sk_buff *skb); int ip6_datagram_send_ctl(struct net *net, struct sock *sk, struct msghdr *msg, - struct flowi6 *fl6, struct ipv6_txoptions *opt, - int *hlimit, int *tclass, int *dontfrag, + struct flowi6 *fl6, struct ipcm6_cookie *ipc6, struct sockcm_cookie *sockc); void ip6_dgram_sock_seq_show(struct seq_file *seq, struct sock *sp, diff --git a/include/net/udp.h b/include/net/udp.h index 3c5a65e0946d..ae07f375370d 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -289,32 +289,32 @@ struct sock *udp6_lib_lookup_skb(struct sk_buff *skb, /* * SNMP statistics for UDP and UDP-Lite */ -#define UDP_INC_STATS_USER(net, field, is_udplite) do { \ - if (is_udplite) SNMP_INC_STATS_USER((net)->mib.udplite_statistics, field); \ - else SNMP_INC_STATS_USER((net)->mib.udp_statistics, field); } while(0) -#define UDP_INC_STATS_BH(net, field, is_udplite) do { \ - if (is_udplite) SNMP_INC_STATS_BH((net)->mib.udplite_statistics, field); \ - else SNMP_INC_STATS_BH((net)->mib.udp_statistics, field); } while(0) - -#define UDP6_INC_STATS_BH(net, field, is_udplite) do { \ - if (is_udplite) SNMP_INC_STATS_BH((net)->mib.udplite_stats_in6, field);\ - else SNMP_INC_STATS_BH((net)->mib.udp_stats_in6, field); \ +#define UDP_INC_STATS(net, field, is_udplite) do { \ + if (is_udplite) SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ + else SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0) +#define __UDP_INC_STATS(net, field, is_udplite) do { \ + if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ + else __SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0) + +#define __UDP6_INC_STATS(net, field, is_udplite) do { \ + if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);\ + else __SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \ } while(0) -#define UDP6_INC_STATS_USER(net, field, __lite) do { \ - if (__lite) SNMP_INC_STATS_USER((net)->mib.udplite_stats_in6, field); \ - else SNMP_INC_STATS_USER((net)->mib.udp_stats_in6, field); \ +#define UDP6_INC_STATS(net, field, __lite) do { \ + if (__lite) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field); \ + else SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \ } while(0) #if IS_ENABLED(CONFIG_IPV6) -#define UDPX_INC_STATS_BH(sk, field) \ +#define __UDPX_INC_STATS(sk, field) \ do { \ if ((sk)->sk_family == AF_INET) \ - UDP_INC_STATS_BH(sock_net(sk), field, 0); \ + __UDP_INC_STATS(sock_net(sk), field, 0); \ else \ - UDP6_INC_STATS_BH(sock_net(sk), field, 0); \ + __UDP6_INC_STATS(sock_net(sk), field, 0); \ } while (0) #else -#define UDPX_INC_STATS_BH(sk, field) UDP_INC_STATS_BH(sock_net(sk), field, 0) +#define __UDPX_INC_STATS(sk, field) __UDP_INC_STATS(sock_net(sk), field, 0) #endif /* /proc */ diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index 2dcf1de948ac..9d14f707e534 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -105,23 +105,13 @@ struct metadata_dst *udp_tun_rx_dst(struct sk_buff *skb, unsigned short family, __be16 flags, __be64 tunnel_id, int md_size); -static inline struct sk_buff *udp_tunnel_handle_offloads(struct sk_buff *skb, - bool udp_csum) +static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum) { int type = udp_csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL; return iptunnel_handle_offloads(skb, type); } -static inline void udp_tunnel_gro_complete(struct sk_buff *skb, int nhoff) -{ - struct udphdr *uh; - - uh = (struct udphdr *)(skb->data + nhoff - sizeof(struct udphdr)); - skb_shinfo(skb)->gso_type |= uh->check ? - SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL; -} - static inline void udp_tunnel_encap_enable(struct socket *sock) { #if IS_ENABLED(CONFIG_IPV6) diff --git a/include/net/vxlan.h b/include/net/vxlan.h index 2f168f0ea32c..b8803165df91 100644 --- a/include/net/vxlan.h +++ b/include/net/vxlan.h @@ -184,9 +184,7 @@ struct vxlan_metadata { /* per UDP socket information */ struct vxlan_sock { struct hlist_node hlist; - struct work_struct del_work; struct socket *sock; - struct rcu_head rcu; struct hlist_head vni_list[VNI_HASH_SIZE]; atomic_t refcnt; u32 flags; @@ -319,7 +317,9 @@ static inline netdev_features_t vxlan_features_check(struct sk_buff *skb, (skb->inner_protocol_type != ENCAP_TYPE_ETHER || skb->inner_protocol != htons(ETH_P_TEB) || (skb_inner_mac_header(skb) - skb_transport_header(skb) != - sizeof(struct udphdr) + sizeof(struct vxlanhdr)))) + sizeof(struct udphdr) + sizeof(struct vxlanhdr)) || + (skb->ip_summed != CHECKSUM_NONE && + !can_checksum_protocol(features, inner_eth_hdr(skb)->h_proto)))) return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); return features; @@ -392,13 +392,11 @@ static inline __be32 vxlan_compute_rco(unsigned int start, unsigned int offset) return vni_field; } -#if IS_ENABLED(CONFIG_VXLAN) -void vxlan_get_rx_port(struct net_device *netdev); -#else static inline void vxlan_get_rx_port(struct net_device *netdev) { + ASSERT_RTNL(); + call_netdevice_notifiers(NETDEV_OFFLOAD_PUSH_VXLAN, netdev); } -#endif static inline unsigned short vxlan_get_sk_family(struct vxlan_sock *vs) { diff --git a/include/net/xfrm.h b/include/net/xfrm.h index d6f6e5006ee9..adfebd6f243c 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -45,12 +45,8 @@ #ifdef CONFIG_XFRM_STATISTICS #define XFRM_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.xfrm_statistics, field) -#define XFRM_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.xfrm_statistics, field) -#define XFRM_INC_STATS_USER(net, field) SNMP_INC_STATS_USER((net)-mib.xfrm_statistics, field) #else #define XFRM_INC_STATS(net, field) ((void)(net)) -#define XFRM_INC_STATS_BH(net, field) ((void)(net)) -#define XFRM_INC_STATS_USER(net, field) ((void)(net)) #endif |