From: Ilya Maximets
Date: Thu, 21 Jan 2016 14:15:18 +0000 (+0300)
Subject: dpif-netdev: Allow different numbers of rx queues for different ports.
X-Git-Url: http://git.cascardo.info/?a=commitdiff_plain;h=a14b8947fd13d4c587addbffd24eedc7bb48ee2b;p=cascardo%2Fovs.git
dpif-netdev: Allow different numbers of rx queues for different ports.
Currently, all of the PMD netdevs can only have the same number of
rx queues, which is specified in other_config:n-dpdk-rxqs.
Fix that by introducing of new option for PMD interfaces: 'n_rxq', which
specifies the maximum number of rx queues to be created for this
interface.
Example:
ovs-vsctl set Interface dpdk0 options:n_rxq=8
Old 'other_config:n-dpdk-rxqs' deleted.
Signed-off-by: Ilya Maximets
Acked-by: Ben Pfaff
Acked-by: Flavio Leitner
Signed-off-by: Daniele Di Proietto
---
diff --git a/INSTALL.DPDK.md b/INSTALL.DPDK.md
index e8ef4b562..d8927880c 100644
--- a/INSTALL.DPDK.md
+++ b/INSTALL.DPDK.md
@@ -294,9 +294,9 @@ Performance Tuning:
3. DPDK port Rx Queues
- `ovs-vsctl set Open_vSwitch . other_config:n-dpdk-rxqs=`
+ `ovs-vsctl set Interface options:n_rxq=`
- The command above sets the number of rx queues for each DPDK interface.
+ The command above sets the number of rx queues for DPDK interface.
The rx queues are assigned to pmd threads on the same NUMA node in a
round-robin fashion. For more information, please refer to the
Open_vSwitch TABLE section in
@@ -568,9 +568,16 @@ Follow the steps below to attach vhost-user port(s) to a VM.
```
3. Optional: Enable multiqueue support
- QEMU needs to be configured with multiple queues and the number queues
- must be less or equal to Open vSwitch other_config:n-dpdk-rxqs.
- The $q below is the number of queues.
+ The vhost-user interface must be configured in Open vSwitch with the
+ desired amount of queues with:
+
+ ```
+ ovs-vsctl set Interface vhost-user-2 options:n_rxq=
+ ```
+
+ QEMU needs to be configured as well.
+ The $q below should match the queues requested in OVS (if $q is more,
+ packets will not be received).
The $v is the number of vectors, which is '$q x 2 + 2'.
```
diff --git a/NEWS b/NEWS
index e683dae5f..c133838fb 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,10 @@ Post-v2.5.0
* OpenFlow 1.4+ OFPMP_QUEUE_DESC is now supported.
- ovs-ofctl:
* queue-get-config command now allows a queue ID to be specified.
+ - DPDK:
+ * New option "n_rxq" for PMD interfaces.
+ Old 'other_config:n-dpdk-rxqs' is no longer supported.
+
v2.5.0 - xx xxx xxxx
---------------------
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index f233b6103..1b9793bbf 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -221,9 +221,7 @@ struct dp_netdev {
* 'struct dp_netdev_pmd_thread' in 'per_pmd_key'. */
ovsthread_key_t per_pmd_key;
- /* Number of rx queues for each dpdk interface and the cpu mask
- * for pin of pmd threads. */
- size_t n_dpdk_rxqs;
+ /* Cpu mask for pin of pmd threads. */
char *pmd_cmask;
uint64_t last_tnl_conf_seq;
};
@@ -254,6 +252,8 @@ struct dp_netdev_port {
struct netdev_rxq **rxq;
struct ovs_refcount ref_cnt;
char *type; /* Port type as requested by user. */
+ int latest_requested_n_rxq; /* Latest requested from netdev number
+ of rx queues. */
};
/* Contained by struct dp_netdev_flow's 'stats' member. */
@@ -866,7 +866,6 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
ovsthread_key_create(&dp->per_pmd_key, NULL);
dp_netdev_set_nonpmd(dp);
- dp->n_dpdk_rxqs = NR_QUEUE;
ovs_mutex_lock(&dp->port_mutex);
error = do_add_port(dp, name, "internal", ODPP_LOCAL);
@@ -1094,7 +1093,8 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
/* There can only be ovs_numa_get_n_cores() pmd threads,
* so creates a txq for each, and one extra for the non
* pmd threads. */
- error = netdev_set_multiq(netdev, n_cores + 1, dp->n_dpdk_rxqs);
+ error = netdev_set_multiq(netdev, n_cores + 1,
+ netdev_requested_n_rxq(netdev));
if (error && (error != EOPNOTSUPP)) {
VLOG_ERR("%s, cannot set multiq", devname);
return errno;
@@ -1105,6 +1105,7 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
port->netdev = netdev;
port->rxq = xmalloc(sizeof *port->rxq * netdev_n_rxq(netdev));
port->type = xstrdup(type);
+ port->latest_requested_n_rxq = netdev_requested_n_rxq(netdev);
for (i = 0; i < netdev_n_rxq(netdev); i++) {
error = netdev_rxq_open(netdev, &port->rxq[i], i);
if (error
@@ -2408,32 +2409,42 @@ dpif_netdev_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops)
/* Returns true if the configuration for rx queues or cpu mask
* is changed. */
static bool
-pmd_config_changed(const struct dp_netdev *dp, size_t rxqs, const char *cmask)
+pmd_config_changed(const struct dp_netdev *dp, const char *cmask)
{
- if (dp->n_dpdk_rxqs != rxqs) {
- return true;
- } else {
- if (dp->pmd_cmask != NULL && cmask != NULL) {
- return strcmp(dp->pmd_cmask, cmask);
- } else {
- return (dp->pmd_cmask != NULL || cmask != NULL);
+ struct dp_netdev_port *port;
+
+ CMAP_FOR_EACH (port, node, &dp->ports) {
+ struct netdev *netdev = port->netdev;
+ int requested_n_rxq = netdev_requested_n_rxq(netdev);
+ if (netdev_is_pmd(netdev)
+ && port->latest_requested_n_rxq != requested_n_rxq) {
+ return true;
}
}
+
+ if (dp->pmd_cmask != NULL && cmask != NULL) {
+ return strcmp(dp->pmd_cmask, cmask);
+ } else {
+ return (dp->pmd_cmask != NULL || cmask != NULL);
+ }
}
/* Resets pmd threads if the configuration for 'rxq's or cpu mask changes. */
static int
-dpif_netdev_pmd_set(struct dpif *dpif, unsigned int n_rxqs, const char *cmask)
+dpif_netdev_pmd_set(struct dpif *dpif, const char *cmask)
{
struct dp_netdev *dp = get_dp_netdev(dpif);
- if (pmd_config_changed(dp, n_rxqs, cmask)) {
+ if (pmd_config_changed(dp, cmask)) {
struct dp_netdev_port *port;
dp_netdev_destroy_all_pmds(dp);
CMAP_FOR_EACH (port, node, &dp->ports) {
- if (netdev_is_pmd(port->netdev)) {
+ struct netdev *netdev = port->netdev;
+ int requested_n_rxq = netdev_requested_n_rxq(netdev);
+ if (netdev_is_pmd(port->netdev)
+ && port->latest_requested_n_rxq != requested_n_rxq) {
int i, err;
/* Closes the existing 'rxq's. */
@@ -2445,14 +2456,14 @@ dpif_netdev_pmd_set(struct dpif *dpif, unsigned int n_rxqs, const char *cmask)
/* Sets the new rx queue config. */
err = netdev_set_multiq(port->netdev,
ovs_numa_get_n_cores() + 1,
- n_rxqs);
+ requested_n_rxq);
if (err && (err != EOPNOTSUPP)) {
VLOG_ERR("Failed to set dpdk interface %s rx_queue to:"
" %u", netdev_get_name(port->netdev),
- n_rxqs);
+ requested_n_rxq);
return err;
}
-
+ port->latest_requested_n_rxq = requested_n_rxq;
/* If the set_multiq() above succeeds, reopens the 'rxq's. */
port->rxq = xrealloc(port->rxq, sizeof *port->rxq
* netdev_n_rxq(port->netdev));
@@ -2461,8 +2472,6 @@ dpif_netdev_pmd_set(struct dpif *dpif, unsigned int n_rxqs, const char *cmask)
}
}
}
- dp->n_dpdk_rxqs = n_rxqs;
-
/* Reconfigures the cpu mask. */
ovs_numa_set_cpu_mask(cmask);
free(dp->pmd_cmask);
diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h
index a9844be9e..fbd370fcc 100644
--- a/lib/dpif-provider.h
+++ b/lib/dpif-provider.h
@@ -318,11 +318,9 @@ struct dpif_class {
int (*handlers_set)(struct dpif *dpif, uint32_t n_handlers);
/* If 'dpif' creates its own I/O polling threads, refreshes poll threads
- * configuration. 'n_rxqs' configures the number of rx_queues, which
- * are distributed among threads. 'cmask' configures the cpu mask
- * for setting the polling threads' cpu affinity. */
- int (*poll_threads_set)(struct dpif *dpif, unsigned int n_rxqs,
- const char *cmask);
+ * configuration. 'cmask' configures the cpu mask for setting the polling
+ * threads' cpu affinity. */
+ int (*poll_threads_set)(struct dpif *dpif, const char *cmask);
/* Translates OpenFlow queue ID 'queue_id' (in host byte order) into a
* priority value used for setting packet priority. */
diff --git a/lib/dpif.c b/lib/dpif.c
index 38e40bac9..81cbe1181 100644
--- a/lib/dpif.c
+++ b/lib/dpif.c
@@ -1406,13 +1406,12 @@ dpif_print_packet(struct dpif *dpif, struct dpif_upcall *upcall)
/* If 'dpif' creates its own I/O polling threads, refreshes poll threads
* configuration. */
int
-dpif_poll_threads_set(struct dpif *dpif, unsigned int n_rxqs,
- const char *cmask)
+dpif_poll_threads_set(struct dpif *dpif, const char *cmask)
{
int error = 0;
if (dpif->dpif_class->poll_threads_set) {
- error = dpif->dpif_class->poll_threads_set(dpif, n_rxqs, cmask);
+ error = dpif->dpif_class->poll_threads_set(dpif, cmask);
if (error) {
log_operation(dpif, "poll_threads_set", error);
}
diff --git a/lib/dpif.h b/lib/dpif.h
index 50174ee95..97d5d064c 100644
--- a/lib/dpif.h
+++ b/lib/dpif.h
@@ -836,8 +836,7 @@ void dpif_register_upcall_cb(struct dpif *, upcall_callback *, void *aux);
int dpif_recv_set(struct dpif *, bool enable);
int dpif_handlers_set(struct dpif *, uint32_t n_handlers);
-int dpif_poll_threads_set(struct dpif *, unsigned int n_rxqs,
- const char *cmask);
+int dpif_poll_threads_set(struct dpif *, const char *cmask);
int dpif_recv(struct dpif *, uint32_t handler_id, struct dpif_upcall *,
struct ofpbuf *);
void dpif_recv_purge(struct dpif *);
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index d115de2c7..2a479e2c7 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -614,6 +614,7 @@ netdev_dpdk_init(struct netdev *netdev_, unsigned int port_no,
netdev_->n_txq = NR_QUEUE;
netdev_->n_rxq = NR_QUEUE;
+ netdev_->requested_n_rxq = NR_QUEUE;
netdev->real_n_txq = NR_QUEUE;
if (type == DPDK_DEV_ETH) {
@@ -769,20 +770,35 @@ netdev_dpdk_dealloc(struct netdev *netdev_)
}
static int
-netdev_dpdk_get_config(const struct netdev *netdev_, struct smap *args)
+netdev_dpdk_get_config(const struct netdev *netdev, struct smap *args)
{
- struct netdev_dpdk *dev = netdev_dpdk_cast(netdev_);
+ struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
ovs_mutex_lock(&dev->mutex);
- smap_add_format(args, "configured_rx_queues", "%d", netdev_->n_rxq);
- smap_add_format(args, "requested_tx_queues", "%d", netdev_->n_txq);
+ smap_add_format(args, "requested_rx_queues", "%d", netdev->requested_n_rxq);
+ smap_add_format(args, "configured_rx_queues", "%d", netdev->n_rxq);
+ smap_add_format(args, "requested_tx_queues", "%d", netdev->n_txq);
smap_add_format(args, "configured_tx_queues", "%d", dev->real_n_txq);
ovs_mutex_unlock(&dev->mutex);
return 0;
}
+static int
+netdev_dpdk_set_config(struct netdev *netdev, const struct smap *args)
+{
+ struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
+
+ ovs_mutex_lock(&dev->mutex);
+ netdev->requested_n_rxq = MAX(smap_get_int(args, "n_rxq",
+ netdev->requested_n_rxq), 1);
+ netdev_change_seq_changed(netdev);
+ ovs_mutex_unlock(&dev->mutex);
+
+ return 0;
+}
+
static int
netdev_dpdk_get_numa_id(const struct netdev *netdev_)
{
@@ -2127,7 +2143,7 @@ unlock_dpdk:
DESTRUCT, \
netdev_dpdk_dealloc, \
netdev_dpdk_get_config, \
- NULL, /* netdev_dpdk_set_config */ \
+ netdev_dpdk_set_config, \
NULL, /* get_tunnel_config */ \
NULL, /* build header */ \
NULL, /* push header */ \
diff --git a/lib/netdev-provider.h b/lib/netdev-provider.h
index a33bb3b3a..d324ffc92 100644
--- a/lib/netdev-provider.h
+++ b/lib/netdev-provider.h
@@ -52,9 +52,13 @@ struct netdev {
* 'netdev''s flags, features, ethernet address, or carrier changes. */
uint64_t change_seq;
- /* The following are protected by 'netdev_mutex' (internal to netdev.c). */
+ /* The core netdev code initializes these at netdev construction and only
+ * provide read-only access to its client. Netdev implementations may
+ * modify them. */
int n_txq;
int n_rxq;
+ /* Number of rx queues requested by user. */
+ int requested_n_rxq;
int ref_cnt; /* Times this devices was opened. */
struct shash_node *node; /* Pointer to element in global map. */
struct ovs_list saved_flags_list; /* Contains "struct netdev_saved_flags". */
diff --git a/lib/netdev.c b/lib/netdev.c
index e3b70b175..c250c93ae 100644
--- a/lib/netdev.c
+++ b/lib/netdev.c
@@ -106,6 +106,12 @@ netdev_n_rxq(const struct netdev *netdev)
return netdev->n_rxq;
}
+int
+netdev_requested_n_rxq(const struct netdev *netdev)
+{
+ return netdev->requested_n_rxq;
+}
+
bool
netdev_is_pmd(const struct netdev *netdev)
{
@@ -376,6 +382,7 @@ netdev_open(const char *name, const char *type, struct netdev **netdevp)
/* By default enable one tx and rx queue per netdev. */
netdev->n_txq = netdev->netdev_class->send ? 1 : 0;
netdev->n_rxq = netdev->netdev_class->rxq_alloc ? 1 : 0;
+ netdev->requested_n_rxq = netdev->n_rxq;
list_init(&netdev->saved_flags_list);
diff --git a/lib/netdev.h b/lib/netdev.h
index 622e2ae2c..8a7f68004 100644
--- a/lib/netdev.h
+++ b/lib/netdev.h
@@ -142,6 +142,7 @@ bool netdev_is_reserved_name(const char *name);
int netdev_n_txq(const struct netdev *netdev);
int netdev_n_rxq(const struct netdev *netdev);
+int netdev_requested_n_rxq(const struct netdev *netdev);
bool netdev_is_pmd(const struct netdev *netdev);
/* Open and close. */
diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index 89e06aa9c..904cc2a5c 100644
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -564,7 +564,7 @@ type_run(const char *type)
udpif_set_threads(backer->udpif, n_handlers, n_revalidators);
}
- dpif_poll_threads_set(backer->dpif, n_dpdk_rxqs, pmd_cpu_mask);
+ dpif_poll_threads_set(backer->dpif, pmd_cpu_mask);
if (backer->need_revalidate) {
struct ofproto_dpif *ofproto;
diff --git a/ofproto/ofproto-provider.h b/ofproto/ofproto-provider.h
index b6aac0a4f..3ba97d042 100644
--- a/ofproto/ofproto-provider.h
+++ b/ofproto/ofproto-provider.h
@@ -482,9 +482,6 @@ extern unsigned ofproto_max_idle;
* ofproto-dpif implementation. */
extern size_t n_handlers, n_revalidators;
-/* Number of rx queues to be created for each dpdk interface. */
-extern size_t n_dpdk_rxqs;
-
/* Cpu mask for pmd threads. */
extern char *pmd_cpu_mask;
diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c
index bba30ae16..939cb3714 100644
--- a/ofproto/ofproto.c
+++ b/ofproto/ofproto.c
@@ -337,7 +337,6 @@ unsigned ofproto_flow_limit = OFPROTO_FLOW_LIMIT_DEFAULT;
unsigned ofproto_max_idle = OFPROTO_MAX_IDLE_DEFAULT;
size_t n_handlers, n_revalidators;
-size_t n_dpdk_rxqs;
char *pmd_cpu_mask;
/* Map from datapath name to struct ofproto, for use by unixctl commands. */
@@ -780,12 +779,6 @@ ofproto_port_set_mcast_snooping(struct ofproto *ofproto, void *aux,
: EOPNOTSUPP);
}
-void
-ofproto_set_n_dpdk_rxqs(int n_rxqs)
-{
- n_dpdk_rxqs = MAX(n_rxqs, 0);
-}
-
void
ofproto_set_cpu_mask(const char *cmask)
{
diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h
index 7504027c2..b99f0cd9a 100644
--- a/ofproto/ofproto.h
+++ b/ofproto/ofproto.h
@@ -316,7 +316,6 @@ int ofproto_set_mcast_snooping(struct ofproto *ofproto,
int ofproto_port_set_mcast_snooping(struct ofproto *ofproto, void *aux,
const struct ofproto_mcast_snooping_port_settings *s);
void ofproto_set_threads(int n_handlers, int n_revalidators);
-void ofproto_set_n_dpdk_rxqs(int n_rxqs);
void ofproto_set_cpu_mask(const char *cmask);
void ofproto_set_dp_desc(struct ofproto *, const char *dp_desc);
int ofproto_set_snoops(struct ofproto *, const struct sset *snoops);
diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
index 2ff23223f..c6fa44526 100644
--- a/vswitchd/bridge.c
+++ b/vswitchd/bridge.c
@@ -581,8 +581,6 @@ bridge_reconfigure(const struct ovsrec_open_vswitch *ovs_cfg)
OFPROTO_FLOW_LIMIT_DEFAULT));
ofproto_set_max_idle(smap_get_int(&ovs_cfg->other_config, "max-idle",
OFPROTO_MAX_IDLE_DEFAULT));
- ofproto_set_n_dpdk_rxqs(smap_get_int(&ovs_cfg->other_config,
- "n-dpdk-rxqs", 0));
ofproto_set_cpu_mask(smap_get(&ovs_cfg->other_config, "pmd-cpu-mask"));
ofproto_set_threads(
diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index e7c7e3c78..c2ec914df 100644
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -167,15 +167,6 @@
-
-
- Specifies the maximum number of rx queues to be created for each dpdk
- interface. If not specified or specified to 0, one rx queue will
- be created for each dpdk interface by default.
-
-
-
Specifies CPU mask for setting the cpu affinity of PMD (Poll
@@ -2197,6 +2188,21 @@
+
+
+ Only PMD netdevs support these options.
+
+
+
+
+ Specifies the maximum number of rx queues to be created for PMD
+ netdev. If not specified or specified to 0, one rx queue will
+ be created by default.
+
+
+
+
Status information about interfaces attached to bridges, updated every