2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/rhashtable.h>
40 #include <linux/bitops.h>
41 #include <linux/in6.h>
42 #include <linux/notifier.h>
43 #include <net/netevent.h>
44 #include <net/neighbour.h>
46 #include <net/ip_fib.h>
52 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
53 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
56 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
57 struct mlxsw_sp_prefix_usage *prefix_usage2)
61 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
62 if (!test_bit(prefix, prefix_usage2->b))
69 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
70 struct mlxsw_sp_prefix_usage *prefix_usage2)
72 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
76 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
78 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
80 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
84 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
85 struct mlxsw_sp_prefix_usage *prefix_usage2)
87 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
91 mlxsw_sp_prefix_usage_zero(struct mlxsw_sp_prefix_usage *prefix_usage)
93 memset(prefix_usage, 0, sizeof(*prefix_usage));
97 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
98 unsigned char prefix_len)
100 set_bit(prefix_len, prefix_usage->b);
104 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
105 unsigned char prefix_len)
107 clear_bit(prefix_len, prefix_usage->b);
110 struct mlxsw_sp_fib_key {
111 struct net_device *dev;
112 unsigned char addr[sizeof(struct in6_addr)];
113 unsigned char prefix_len;
116 enum mlxsw_sp_fib_entry_type {
117 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
118 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
119 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
122 struct mlxsw_sp_nexthop_group;
124 struct mlxsw_sp_fib_entry {
125 struct rhash_head ht_node;
126 struct list_head list;
127 struct mlxsw_sp_fib_key key;
128 enum mlxsw_sp_fib_entry_type type;
129 unsigned int ref_count;
130 u16 rif; /* used for action local */
131 struct mlxsw_sp_vr *vr;
133 struct list_head nexthop_group_node;
134 struct mlxsw_sp_nexthop_group *nh_group;
137 struct mlxsw_sp_fib {
138 struct rhashtable ht;
139 struct list_head entry_list;
140 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
141 struct mlxsw_sp_prefix_usage prefix_usage;
144 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
145 .key_offset = offsetof(struct mlxsw_sp_fib_entry, key),
146 .head_offset = offsetof(struct mlxsw_sp_fib_entry, ht_node),
147 .key_len = sizeof(struct mlxsw_sp_fib_key),
148 .automatic_shrinking = true,
151 static int mlxsw_sp_fib_entry_insert(struct mlxsw_sp_fib *fib,
152 struct mlxsw_sp_fib_entry *fib_entry)
154 unsigned char prefix_len = fib_entry->key.prefix_len;
157 err = rhashtable_insert_fast(&fib->ht, &fib_entry->ht_node,
158 mlxsw_sp_fib_ht_params);
161 list_add_tail(&fib_entry->list, &fib->entry_list);
162 if (fib->prefix_ref_count[prefix_len]++ == 0)
163 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
167 static void mlxsw_sp_fib_entry_remove(struct mlxsw_sp_fib *fib,
168 struct mlxsw_sp_fib_entry *fib_entry)
170 unsigned char prefix_len = fib_entry->key.prefix_len;
172 if (--fib->prefix_ref_count[prefix_len] == 0)
173 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
174 list_del(&fib_entry->list);
175 rhashtable_remove_fast(&fib->ht, &fib_entry->ht_node,
176 mlxsw_sp_fib_ht_params);
179 static struct mlxsw_sp_fib_entry *
180 mlxsw_sp_fib_entry_create(struct mlxsw_sp_fib *fib, const void *addr,
181 size_t addr_len, unsigned char prefix_len,
182 struct net_device *dev)
184 struct mlxsw_sp_fib_entry *fib_entry;
186 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
189 fib_entry->key.dev = dev;
190 memcpy(fib_entry->key.addr, addr, addr_len);
191 fib_entry->key.prefix_len = prefix_len;
195 static void mlxsw_sp_fib_entry_destroy(struct mlxsw_sp_fib_entry *fib_entry)
200 static struct mlxsw_sp_fib_entry *
201 mlxsw_sp_fib_entry_lookup(struct mlxsw_sp_fib *fib, const void *addr,
202 size_t addr_len, unsigned char prefix_len,
203 struct net_device *dev)
205 struct mlxsw_sp_fib_key key;
207 memset(&key, 0, sizeof(key));
209 memcpy(key.addr, addr, addr_len);
210 key.prefix_len = prefix_len;
211 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
214 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
216 struct mlxsw_sp_fib *fib;
219 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
221 return ERR_PTR(-ENOMEM);
222 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
224 goto err_rhashtable_init;
225 INIT_LIST_HEAD(&fib->entry_list);
233 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
235 rhashtable_destroy(&fib->ht);
239 static struct mlxsw_sp_lpm_tree *
240 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp, bool one_reserved)
242 static struct mlxsw_sp_lpm_tree *lpm_tree;
245 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
246 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
247 if (lpm_tree->ref_count == 0) {
249 one_reserved = false;
257 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
258 struct mlxsw_sp_lpm_tree *lpm_tree)
260 char ralta_pl[MLXSW_REG_RALTA_LEN];
262 mlxsw_reg_ralta_pack(ralta_pl, true,
263 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
265 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
268 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
269 struct mlxsw_sp_lpm_tree *lpm_tree)
271 char ralta_pl[MLXSW_REG_RALTA_LEN];
273 mlxsw_reg_ralta_pack(ralta_pl, false,
274 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
276 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
280 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
281 struct mlxsw_sp_prefix_usage *prefix_usage,
282 struct mlxsw_sp_lpm_tree *lpm_tree)
284 char ralst_pl[MLXSW_REG_RALST_LEN];
287 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
289 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
292 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
293 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
296 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
297 MLXSW_REG_RALST_BIN_NO_CHILD);
298 last_prefix = prefix;
300 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
303 static struct mlxsw_sp_lpm_tree *
304 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
305 struct mlxsw_sp_prefix_usage *prefix_usage,
306 enum mlxsw_sp_l3proto proto, bool one_reserved)
308 struct mlxsw_sp_lpm_tree *lpm_tree;
311 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp, one_reserved);
313 return ERR_PTR(-EBUSY);
314 lpm_tree->proto = proto;
315 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
319 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
322 goto err_left_struct_set;
326 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
330 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
331 struct mlxsw_sp_lpm_tree *lpm_tree)
333 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
336 static struct mlxsw_sp_lpm_tree *
337 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
338 struct mlxsw_sp_prefix_usage *prefix_usage,
339 enum mlxsw_sp_l3proto proto, bool one_reserved)
341 struct mlxsw_sp_lpm_tree *lpm_tree;
344 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
345 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
346 if (lpm_tree->proto == proto &&
347 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
351 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
352 proto, one_reserved);
353 if (IS_ERR(lpm_tree))
357 lpm_tree->ref_count++;
361 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
362 struct mlxsw_sp_lpm_tree *lpm_tree)
364 if (--lpm_tree->ref_count == 0)
365 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
369 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
371 struct mlxsw_sp_lpm_tree *lpm_tree;
374 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
375 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
376 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
380 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
382 struct mlxsw_resources *resources;
383 struct mlxsw_sp_vr *vr;
386 resources = mlxsw_core_resources_get(mlxsw_sp->core);
387 for (i = 0; i < resources->max_virtual_routers; i++) {
388 vr = &mlxsw_sp->router.vrs[i];
395 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
396 struct mlxsw_sp_vr *vr)
398 char raltb_pl[MLXSW_REG_RALTB_LEN];
400 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
401 (enum mlxsw_reg_ralxx_protocol) vr->proto,
403 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
406 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
407 struct mlxsw_sp_vr *vr)
409 char raltb_pl[MLXSW_REG_RALTB_LEN];
411 /* Bind to tree 0 which is default */
412 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
413 (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
414 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
417 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
419 /* For our purpose, squash main and local table into one */
420 if (tb_id == RT_TABLE_LOCAL)
421 tb_id = RT_TABLE_MAIN;
425 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
427 enum mlxsw_sp_l3proto proto)
429 struct mlxsw_resources *resources;
430 struct mlxsw_sp_vr *vr;
433 tb_id = mlxsw_sp_fix_tb_id(tb_id);
435 resources = mlxsw_core_resources_get(mlxsw_sp->core);
436 for (i = 0; i < resources->max_virtual_routers; i++) {
437 vr = &mlxsw_sp->router.vrs[i];
438 if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
444 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
445 unsigned char prefix_len,
447 enum mlxsw_sp_l3proto proto)
449 struct mlxsw_sp_prefix_usage req_prefix_usage;
450 struct mlxsw_sp_lpm_tree *lpm_tree;
451 struct mlxsw_sp_vr *vr;
454 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
456 return ERR_PTR(-EBUSY);
457 vr->fib = mlxsw_sp_fib_create();
459 return ERR_CAST(vr->fib);
463 mlxsw_sp_prefix_usage_zero(&req_prefix_usage);
464 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
465 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
467 if (IS_ERR(lpm_tree)) {
468 err = PTR_ERR(lpm_tree);
471 vr->lpm_tree = lpm_tree;
472 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
480 mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
482 mlxsw_sp_fib_destroy(vr->fib);
487 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
488 struct mlxsw_sp_vr *vr)
490 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
491 mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
492 mlxsw_sp_fib_destroy(vr->fib);
497 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
498 struct mlxsw_sp_prefix_usage *req_prefix_usage)
500 struct mlxsw_sp_lpm_tree *lpm_tree;
502 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage,
503 &vr->lpm_tree->prefix_usage))
506 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
508 if (IS_ERR(lpm_tree)) {
509 /* We failed to get a tree according to the required
510 * prefix usage. However, the current tree might be still good
511 * for us if our requirement is subset of the prefixes used
514 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
515 &vr->lpm_tree->prefix_usage))
517 return PTR_ERR(lpm_tree);
520 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
521 mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
522 vr->lpm_tree = lpm_tree;
523 return mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
526 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
527 unsigned char prefix_len,
529 enum mlxsw_sp_l3proto proto)
531 struct mlxsw_sp_vr *vr;
534 tb_id = mlxsw_sp_fix_tb_id(tb_id);
535 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id, proto);
537 vr = mlxsw_sp_vr_create(mlxsw_sp, prefix_len, tb_id, proto);
541 struct mlxsw_sp_prefix_usage req_prefix_usage;
543 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
544 &vr->fib->prefix_usage);
545 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
546 /* Need to replace LPM tree in case new prefix is required. */
547 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
555 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
557 /* Destroy virtual router entity in case the associated FIB is empty
558 * and allow it to be used for other tables in future. Otherwise,
559 * check if some prefix usage did not disappear and change tree if
560 * that is the case. Note that in case new, smaller tree cannot be
561 * allocated, the original one will be kept being used.
563 if (mlxsw_sp_prefix_usage_none(&vr->fib->prefix_usage))
564 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
566 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
567 &vr->fib->prefix_usage);
570 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
572 struct mlxsw_resources *resources;
573 struct mlxsw_sp_vr *vr;
576 resources = mlxsw_core_resources_get(mlxsw_sp->core);
577 if (!resources->max_virtual_routers_valid)
580 mlxsw_sp->router.vrs = kcalloc(resources->max_virtual_routers,
581 sizeof(struct mlxsw_sp_vr),
583 if (!mlxsw_sp->router.vrs)
586 for (i = 0; i < resources->max_virtual_routers; i++) {
587 vr = &mlxsw_sp->router.vrs[i];
594 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
596 kfree(mlxsw_sp->router.vrs);
599 struct mlxsw_sp_neigh_key {
600 unsigned char addr[sizeof(struct in6_addr)];
601 struct net_device *dev;
604 struct mlxsw_sp_neigh_entry {
605 struct rhash_head ht_node;
606 struct mlxsw_sp_neigh_key key;
610 struct delayed_work dw;
611 struct mlxsw_sp_port *mlxsw_sp_port;
612 unsigned char ha[ETH_ALEN];
613 struct list_head nexthop_list; /* list of nexthops using
616 struct list_head nexthop_neighs_list_node;
619 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
620 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
621 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
622 .key_len = sizeof(struct mlxsw_sp_neigh_key),
626 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
627 struct mlxsw_sp_neigh_entry *neigh_entry)
629 return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
630 &neigh_entry->ht_node,
631 mlxsw_sp_neigh_ht_params);
635 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
636 struct mlxsw_sp_neigh_entry *neigh_entry)
638 rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
639 &neigh_entry->ht_node,
640 mlxsw_sp_neigh_ht_params);
643 static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work);
645 static struct mlxsw_sp_neigh_entry *
646 mlxsw_sp_neigh_entry_create(const void *addr, size_t addr_len,
647 struct net_device *dev, u16 rif,
650 struct mlxsw_sp_neigh_entry *neigh_entry;
652 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_ATOMIC);
655 memcpy(neigh_entry->key.addr, addr, addr_len);
656 neigh_entry->key.dev = dev;
657 neigh_entry->rif = rif;
659 INIT_DELAYED_WORK(&neigh_entry->dw, mlxsw_sp_router_neigh_update_hw);
660 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
665 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp_neigh_entry *neigh_entry)
670 static struct mlxsw_sp_neigh_entry *
671 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, const void *addr,
672 size_t addr_len, struct net_device *dev)
674 struct mlxsw_sp_neigh_key key = {{ 0 } };
676 memcpy(key.addr, addr, addr_len);
678 return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
679 &key, mlxsw_sp_neigh_ht_params);
682 int mlxsw_sp_router_neigh_construct(struct net_device *dev,
685 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
686 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
687 struct mlxsw_sp_neigh_entry *neigh_entry;
688 struct mlxsw_sp_rif *r;
692 if (n->tbl != &arp_tbl)
695 dip = ntohl(*((__be32 *) n->primary_key));
696 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, &dip, sizeof(dip),
699 WARN_ON(neigh_entry->n != n);
703 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
707 neigh_entry = mlxsw_sp_neigh_entry_create(&dip, sizeof(dip), n->dev,
711 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
713 goto err_neigh_entry_insert;
716 err_neigh_entry_insert:
717 mlxsw_sp_neigh_entry_destroy(neigh_entry);
721 void mlxsw_sp_router_neigh_destroy(struct net_device *dev,
724 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
725 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
726 struct mlxsw_sp_neigh_entry *neigh_entry;
729 if (n->tbl != &arp_tbl)
732 dip = ntohl(*((__be32 *) n->primary_key));
733 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, &dip, sizeof(dip),
737 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
738 mlxsw_sp_neigh_entry_destroy(neigh_entry);
742 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
744 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
746 mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
749 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
753 struct net_device *dev;
759 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
761 if (!mlxsw_sp->rifs[rif]) {
762 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
767 dev = mlxsw_sp->rifs[rif]->dev;
768 n = neigh_lookup(&arp_tbl, &dipn, dev);
770 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
775 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
776 neigh_event_send(n, NULL);
780 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
787 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
789 /* Hardware starts counting at 0, so add 1. */
792 /* Each record consists of several neighbour entries. */
793 for (i = 0; i < num_entries; i++) {
796 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
797 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
803 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
804 char *rauhtd_pl, int rec_index)
806 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
807 case MLXSW_REG_RAUHTD_TYPE_IPV4:
808 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
811 case MLXSW_REG_RAUHTD_TYPE_IPV6:
817 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
823 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
827 /* Make sure the neighbour's netdev isn't removed in the
832 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
833 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
836 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
839 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
840 for (i = 0; i < num_rec; i++)
841 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
850 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
852 struct mlxsw_sp_neigh_entry *neigh_entry;
854 /* Take RTNL mutex here to prevent lists from changes */
856 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
857 nexthop_neighs_list_node) {
858 /* If this neigh have nexthops, make the kernel think this neigh
859 * is active regardless of the traffic.
861 if (!list_empty(&neigh_entry->nexthop_list))
862 neigh_event_send(neigh_entry->n, NULL);
868 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
870 unsigned long interval = mlxsw_sp->router.neighs_update.interval;
872 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
873 msecs_to_jiffies(interval));
876 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
878 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
879 router.neighs_update.dw.work);
882 err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
884 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
886 mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
888 mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
891 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
893 struct mlxsw_sp_neigh_entry *neigh_entry;
894 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
895 router.nexthop_probe_dw.work);
897 /* Iterate over nexthop neighbours, find those who are unresolved and
898 * send arp on them. This solves the chicken-egg problem when
899 * the nexthop wouldn't get offloaded until the neighbor is resolved
900 * but it wouldn't get resolved ever in case traffic is flowing in HW
901 * using different nexthop.
903 * Take RTNL mutex here to prevent lists from changes.
906 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
907 nexthop_neighs_list_node) {
908 if (!(neigh_entry->n->nud_state & NUD_VALID) &&
909 !list_empty(&neigh_entry->nexthop_list))
910 neigh_event_send(neigh_entry->n, NULL);
914 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
915 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
919 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
920 struct mlxsw_sp_neigh_entry *neigh_entry,
923 static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work)
925 struct mlxsw_sp_neigh_entry *neigh_entry =
926 container_of(work, struct mlxsw_sp_neigh_entry, dw.work);
927 struct neighbour *n = neigh_entry->n;
928 struct mlxsw_sp_port *mlxsw_sp_port = neigh_entry->mlxsw_sp_port;
929 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
930 char rauht_pl[MLXSW_REG_RAUHT_LEN];
931 struct net_device *dev;
932 bool entry_connected;
940 read_lock_bh(&n->lock);
941 dip = ntohl(*((__be32 *) n->primary_key));
942 memcpy(neigh_entry->ha, n->ha, sizeof(neigh_entry->ha));
943 nud_state = n->nud_state;
945 read_unlock_bh(&n->lock);
947 entry_connected = nud_state & NUD_VALID;
948 adding = (!neigh_entry->offloaded) && entry_connected;
949 updating = neigh_entry->offloaded && entry_connected;
950 removing = neigh_entry->offloaded && !entry_connected;
952 if (adding || updating) {
953 mlxsw_reg_rauht_pack4(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_ADD,
955 neigh_entry->ha, dip);
956 err = mlxsw_reg_write(mlxsw_sp->core,
957 MLXSW_REG(rauht), rauht_pl);
959 netdev_err(dev, "Could not add neigh %pI4h\n", &dip);
960 neigh_entry->offloaded = false;
962 neigh_entry->offloaded = true;
964 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, false);
965 } else if (removing) {
966 mlxsw_reg_rauht_pack4(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE,
968 neigh_entry->ha, dip);
969 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht),
972 netdev_err(dev, "Could not delete neigh %pI4h\n", &dip);
973 neigh_entry->offloaded = true;
975 neigh_entry->offloaded = false;
977 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, true);
981 mlxsw_sp_port_dev_put(mlxsw_sp_port);
984 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
985 unsigned long event, void *ptr)
987 struct mlxsw_sp_neigh_entry *neigh_entry;
988 struct mlxsw_sp_port *mlxsw_sp_port;
989 struct mlxsw_sp *mlxsw_sp;
990 unsigned long interval;
991 struct net_device *dev;
992 struct neigh_parms *p;
997 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1000 /* We don't care about changes in the default table. */
1001 if (!p->dev || p->tbl != &arp_tbl)
1004 /* We are in atomic context and can't take RTNL mutex,
1005 * so use RCU variant to walk the device chain.
1007 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1011 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1012 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1013 mlxsw_sp->router.neighs_update.interval = interval;
1015 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1017 case NETEVENT_NEIGH_UPDATE:
1021 if (n->tbl != &arp_tbl)
1024 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(dev);
1028 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1029 dip = ntohl(*((__be32 *) n->primary_key));
1030 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp,
1034 if (WARN_ON(!neigh_entry) || WARN_ON(neigh_entry->n != n)) {
1035 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1038 neigh_entry->mlxsw_sp_port = mlxsw_sp_port;
1040 /* Take a reference to ensure the neighbour won't be
1041 * destructed until we drop the reference in delayed
1045 if (!mlxsw_core_schedule_dw(&neigh_entry->dw, 0)) {
1047 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1055 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1059 err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1060 &mlxsw_sp_neigh_ht_params);
1064 /* Initialize the polling interval according to the default
1067 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1069 /* Create the delayed works for the activity_update */
1070 INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1071 mlxsw_sp_router_neighs_update_work);
1072 INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1073 mlxsw_sp_router_probe_unresolved_nexthops);
1074 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1075 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1079 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1081 cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1082 cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1083 rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1086 struct mlxsw_sp_nexthop {
1087 struct list_head neigh_list_node; /* member of neigh entry list */
1088 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1091 u8 should_offload:1, /* set indicates this neigh is connected and
1092 * should be put to KVD linear area of this group.
1094 offloaded:1, /* set in case the neigh is actually put into
1095 * KVD linear area of this group.
1097 update:1; /* set indicates that MAC of this neigh should be
1100 struct mlxsw_sp_neigh_entry *neigh_entry;
1103 struct mlxsw_sp_nexthop_group {
1104 struct list_head list; /* node in mlxsw->router.nexthop_group_list */
1105 struct list_head fib_list; /* list of fib entries that use this group */
1106 u8 adj_index_valid:1;
1110 struct mlxsw_sp_nexthop nexthops[0];
1113 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1114 struct mlxsw_sp_vr *vr,
1115 u32 adj_index, u16 ecmp_size,
1119 char raleu_pl[MLXSW_REG_RALEU_LEN];
1121 mlxsw_reg_raleu_pack(raleu_pl,
1122 (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
1123 adj_index, ecmp_size, new_adj_index,
1125 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1128 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1129 struct mlxsw_sp_nexthop_group *nh_grp,
1130 u32 old_adj_index, u16 old_ecmp_size)
1132 struct mlxsw_sp_fib_entry *fib_entry;
1133 struct mlxsw_sp_vr *vr = NULL;
1136 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1137 if (vr == fib_entry->vr)
1140 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr,
1151 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1152 struct mlxsw_sp_nexthop *nh)
1154 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1155 char ratr_pl[MLXSW_REG_RATR_LEN];
1157 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1158 true, adj_index, neigh_entry->rif);
1159 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1160 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1164 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1165 struct mlxsw_sp_nexthop_group *nh_grp)
1167 u32 adj_index = nh_grp->adj_index; /* base */
1168 struct mlxsw_sp_nexthop *nh;
1172 for (i = 0; i < nh_grp->count; i++) {
1173 nh = &nh_grp->nexthops[i];
1175 if (!nh->should_offload) {
1181 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1193 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1194 struct mlxsw_sp_fib_entry *fib_entry);
1197 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1198 struct mlxsw_sp_nexthop_group *nh_grp)
1200 struct mlxsw_sp_fib_entry *fib_entry;
1203 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1204 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1212 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1213 struct mlxsw_sp_nexthop_group *nh_grp)
1215 struct mlxsw_sp_nexthop *nh;
1216 bool offload_change = false;
1219 bool old_adj_index_valid;
1226 for (i = 0; i < nh_grp->count; i++) {
1227 nh = &nh_grp->nexthops[i];
1229 if (nh->should_offload ^ nh->offloaded) {
1230 offload_change = true;
1231 if (nh->should_offload)
1234 if (nh->should_offload)
1237 if (!offload_change) {
1238 /* Nothing was added or removed, so no need to reallocate. Just
1239 * update MAC on existing adjacency indexes.
1241 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp);
1243 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1249 /* No neigh of this group is connected so we just set
1250 * the trap and let everthing flow through kernel.
1254 ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1256 /* We ran out of KVD linear space, just set the
1257 * trap and let everything flow through kernel.
1259 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1263 old_adj_index_valid = nh_grp->adj_index_valid;
1264 old_adj_index = nh_grp->adj_index;
1265 old_ecmp_size = nh_grp->ecmp_size;
1266 nh_grp->adj_index_valid = 1;
1267 nh_grp->adj_index = adj_index;
1268 nh_grp->ecmp_size = ecmp_size;
1269 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp);
1271 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1275 if (!old_adj_index_valid) {
1276 /* The trap was set for fib entries, so we have to call
1277 * fib entry update to unset it and use adjacency index.
1279 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1281 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1287 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1288 old_adj_index, old_ecmp_size);
1289 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1291 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1297 old_adj_index_valid = nh_grp->adj_index_valid;
1298 nh_grp->adj_index_valid = 0;
1299 for (i = 0; i < nh_grp->count; i++) {
1300 nh = &nh_grp->nexthops[i];
1303 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1305 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1306 if (old_adj_index_valid)
1307 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1310 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1313 if (!removing && !nh->should_offload)
1314 nh->should_offload = 1;
1315 else if (removing && nh->offloaded)
1316 nh->should_offload = 0;
1321 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1322 struct mlxsw_sp_neigh_entry *neigh_entry,
1325 struct mlxsw_sp_nexthop *nh;
1327 /* Take RTNL mutex here to prevent lists from changes */
1329 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1331 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1332 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1337 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1338 struct mlxsw_sp_nexthop_group *nh_grp,
1339 struct mlxsw_sp_nexthop *nh,
1340 struct fib_nh *fib_nh)
1342 struct mlxsw_sp_neigh_entry *neigh_entry;
1343 u32 gwip = ntohl(fib_nh->nh_gw);
1344 struct net_device *dev = fib_nh->nh_dev;
1345 struct neighbour *n;
1348 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, &gwip,
1351 __be32 gwipn = htonl(gwip);
1353 n = neigh_create(&arp_tbl, &gwipn, dev);
1356 neigh_event_send(n, NULL);
1357 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, &gwip,
1364 /* Take a reference of neigh here ensuring that neigh would
1365 * not be detructed before the nexthop entry is finished.
1366 * The second branch takes the reference in neith_create()
1372 /* If that is the first nexthop connected to that neigh, add to
1373 * nexthop_neighs_list
1375 if (list_empty(&neigh_entry->nexthop_list))
1376 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1377 &mlxsw_sp->router.nexthop_neighs_list);
1379 nh->nh_grp = nh_grp;
1380 nh->neigh_entry = neigh_entry;
1381 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1382 read_lock_bh(&n->lock);
1383 nud_state = n->nud_state;
1384 read_unlock_bh(&n->lock);
1385 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID));
1390 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1391 struct mlxsw_sp_nexthop *nh)
1393 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1395 list_del(&nh->neigh_list_node);
1397 /* If that is the last nexthop connected to that neigh, remove from
1398 * nexthop_neighs_list
1400 if (list_empty(&nh->neigh_entry->nexthop_list))
1401 list_del(&nh->neigh_entry->nexthop_neighs_list_node);
1403 neigh_release(neigh_entry->n);
1406 static struct mlxsw_sp_nexthop_group *
1407 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1409 struct mlxsw_sp_nexthop_group *nh_grp;
1410 struct mlxsw_sp_nexthop *nh;
1411 struct fib_nh *fib_nh;
1416 alloc_size = sizeof(*nh_grp) +
1417 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1418 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1420 return ERR_PTR(-ENOMEM);
1421 INIT_LIST_HEAD(&nh_grp->fib_list);
1422 nh_grp->count = fi->fib_nhs;
1423 for (i = 0; i < nh_grp->count; i++) {
1424 nh = &nh_grp->nexthops[i];
1425 fib_nh = &fi->fib_nh[i];
1426 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1428 goto err_nexthop_init;
1430 list_add_tail(&nh_grp->list, &mlxsw_sp->router.nexthop_group_list);
1431 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1435 for (i--; i >= 0; i--)
1436 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1438 return ERR_PTR(err);
1442 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1443 struct mlxsw_sp_nexthop_group *nh_grp)
1445 struct mlxsw_sp_nexthop *nh;
1448 list_del(&nh_grp->list);
1449 for (i = 0; i < nh_grp->count; i++) {
1450 nh = &nh_grp->nexthops[i];
1451 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1456 static bool mlxsw_sp_nexthop_match(struct mlxsw_sp_nexthop *nh,
1457 struct fib_info *fi)
1461 for (i = 0; i < fi->fib_nhs; i++) {
1462 struct fib_nh *fib_nh = &fi->fib_nh[i];
1463 u32 gwip = ntohl(fib_nh->nh_gw);
1465 if (memcmp(nh->neigh_entry->key.addr,
1466 &gwip, sizeof(u32)) == 0 &&
1467 nh->neigh_entry->key.dev == fib_nh->nh_dev)
1473 static bool mlxsw_sp_nexthop_group_match(struct mlxsw_sp_nexthop_group *nh_grp,
1474 struct fib_info *fi)
1478 if (nh_grp->count != fi->fib_nhs)
1480 for (i = 0; i < nh_grp->count; i++) {
1481 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
1483 if (!mlxsw_sp_nexthop_match(nh, fi))
1489 static struct mlxsw_sp_nexthop_group *
1490 mlxsw_sp_nexthop_group_find(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1492 struct mlxsw_sp_nexthop_group *nh_grp;
1494 list_for_each_entry(nh_grp, &mlxsw_sp->router.nexthop_group_list,
1496 if (mlxsw_sp_nexthop_group_match(nh_grp, fi))
1502 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1503 struct mlxsw_sp_fib_entry *fib_entry,
1504 struct fib_info *fi)
1506 struct mlxsw_sp_nexthop_group *nh_grp;
1508 nh_grp = mlxsw_sp_nexthop_group_find(mlxsw_sp, fi);
1510 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1512 return PTR_ERR(nh_grp);
1514 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1515 fib_entry->nh_group = nh_grp;
1519 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1520 struct mlxsw_sp_fib_entry *fib_entry)
1522 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1524 list_del(&fib_entry->nexthop_group_node);
1525 if (!list_empty(&nh_grp->fib_list))
1527 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1530 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1531 struct mlxsw_sp_fib_entry *fib_entry,
1532 enum mlxsw_reg_ralue_op op)
1534 char ralue_pl[MLXSW_REG_RALUE_LEN];
1535 u32 *p_dip = (u32 *) fib_entry->key.addr;
1536 struct mlxsw_sp_vr *vr = fib_entry->vr;
1537 enum mlxsw_reg_ralue_trap_action trap_action;
1539 u32 adjacency_index = 0;
1542 /* In case the nexthop group adjacency index is valid, use it
1543 * with provided ECMP size. Otherwise, setup trap and pass
1544 * traffic to kernel.
1546 if (fib_entry->nh_group->adj_index_valid) {
1547 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1548 adjacency_index = fib_entry->nh_group->adj_index;
1549 ecmp_size = fib_entry->nh_group->ecmp_size;
1551 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1552 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1555 mlxsw_reg_ralue_pack4(ralue_pl,
1556 (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1557 vr->id, fib_entry->key.prefix_len, *p_dip);
1558 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1559 adjacency_index, ecmp_size);
1560 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1563 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1564 struct mlxsw_sp_fib_entry *fib_entry,
1565 enum mlxsw_reg_ralue_op op)
1567 char ralue_pl[MLXSW_REG_RALUE_LEN];
1568 u32 *p_dip = (u32 *) fib_entry->key.addr;
1569 struct mlxsw_sp_vr *vr = fib_entry->vr;
1571 mlxsw_reg_ralue_pack4(ralue_pl,
1572 (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1573 vr->id, fib_entry->key.prefix_len, *p_dip);
1574 mlxsw_reg_ralue_act_local_pack(ralue_pl,
1575 MLXSW_REG_RALUE_TRAP_ACTION_NOP, 0,
1577 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1580 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1581 struct mlxsw_sp_fib_entry *fib_entry,
1582 enum mlxsw_reg_ralue_op op)
1584 char ralue_pl[MLXSW_REG_RALUE_LEN];
1585 u32 *p_dip = (u32 *) fib_entry->key.addr;
1586 struct mlxsw_sp_vr *vr = fib_entry->vr;
1588 mlxsw_reg_ralue_pack4(ralue_pl,
1589 (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1590 vr->id, fib_entry->key.prefix_len, *p_dip);
1591 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1592 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1595 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1596 struct mlxsw_sp_fib_entry *fib_entry,
1597 enum mlxsw_reg_ralue_op op)
1599 switch (fib_entry->type) {
1600 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1601 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1602 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1603 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1604 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1605 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1610 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1611 struct mlxsw_sp_fib_entry *fib_entry,
1612 enum mlxsw_reg_ralue_op op)
1614 switch (fib_entry->vr->proto) {
1615 case MLXSW_SP_L3_PROTO_IPV4:
1616 return mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1617 case MLXSW_SP_L3_PROTO_IPV6:
1623 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1624 struct mlxsw_sp_fib_entry *fib_entry)
1626 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1627 MLXSW_REG_RALUE_OP_WRITE_WRITE);
1630 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1631 struct mlxsw_sp_fib_entry *fib_entry)
1633 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1634 MLXSW_REG_RALUE_OP_WRITE_DELETE);
1638 mlxsw_sp_router_fib4_entry_init(struct mlxsw_sp *mlxsw_sp,
1639 const struct fib_entry_notifier_info *fen_info,
1640 struct mlxsw_sp_fib_entry *fib_entry)
1642 struct fib_info *fi = fen_info->fi;
1643 struct mlxsw_sp_rif *r = NULL;
1647 if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
1648 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1651 if (fen_info->type != RTN_UNICAST)
1654 for (nhsel = 0; nhsel < fi->fib_nhs; nhsel++) {
1655 const struct fib_nh *nh = &fi->fib_nh[nhsel];
1659 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, nh->nh_dev);
1661 /* In case router interface is not found for
1662 * at least one of the nexthops, that means
1663 * the nexthop points to some device unrelated
1664 * to us. Set trap and pass the packets for
1665 * this prefix to kernel.
1672 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1676 if (fi->fib_scope != RT_SCOPE_UNIVERSE) {
1677 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1678 fib_entry->rif = r->rif;
1680 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1681 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fi);
1685 fib_info_offload_inc(fen_info->fi);
1690 mlxsw_sp_router_fib4_entry_fini(struct mlxsw_sp *mlxsw_sp,
1691 struct mlxsw_sp_fib_entry *fib_entry)
1693 if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1694 fib_info_offload_dec(fib_entry->fi);
1695 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_REMOTE)
1696 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1699 static struct mlxsw_sp_fib_entry *
1700 mlxsw_sp_fib_entry_get(struct mlxsw_sp *mlxsw_sp,
1701 const struct fib_entry_notifier_info *fen_info)
1703 struct mlxsw_sp_fib_entry *fib_entry;
1704 struct fib_info *fi = fen_info->fi;
1705 struct mlxsw_sp_vr *vr;
1708 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
1709 MLXSW_SP_L3_PROTO_IPV4);
1711 return ERR_CAST(vr);
1713 fib_entry = mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1714 sizeof(fen_info->dst),
1715 fen_info->dst_len, fi->fib_dev);
1717 /* Already exists, just take a reference */
1718 fib_entry->ref_count++;
1721 fib_entry = mlxsw_sp_fib_entry_create(vr->fib, &fen_info->dst,
1722 sizeof(fen_info->dst),
1723 fen_info->dst_len, fi->fib_dev);
1726 goto err_fib_entry_create;
1730 fib_entry->ref_count = 1;
1732 err = mlxsw_sp_router_fib4_entry_init(mlxsw_sp, fen_info, fib_entry);
1734 goto err_fib4_entry_init;
1738 err_fib4_entry_init:
1739 mlxsw_sp_fib_entry_destroy(fib_entry);
1740 err_fib_entry_create:
1741 mlxsw_sp_vr_put(mlxsw_sp, vr);
1743 return ERR_PTR(err);
1746 static struct mlxsw_sp_fib_entry *
1747 mlxsw_sp_fib_entry_find(struct mlxsw_sp *mlxsw_sp,
1748 const struct fib_entry_notifier_info *fen_info)
1750 struct mlxsw_sp_vr *vr;
1752 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id,
1753 MLXSW_SP_L3_PROTO_IPV4);
1757 return mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1758 sizeof(fen_info->dst),
1760 fen_info->fi->fib_dev);
1763 static void mlxsw_sp_fib_entry_put(struct mlxsw_sp *mlxsw_sp,
1764 struct mlxsw_sp_fib_entry *fib_entry)
1766 struct mlxsw_sp_vr *vr = fib_entry->vr;
1768 if (--fib_entry->ref_count == 0) {
1769 mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
1770 mlxsw_sp_fib_entry_destroy(fib_entry);
1772 mlxsw_sp_vr_put(mlxsw_sp, vr);
1775 static void mlxsw_sp_fib_entry_put_all(struct mlxsw_sp *mlxsw_sp,
1776 struct mlxsw_sp_fib_entry *fib_entry)
1778 unsigned int last_ref_count;
1781 last_ref_count = fib_entry->ref_count;
1782 mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1783 } while (last_ref_count != 1);
1786 static int mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
1787 struct fib_entry_notifier_info *fen_info)
1789 struct mlxsw_sp_fib_entry *fib_entry;
1790 struct mlxsw_sp_vr *vr;
1793 if (mlxsw_sp->router.aborted)
1796 fib_entry = mlxsw_sp_fib_entry_get(mlxsw_sp, fen_info);
1797 if (IS_ERR(fib_entry)) {
1798 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB4 entry being added.\n");
1799 return PTR_ERR(fib_entry);
1802 if (fib_entry->ref_count != 1)
1806 err = mlxsw_sp_fib_entry_insert(vr->fib, fib_entry);
1808 dev_warn(mlxsw_sp->bus_info->dev, "Failed to insert FIB4 entry being added.\n");
1809 goto err_fib_entry_insert;
1811 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1813 goto err_fib_entry_add;
1817 mlxsw_sp_fib_entry_remove(vr->fib, fib_entry);
1818 err_fib_entry_insert:
1819 mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1823 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
1824 struct fib_entry_notifier_info *fen_info)
1826 struct mlxsw_sp_fib_entry *fib_entry;
1828 if (mlxsw_sp->router.aborted)
1831 fib_entry = mlxsw_sp_fib_entry_find(mlxsw_sp, fen_info);
1833 dev_warn(mlxsw_sp->bus_info->dev, "Failed to find FIB4 entry being removed.\n");
1837 if (fib_entry->ref_count == 1) {
1838 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1839 mlxsw_sp_fib_entry_remove(fib_entry->vr->fib, fib_entry);
1842 mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1846 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
1848 char ralta_pl[MLXSW_REG_RALTA_LEN];
1849 char ralst_pl[MLXSW_REG_RALST_LEN];
1850 char raltb_pl[MLXSW_REG_RALTB_LEN];
1851 char ralue_pl[MLXSW_REG_RALUE_LEN];
1854 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
1855 MLXSW_SP_LPM_TREE_MIN);
1856 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
1860 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
1861 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
1865 mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4, 0);
1866 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
1870 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
1871 MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
1872 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1873 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1876 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
1878 struct mlxsw_resources *resources;
1879 struct mlxsw_sp_fib_entry *fib_entry;
1880 struct mlxsw_sp_fib_entry *tmp;
1881 struct mlxsw_sp_vr *vr;
1885 resources = mlxsw_core_resources_get(mlxsw_sp->core);
1886 for (i = 0; i < resources->max_virtual_routers; i++) {
1887 vr = &mlxsw_sp->router.vrs[i];
1891 list_for_each_entry_safe(fib_entry, tmp,
1892 &vr->fib->entry_list, list) {
1893 bool do_break = &tmp->list == &vr->fib->entry_list;
1895 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1896 mlxsw_sp_fib_entry_remove(fib_entry->vr->fib,
1898 mlxsw_sp_fib_entry_put_all(mlxsw_sp, fib_entry);
1903 mlxsw_sp->router.aborted = true;
1904 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
1906 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
1909 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
1911 struct mlxsw_resources *resources;
1912 char rgcr_pl[MLXSW_REG_RGCR_LEN];
1915 resources = mlxsw_core_resources_get(mlxsw_sp->core);
1916 if (!resources->max_rif_valid)
1919 mlxsw_sp->rifs = kcalloc(resources->max_rif,
1920 sizeof(struct mlxsw_sp_rif *), GFP_KERNEL);
1921 if (!mlxsw_sp->rifs)
1924 mlxsw_reg_rgcr_pack(rgcr_pl, true);
1925 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, resources->max_rif);
1926 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
1933 kfree(mlxsw_sp->rifs);
1937 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
1939 struct mlxsw_resources *resources;
1940 char rgcr_pl[MLXSW_REG_RGCR_LEN];
1943 mlxsw_reg_rgcr_pack(rgcr_pl, false);
1944 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
1946 resources = mlxsw_core_resources_get(mlxsw_sp->core);
1947 for (i = 0; i < resources->max_rif; i++)
1948 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
1950 kfree(mlxsw_sp->rifs);
1953 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
1954 unsigned long event, void *ptr)
1956 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
1957 struct fib_entry_notifier_info *fen_info = ptr;
1961 case FIB_EVENT_ENTRY_ADD:
1962 err = mlxsw_sp_router_fib4_add(mlxsw_sp, fen_info);
1964 mlxsw_sp_router_fib4_abort(mlxsw_sp);
1966 case FIB_EVENT_ENTRY_DEL:
1967 mlxsw_sp_router_fib4_del(mlxsw_sp, fen_info);
1969 case FIB_EVENT_RULE_ADD: /* fall through */
1970 case FIB_EVENT_RULE_DEL:
1971 mlxsw_sp_router_fib4_abort(mlxsw_sp);
1977 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
1981 INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
1982 INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_group_list);
1983 err = __mlxsw_sp_router_init(mlxsw_sp);
1987 mlxsw_sp_lpm_init(mlxsw_sp);
1988 err = mlxsw_sp_vrs_init(mlxsw_sp);
1992 err = mlxsw_sp_neigh_init(mlxsw_sp);
1994 goto err_neigh_init;
1996 mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
1997 register_fib_notifier(&mlxsw_sp->fib_nb);
2001 mlxsw_sp_vrs_fini(mlxsw_sp);
2003 __mlxsw_sp_router_fini(mlxsw_sp);
2007 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2009 unregister_fib_notifier(&mlxsw_sp->fib_nb);
2010 mlxsw_sp_neigh_fini(mlxsw_sp);
2011 mlxsw_sp_vrs_fini(mlxsw_sp);
2012 __mlxsw_sp_router_fini(mlxsw_sp);