drivers/net/ethernet/ibm/ibmveth.c

   1 /*
   2  * IBM Power Virtual Ethernet Device Driver
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  16  *
  17  * Copyright (C) IBM Corporation, 2003, 2010
  18  *
  19  * Authors: Dave Larson <larson1@us.ibm.com>
  20  *          Santiago Leon <santil@linux.vnet.ibm.com>
  21  *          Brian King <brking@linux.vnet.ibm.com>
  22  *          Robert Jennings <rcj@linux.vnet.ibm.com>
  23  *          Anton Blanchard <anton@au.ibm.com>
  24  */
  25
  26 #include <linux/module.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/types.h>
  29 #include <linux/errno.h>
  30 #include <linux/dma-mapping.h>
  31 #include <linux/kernel.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/etherdevice.h>
  34 #include <linux/skbuff.h>
  35 #include <linux/init.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/mm.h>
  38 #include <linux/pm.h>
  39 #include <linux/ethtool.h>
  40 #include <linux/in.h>
  41 #include <linux/ip.h>
  42 #include <linux/ipv6.h>
  43 #include <linux/slab.h>
  44 #include <asm/hvcall.h>
  45 #include <linux/atomic.h>
  46 #include <asm/vio.h>
  47 #include <asm/iommu.h>
  48 #include <asm/firmware.h>
  49
  50 #include "ibmveth.h"
  51
  52 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance);
  53 static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter);
  54 static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev);
  55
  56 static struct kobj_type ktype_veth_pool;
  57
  58
  59 static const char ibmveth_driver_name[] = "ibmveth";
  60 static const char ibmveth_driver_string[] = "IBM Power Virtual Ethernet Driver";
  61 #define ibmveth_driver_version "1.05"
  62
  63 MODULE_AUTHOR("Santiago Leon <santil@linux.vnet.ibm.com>");
  64 MODULE_DESCRIPTION("IBM Power Virtual Ethernet Driver");
  65 MODULE_LICENSE("GPL");
  66 MODULE_VERSION(ibmveth_driver_version);
  67
  68 static unsigned int tx_copybreak __read_mostly = 128;
  69 module_param(tx_copybreak, uint, 0644);
  70 MODULE_PARM_DESC(tx_copybreak,
  71         "Maximum size of packet that is copied to a new buffer on transmit");
  72
  73 static unsigned int rx_copybreak __read_mostly = 128;
  74 module_param(rx_copybreak, uint, 0644);
  75 MODULE_PARM_DESC(rx_copybreak,
  76         "Maximum size of packet that is copied to a new buffer on receive");
  77
  78 static unsigned int rx_flush __read_mostly = 0;
  79 module_param(rx_flush, uint, 0644);
  80 MODULE_PARM_DESC(rx_flush, "Flush receive buffers before use");
  81
  82 struct ibmveth_stat {
  83         char name[ETH_GSTRING_LEN];
  84         int offset;
  85 };
  86
  87 #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat)
  88 #define IBMVETH_GET_STAT(a, off) *((u64 *)(((unsigned long)(a)) + off))
  89
  90 struct ibmveth_stat ibmveth_stats[] = {
  91         { "replenish_task_cycles", IBMVETH_STAT_OFF(replenish_task_cycles) },
  92         { "replenish_no_mem", IBMVETH_STAT_OFF(replenish_no_mem) },
  93         { "replenish_add_buff_failure",
  94                         IBMVETH_STAT_OFF(replenish_add_buff_failure) },
  95         { "replenish_add_buff_success",
  96                         IBMVETH_STAT_OFF(replenish_add_buff_success) },
  97         { "rx_invalid_buffer", IBMVETH_STAT_OFF(rx_invalid_buffer) },
  98         { "rx_no_buffer", IBMVETH_STAT_OFF(rx_no_buffer) },
  99         { "tx_map_failed", IBMVETH_STAT_OFF(tx_map_failed) },
 100         { "tx_send_failed", IBMVETH_STAT_OFF(tx_send_failed) },
 101         { "fw_enabled_ipv4_csum", IBMVETH_STAT_OFF(fw_ipv4_csum_support) },
 102         { "fw_enabled_ipv6_csum", IBMVETH_STAT_OFF(fw_ipv6_csum_support) },
 103         { "tx_large_packets", IBMVETH_STAT_OFF(tx_large_packets) },
 104 };
 105
 106 /* simple methods of getting data from the current rxq entry */
 107 static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter)
 108 {
 109         return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].flags_off);
 110 }
 111
 112 static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter)
 113 {
 114         return (ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_TOGGLE) >>
 115                         IBMVETH_RXQ_TOGGLE_SHIFT;
 116 }
 117
 118 static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter)
 119 {
 120         return ibmveth_rxq_toggle(adapter) == adapter->rx_queue.toggle;
 121 }
 122
 123 static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter)
 124 {
 125         return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_VALID;
 126 }
 127
 128 static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter)
 129 {
 130         return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_OFF_MASK;
 131 }
 132
 133 static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter)
 134 {
 135         return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].length);
 136 }
 137
 138 static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter)
 139 {
 140         return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD;
 141 }
 142
 143 /* setup the initial settings for a buffer pool */
 144 static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,
 145                                      u32 pool_index, u32 pool_size,
 146                                      u32 buff_size, u32 pool_active)
 147 {
 148         pool->size = pool_size;
 149         pool->index = pool_index;
 150         pool->buff_size = buff_size;
 151         pool->threshold = pool_size * 7 / 8;
 152         pool->active = pool_active;
 153 }
 154
 155 /* allocate and setup an buffer pool - called during open */
 156 static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool)
 157 {
 158         int i;
 159
 160         pool->free_map = kmalloc(sizeof(u16) * pool->size, GFP_KERNEL);
 161
 162         if (!pool->free_map)
 163                 return -1;
 164
 165         pool->dma_addr = kmalloc(sizeof(dma_addr_t) * pool->size, GFP_KERNEL);
 166         if (!pool->dma_addr) {
 167                 kfree(pool->free_map);
 168                 pool->free_map = NULL;
 169                 return -1;
 170         }
 171
 172         pool->skbuff = kcalloc(pool->size, sizeof(void *), GFP_KERNEL);
 173
 174         if (!pool->skbuff) {
 175                 kfree(pool->dma_addr);
 176                 pool->dma_addr = NULL;
 177
 178                 kfree(pool->free_map);
 179                 pool->free_map = NULL;
 180                 return -1;
 181         }
 182
 183         memset(pool->dma_addr, 0, sizeof(dma_addr_t) * pool->size);
 184
 185         for (i = 0; i < pool->size; ++i)
 186                 pool->free_map[i] = i;
 187
 188         atomic_set(&pool->available, 0);
 189         pool->producer_index = 0;
 190         pool->consumer_index = 0;
 191
 192         return 0;
 193 }
 194
 195 static inline void ibmveth_flush_buffer(void *addr, unsigned long length)
 196 {
 197         unsigned long offset;
 198
 199         for (offset = 0; offset < length; offset += SMP_CACHE_BYTES)
 200                 asm("dcbfl %0,%1" :: "b" (addr), "r" (offset));
 201 }
 202
 203 /* replenish the buffers for a pool.  note that we don't need to
 204  * skb_reserve these since they are used for incoming...
 205  */
 206 static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,
 207                                           struct ibmveth_buff_pool *pool)
 208 {
 209         u32 i;
 210         u32 count = pool->size - atomic_read(&pool->available);
 211         u32 buffers_added = 0;
 212         struct sk_buff *skb;
 213         unsigned int free_index, index;
 214         u64 correlator;
 215         unsigned long lpar_rc;
 216         dma_addr_t dma_addr;
 217
 218         mb();
 219
 220         for (i = 0; i < count; ++i) {
 221                 union ibmveth_buf_desc desc;
 222
 223                 skb = netdev_alloc_skb(adapter->netdev, pool->buff_size);
 224
 225                 if (!skb) {
 226                         netdev_dbg(adapter->netdev,
 227                                    "replenish: unable to allocate skb\n");
 228                         adapter->replenish_no_mem++;
 229                         break;
 230                 }
 231
 232                 free_index = pool->consumer_index;
 233                 pool->consumer_index++;
 234                 if (pool->consumer_index >= pool->size)
 235                         pool->consumer_index = 0;
 236                 index = pool->free_map[free_index];
 237
 238                 BUG_ON(index == IBM_VETH_INVALID_MAP);
 239                 BUG_ON(pool->skbuff[index] != NULL);
 240
 241                 dma_addr = dma_map_single(&adapter->vdev->dev, skb->data,
 242                                 pool->buff_size, DMA_FROM_DEVICE);
 243
 244                 if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
 245                         goto failure;
 246
 247                 pool->free_map[free_index] = IBM_VETH_INVALID_MAP;
 248                 pool->dma_addr[index] = dma_addr;
 249                 pool->skbuff[index] = skb;
 250
 251                 correlator = ((u64)pool->index << 32) | index;
 252                 *(u64 *)skb->data = correlator;
 253
 254                 desc.fields.flags_len = IBMVETH_BUF_VALID | pool->buff_size;
 255                 desc.fields.address = dma_addr;
 256
 257                 if (rx_flush) {
 258                         unsigned int len = min(pool->buff_size,
 259                                                 adapter->netdev->mtu +
 260                                                 IBMVETH_BUFF_OH);
 261                         ibmveth_flush_buffer(skb->data, len);
 262                 }
 263                 lpar_rc = h_add_logical_lan_buffer(adapter->vdev->unit_address,
 264                                                    desc.desc);
 265
 266                 if (lpar_rc != H_SUCCESS) {
 267                         goto failure;
 268                 } else {
 269                         buffers_added++;
 270                         adapter->replenish_add_buff_success++;
 271                 }
 272         }
 273
 274         mb();
 275         atomic_add(buffers_added, &(pool->available));
 276         return;
 277
 278 failure:
 279         pool->free_map[free_index] = index;
 280         pool->skbuff[index] = NULL;
 281         if (pool->consumer_index == 0)
 282                 pool->consumer_index = pool->size - 1;
 283         else
 284                 pool->consumer_index--;
 285         if (!dma_mapping_error(&adapter->vdev->dev, dma_addr))
 286                 dma_unmap_single(&adapter->vdev->dev,
 287                                  pool->dma_addr[index], pool->buff_size,
 288                                  DMA_FROM_DEVICE);
 289         dev_kfree_skb_any(skb);
 290         adapter->replenish_add_buff_failure++;
 291
 292         mb();
 293         atomic_add(buffers_added, &(pool->available));
 294 }
 295
 296 /*
 297  * The final 8 bytes of the buffer list is a counter of frames dropped
 298  * because there was not a buffer in the buffer list capable of holding
 299  * the frame.
 300  */
 301 static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter)
 302 {
 303         __be64 *p = adapter->buffer_list_addr + 4096 - 8;
 304
 305         adapter->rx_no_buffer = be64_to_cpup(p);
 306 }
 307
 308 /* replenish routine */
 309 static void ibmveth_replenish_task(struct ibmveth_adapter *adapter)
 310 {
 311         int i;
 312
 313         adapter->replenish_task_cycles++;
 314
 315         for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) {
 316                 struct ibmveth_buff_pool *pool = &adapter->rx_buff_pool[i];
 317
 318                 if (pool->active &&
 319                     (atomic_read(&pool->available) < pool->threshold))
 320                         ibmveth_replenish_buffer_pool(adapter, pool);
 321         }
 322
 323         ibmveth_update_rx_no_buffer(adapter);
 324 }
 325
 326 /* empty and free ana buffer pool - also used to do cleanup in error paths */
 327 static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter,
 328                                      struct ibmveth_buff_pool *pool)
 329 {
 330         int i;
 331
 332         kfree(pool->free_map);
 333         pool->free_map = NULL;
 334
 335         if (pool->skbuff && pool->dma_addr) {
 336                 for (i = 0; i < pool->size; ++i) {
 337                         struct sk_buff *skb = pool->skbuff[i];
 338                         if (skb) {
 339                                 dma_unmap_single(&adapter->vdev->dev,
 340                                                  pool->dma_addr[i],
 341                                                  pool->buff_size,
 342                                                  DMA_FROM_DEVICE);
 343                                 dev_kfree_skb_any(skb);
 344                                 pool->skbuff[i] = NULL;
 345                         }
 346                 }
 347         }
 348
 349         if (pool->dma_addr) {
 350                 kfree(pool->dma_addr);
 351                 pool->dma_addr = NULL;
 352         }
 353
 354         if (pool->skbuff) {
 355                 kfree(pool->skbuff);
 356                 pool->skbuff = NULL;
 357         }
 358 }
 359
 360 /* remove a buffer from a pool */
 361 static void ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,
 362                                             u64 correlator)
 363 {
 364         unsigned int pool  = correlator >> 32;
 365         unsigned int index = correlator & 0xffffffffUL;
 366         unsigned int free_index;
 367         struct sk_buff *skb;
 368
 369         BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
 370         BUG_ON(index >= adapter->rx_buff_pool[pool].size);
 371
 372         skb = adapter->rx_buff_pool[pool].skbuff[index];
 373
 374         BUG_ON(skb == NULL);
 375
 376         adapter->rx_buff_pool[pool].skbuff[index] = NULL;
 377
 378         dma_unmap_single(&adapter->vdev->dev,
 379                          adapter->rx_buff_pool[pool].dma_addr[index],
 380                          adapter->rx_buff_pool[pool].buff_size,
 381                          DMA_FROM_DEVICE);
 382
 383         free_index = adapter->rx_buff_pool[pool].producer_index;
 384         adapter->rx_buff_pool[pool].producer_index++;
 385         if (adapter->rx_buff_pool[pool].producer_index >=
 386             adapter->rx_buff_pool[pool].size)
 387                 adapter->rx_buff_pool[pool].producer_index = 0;
 388         adapter->rx_buff_pool[pool].free_map[free_index] = index;
 389
 390         mb();
 391
 392         atomic_dec(&(adapter->rx_buff_pool[pool].available));
 393 }
 394
 395 /* get the current buffer on the rx queue */
 396 static inline struct sk_buff *ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter)
 397 {
 398         u64 correlator = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator;
 399         unsigned int pool = correlator >> 32;
 400         unsigned int index = correlator & 0xffffffffUL;
 401
 402         BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
 403         BUG_ON(index >= adapter->rx_buff_pool[pool].size);
 404
 405         return adapter->rx_buff_pool[pool].skbuff[index];
 406 }
 407
 408 /* recycle the current buffer on the rx queue */
 409 static int ibmveth_rxq_recycle_buffer(struct ibmveth_adapter *adapter)
 410 {
 411         u32 q_index = adapter->rx_queue.index;
 412         u64 correlator = adapter->rx_queue.queue_addr[q_index].correlator;
 413         unsigned int pool = correlator >> 32;
 414         unsigned int index = correlator & 0xffffffffUL;
 415         union ibmveth_buf_desc desc;
 416         unsigned long lpar_rc;
 417         int ret = 1;
 418
 419         BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
 420         BUG_ON(index >= adapter->rx_buff_pool[pool].size);
 421
 422         if (!adapter->rx_buff_pool[pool].active) {
 423                 ibmveth_rxq_harvest_buffer(adapter);
 424                 ibmveth_free_buffer_pool(adapter, &adapter->rx_buff_pool[pool]);
 425                 goto out;
 426         }
 427
 428         desc.fields.flags_len = IBMVETH_BUF_VALID |
 429                 adapter->rx_buff_pool[pool].buff_size;
 430         desc.fields.address = adapter->rx_buff_pool[pool].dma_addr[index];
 431
 432         lpar_rc = h_add_logical_lan_buffer(adapter->vdev->unit_address, desc.desc);
 433
 434         if (lpar_rc != H_SUCCESS) {
 435                 netdev_dbg(adapter->netdev, "h_add_logical_lan_buffer failed "
 436                            "during recycle rc=%ld", lpar_rc);
 437                 ibmveth_remove_buffer_from_pool(adapter, adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator);
 438                 ret = 0;
 439         }
 440
 441         if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
 442                 adapter->rx_queue.index = 0;
 443                 adapter->rx_queue.toggle = !adapter->rx_queue.toggle;
 444         }
 445
 446 out:
 447         return ret;
 448 }
 449
 450 static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter)
 451 {
 452         ibmveth_remove_buffer_from_pool(adapter, adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator);
 453
 454         if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
 455                 adapter->rx_queue.index = 0;
 456                 adapter->rx_queue.toggle = !adapter->rx_queue.toggle;
 457         }
 458 }
 459
 460 static void ibmveth_cleanup(struct ibmveth_adapter *adapter)
 461 {
 462         int i;
 463         struct device *dev = &adapter->vdev->dev;
 464
 465         if (adapter->buffer_list_addr != NULL) {
 466                 if (!dma_mapping_error(dev, adapter->buffer_list_dma)) {
 467                         dma_unmap_single(dev, adapter->buffer_list_dma, 4096,
 468                                         DMA_BIDIRECTIONAL);
 469                         adapter->buffer_list_dma = DMA_ERROR_CODE;
 470                 }
 471                 free_page((unsigned long)adapter->buffer_list_addr);
 472                 adapter->buffer_list_addr = NULL;
 473         }
 474
 475         if (adapter->filter_list_addr != NULL) {
 476                 if (!dma_mapping_error(dev, adapter->filter_list_dma)) {
 477                         dma_unmap_single(dev, adapter->filter_list_dma, 4096,
 478                                         DMA_BIDIRECTIONAL);
 479                         adapter->filter_list_dma = DMA_ERROR_CODE;
 480                 }
 481                 free_page((unsigned long)adapter->filter_list_addr);
 482                 adapter->filter_list_addr = NULL;
 483         }
 484
 485         if (adapter->rx_queue.queue_addr != NULL) {
 486                 dma_free_coherent(dev, adapter->rx_queue.queue_len,
 487                                   adapter->rx_queue.queue_addr,
 488                                   adapter->rx_queue.queue_dma);
 489                 adapter->rx_queue.queue_addr = NULL;
 490         }
 491
 492         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
 493                 if (adapter->rx_buff_pool[i].active)
 494                         ibmveth_free_buffer_pool(adapter,
 495                                                  &adapter->rx_buff_pool[i]);
 496
 497         if (adapter->bounce_buffer != NULL) {
 498                 if (!dma_mapping_error(dev, adapter->bounce_buffer_dma)) {
 499                         dma_unmap_single(&adapter->vdev->dev,
 500                                         adapter->bounce_buffer_dma,
 501                                         adapter->netdev->mtu + IBMVETH_BUFF_OH,
 502                                         DMA_BIDIRECTIONAL);
 503                         adapter->bounce_buffer_dma = DMA_ERROR_CODE;
 504                 }
 505                 kfree(adapter->bounce_buffer);
 506                 adapter->bounce_buffer = NULL;
 507         }
 508 }
 509
 510 static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,
 511         union ibmveth_buf_desc rxq_desc, u64 mac_address)
 512 {
 513         int rc, try_again = 1;
 514
 515         /*
 516          * After a kexec the adapter will still be open, so our attempt to
 517          * open it will fail. So if we get a failure we free the adapter and
 518          * try again, but only once.
 519          */
 520 retry:
 521         rc = h_register_logical_lan(adapter->vdev->unit_address,
 522                                     adapter->buffer_list_dma, rxq_desc.desc,
 523                                     adapter->filter_list_dma, mac_address);
 524
 525         if (rc != H_SUCCESS && try_again) {
 526                 do {
 527                         rc = h_free_logical_lan(adapter->vdev->unit_address);
 528                 } while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY));
 529
 530                 try_again = 0;
 531                 goto retry;
 532         }
 533
 534         return rc;
 535 }
 536
 537 static u64 ibmveth_encode_mac_addr(u8 *mac)
 538 {
 539         int i;
 540         u64 encoded = 0;
 541
 542         for (i = 0; i < ETH_ALEN; i++)
 543                 encoded = (encoded << 8) | mac[i];
 544
 545         return encoded;
 546 }
 547
 548 static int ibmveth_open(struct net_device *netdev)
 549 {
 550         struct ibmveth_adapter *adapter = netdev_priv(netdev);
 551         u64 mac_address;
 552         int rxq_entries = 1;
 553         unsigned long lpar_rc;
 554         int rc;
 555         union ibmveth_buf_desc rxq_desc;
 556         int i;
 557         struct device *dev;
 558
 559         netdev_dbg(netdev, "open starting\n");
 560
 561         napi_enable(&adapter->napi);
 562
 563         for(i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
 564                 rxq_entries += adapter->rx_buff_pool[i].size;
 565
 566         adapter->buffer_list_addr = (void*) get_zeroed_page(GFP_KERNEL);
 567         adapter->filter_list_addr = (void*) get_zeroed_page(GFP_KERNEL);
 568
 569         if (!adapter->buffer_list_addr || !adapter->filter_list_addr) {
 570                 netdev_err(netdev, "unable to allocate filter or buffer list "
 571                            "pages\n");
 572                 rc = -ENOMEM;
 573                 goto err_out;
 574         }
 575
 576         dev = &adapter->vdev->dev;
 577
 578         adapter->rx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) *
 579                                                 rxq_entries;
 580         adapter->rx_queue.queue_addr =
 581                 dma_alloc_coherent(dev, adapter->rx_queue.queue_len,
 582                                    &adapter->rx_queue.queue_dma, GFP_KERNEL);
 583         if (!adapter->rx_queue.queue_addr) {
 584                 rc = -ENOMEM;
 585                 goto err_out;
 586         }
 587
 588         adapter->buffer_list_dma = dma_map_single(dev,
 589                         adapter->buffer_list_addr, 4096, DMA_BIDIRECTIONAL);
 590         adapter->filter_list_dma = dma_map_single(dev,
 591                         adapter->filter_list_addr, 4096, DMA_BIDIRECTIONAL);
 592
 593         if ((dma_mapping_error(dev, adapter->buffer_list_dma)) ||
 594             (dma_mapping_error(dev, adapter->filter_list_dma))) {
 595                 netdev_err(netdev, "unable to map filter or buffer list "
 596                            "pages\n");
 597                 rc = -ENOMEM;
 598                 goto err_out;
 599         }
 600
 601         adapter->rx_queue.index = 0;
 602         adapter->rx_queue.num_slots = rxq_entries;
 603         adapter->rx_queue.toggle = 1;
 604
 605         mac_address = ibmveth_encode_mac_addr(netdev->dev_addr);
 606
 607         rxq_desc.fields.flags_len = IBMVETH_BUF_VALID |
 608                                         adapter->rx_queue.queue_len;
 609         rxq_desc.fields.address = adapter->rx_queue.queue_dma;
 610
 611         netdev_dbg(netdev, "buffer list @ 0x%p\n", adapter->buffer_list_addr);
 612         netdev_dbg(netdev, "filter list @ 0x%p\n", adapter->filter_list_addr);
 613         netdev_dbg(netdev, "receive q   @ 0x%p\n", adapter->rx_queue.queue_addr);
 614
 615         h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
 616
 617         lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);
 618
 619         if (lpar_rc != H_SUCCESS) {
 620                 netdev_err(netdev, "h_register_logical_lan failed with %ld\n",
 621                            lpar_rc);
 622                 netdev_err(netdev, "buffer TCE:0x%llx filter TCE:0x%llx rxq "
 623                            "desc:0x%llx MAC:0x%llx\n",
 624                                      adapter->buffer_list_dma,
 625                                      adapter->filter_list_dma,
 626                                      rxq_desc.desc,
 627                                      mac_address);
 628                 rc = -ENONET;
 629                 goto err_out;
 630         }
 631
 632         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
 633                 if (!adapter->rx_buff_pool[i].active)
 634                         continue;
 635                 if (ibmveth_alloc_buffer_pool(&adapter->rx_buff_pool[i])) {
 636                         netdev_err(netdev, "unable to alloc pool\n");
 637                         adapter->rx_buff_pool[i].active = 0;
 638                         rc = -ENOMEM;
 639                         goto err_out;
 640                 }
 641         }
 642
 643         netdev_dbg(netdev, "registering irq 0x%x\n", netdev->irq);
 644         rc = request_irq(netdev->irq, ibmveth_interrupt, 0, netdev->name,
 645                          netdev);
 646         if (rc != 0) {
 647                 netdev_err(netdev, "unable to request irq 0x%x, rc %d\n",
 648                            netdev->irq, rc);
 649                 do {
 650                         lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
 651                 } while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
 652
 653                 goto err_out;
 654         }
 655
 656         adapter->bounce_buffer =
 657             kmalloc(netdev->mtu + IBMVETH_BUFF_OH, GFP_KERNEL);
 658         if (!adapter->bounce_buffer) {
 659                 rc = -ENOMEM;
 660                 goto err_out_free_irq;
 661         }
 662         adapter->bounce_buffer_dma =
 663             dma_map_single(&adapter->vdev->dev, adapter->bounce_buffer,
 664                            netdev->mtu + IBMVETH_BUFF_OH, DMA_BIDIRECTIONAL);
 665         if (dma_mapping_error(dev, adapter->bounce_buffer_dma)) {
 666                 netdev_err(netdev, "unable to map bounce buffer\n");
 667                 rc = -ENOMEM;
 668                 goto err_out_free_irq;
 669         }
 670
 671         netdev_dbg(netdev, "initial replenish cycle\n");
 672         ibmveth_interrupt(netdev->irq, netdev);
 673
 674         netif_start_queue(netdev);
 675
 676         netdev_dbg(netdev, "open complete\n");
 677
 678         return 0;
 679
 680 err_out_free_irq:
 681         free_irq(netdev->irq, netdev);
 682 err_out:
 683         ibmveth_cleanup(adapter);
 684         napi_disable(&adapter->napi);
 685         return rc;
 686 }
 687
 688 static int ibmveth_close(struct net_device *netdev)
 689 {
 690         struct ibmveth_adapter *adapter = netdev_priv(netdev);
 691         long lpar_rc;
 692
 693         netdev_dbg(netdev, "close starting\n");
 694
 695         napi_disable(&adapter->napi);
 696
 697         if (!adapter->pool_config)
 698                 netif_stop_queue(netdev);
 699
 700         h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
 701
 702         do {
 703                 lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
 704         } while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
 705
 706         if (lpar_rc != H_SUCCESS) {
 707                 netdev_err(netdev, "h_free_logical_lan failed with %lx, "
 708                            "continuing with close\n", lpar_rc);
 709         }
 710
 711         free_irq(netdev->irq, netdev);
 712
 713         ibmveth_update_rx_no_buffer(adapter);
 714
 715         ibmveth_cleanup(adapter);
 716
 717         netdev_dbg(netdev, "close complete\n");
 718
 719         return 0;
 720 }
 721
 722 static int netdev_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 723 {
 724         cmd->supported = (SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
 725                                 SUPPORTED_FIBRE);
 726         cmd->advertising = (ADVERTISED_1000baseT_Full | ADVERTISED_Autoneg |
 727                                 ADVERTISED_FIBRE);
 728         ethtool_cmd_speed_set(cmd, SPEED_1000);
 729         cmd->duplex = DUPLEX_FULL;
 730         cmd->port = PORT_FIBRE;
 731         cmd->phy_address = 0;
 732         cmd->transceiver = XCVR_INTERNAL;
 733         cmd->autoneg = AUTONEG_ENABLE;
 734         cmd->maxtxpkt = 0;
 735         cmd->maxrxpkt = 1;
 736         return 0;
 737 }
 738
 739 static void netdev_get_drvinfo(struct net_device *dev,
 740                                struct ethtool_drvinfo *info)
 741 {
 742         strlcpy(info->driver, ibmveth_driver_name, sizeof(info->driver));
 743         strlcpy(info->version, ibmveth_driver_version, sizeof(info->version));
 744 }
 745
 746 static netdev_features_t ibmveth_fix_features(struct net_device *dev,
 747         netdev_features_t features)
 748 {
 749         /*
 750          * Since the ibmveth firmware interface does not have the
 751          * concept of separate tx/rx checksum offload enable, if rx
 752          * checksum is disabled we also have to disable tx checksum
 753          * offload. Once we disable rx checksum offload, we are no
 754          * longer allowed to send tx buffers that are not properly
 755          * checksummed.
 756          */
 757
 758         if (!(features & NETIF_F_RXCSUM))
 759                 features &= ~NETIF_F_ALL_CSUM;
 760
 761         return features;
 762 }
 763
 764 static int ibmveth_set_csum_offload(struct net_device *dev, u32 data)
 765 {
 766         struct ibmveth_adapter *adapter = netdev_priv(dev);
 767         unsigned long set_attr, clr_attr, ret_attr;
 768         unsigned long set_attr6, clr_attr6;
 769         long ret, ret4, ret6;
 770         int rc1 = 0, rc2 = 0;
 771         int restart = 0;
 772
 773         if (netif_running(dev)) {
 774                 restart = 1;
 775                 adapter->pool_config = 1;
 776                 ibmveth_close(dev);
 777                 adapter->pool_config = 0;
 778         }
 779
 780         set_attr = 0;
 781         clr_attr = 0;
 782         set_attr6 = 0;
 783         clr_attr6 = 0;
 784
 785         if (data) {
 786                 set_attr = IBMVETH_ILLAN_IPV4_TCP_CSUM;
 787                 set_attr6 = IBMVETH_ILLAN_IPV6_TCP_CSUM;
 788         } else {
 789                 clr_attr = IBMVETH_ILLAN_IPV4_TCP_CSUM;
 790                 clr_attr6 = IBMVETH_ILLAN_IPV6_TCP_CSUM;
 791         }
 792
 793         ret = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr);
 794
 795         if (ret == H_SUCCESS && !(ret_attr & IBMVETH_ILLAN_ACTIVE_TRUNK) &&
 796             !(ret_attr & IBMVETH_ILLAN_TRUNK_PRI_MASK) &&
 797             (ret_attr & IBMVETH_ILLAN_PADDED_PKT_CSUM)) {
 798                 ret4 = h_illan_attributes(adapter->vdev->unit_address, clr_attr,
 799                                          set_attr, &ret_attr);
 800
 801                 if (ret4 != H_SUCCESS) {
 802                         netdev_err(dev, "unable to change IPv4 checksum "
 803                                         "offload settings. %d rc=%ld\n",
 804                                         data, ret4);
 805
 806                         h_illan_attributes(adapter->vdev->unit_address,
 807                                            set_attr, clr_attr, &ret_attr);
 808
 809                         if (data == 1)
 810                                 dev->features &= ~NETIF_F_IP_CSUM;
 811
 812                 } else {
 813                         adapter->fw_ipv4_csum_support = data;
 814                 }
 815
 816                 ret6 = h_illan_attributes(adapter->vdev->unit_address,
 817                                          clr_attr6, set_attr6, &ret_attr);
 818
 819                 if (ret6 != H_SUCCESS) {
 820                         netdev_err(dev, "unable to change IPv6 checksum "
 821                                         "offload settings. %d rc=%ld\n",
 822                                         data, ret6);
 823
 824                         h_illan_attributes(adapter->vdev->unit_address,
 825                                            set_attr6, clr_attr6, &ret_attr);
 826
 827                         if (data == 1)
 828                                 dev->features &= ~NETIF_F_IPV6_CSUM;
 829
 830                 } else
 831                         adapter->fw_ipv6_csum_support = data;
 832
 833                 if (ret4 == H_SUCCESS || ret6 == H_SUCCESS)
 834                         adapter->rx_csum = data;
 835                 else
 836                         rc1 = -EIO;
 837         } else {
 838                 rc1 = -EIO;
 839                 netdev_err(dev, "unable to change checksum offload settings."
 840                                      " %d rc=%ld ret_attr=%lx\n", data, ret,
 841                                      ret_attr);
 842         }
 843
 844         if (restart)
 845                 rc2 = ibmveth_open(dev);
 846
 847         return rc1 ? rc1 : rc2;
 848 }
 849
 850 static int ibmveth_set_features(struct net_device *dev,
 851         netdev_features_t features)
 852 {
 853         struct ibmveth_adapter *adapter = netdev_priv(dev);
 854         int rx_csum = !!(features & NETIF_F_RXCSUM);
 855         int rc;
 856         netdev_features_t changed = features ^ dev->features;
 857
 858         if (features & NETIF_F_TSO & changed)
 859                 netdev_info(dev, "TSO feature requires all partitions to have updated driver");
 860
 861         if (rx_csum == adapter->rx_csum)
 862                 return 0;
 863
 864         rc = ibmveth_set_csum_offload(dev, rx_csum);
 865         if (rc && !adapter->rx_csum)
 866                 dev->features = features & ~(NETIF_F_ALL_CSUM | NETIF_F_RXCSUM);
 867
 868         return rc;
 869 }
 870
 871 static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 872 {
 873         int i;
 874
 875         if (stringset != ETH_SS_STATS)
 876                 return;
 877
 878         for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN)
 879                 memcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN);
 880 }
 881
 882 static int ibmveth_get_sset_count(struct net_device *dev, int sset)
 883 {
 884         switch (sset) {
 885         case ETH_SS_STATS:
 886                 return ARRAY_SIZE(ibmveth_stats);
 887         default:
 888                 return -EOPNOTSUPP;
 889         }
 890 }
 891
 892 static void ibmveth_get_ethtool_stats(struct net_device *dev,
 893                                       struct ethtool_stats *stats, u64 *data)
 894 {
 895         int i;
 896         struct ibmveth_adapter *adapter = netdev_priv(dev);
 897
 898         for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++)
 899                 data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset);
 900 }
 901
 902 static const struct ethtool_ops netdev_ethtool_ops = {
 903         .get_drvinfo            = netdev_get_drvinfo,
 904         .get_settings           = netdev_get_settings,
 905         .get_link               = ethtool_op_get_link,
 906         .get_strings            = ibmveth_get_strings,
 907         .get_sset_count         = ibmveth_get_sset_count,
 908         .get_ethtool_stats      = ibmveth_get_ethtool_stats,
 909 };
 910
 911 static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 912 {
 913         return -EOPNOTSUPP;
 914 }
 915
 916 #define page_offset(v) ((unsigned long)(v) & ((1 << 12) - 1))
 917
 918 static int ibmveth_send(struct ibmveth_adapter *adapter,
 919                         union ibmveth_buf_desc *descs)
 920 {
 921         unsigned long correlator;
 922         unsigned int retry_count;
 923         unsigned long ret;
 924
 925         /*
 926          * The retry count sets a maximum for the number of broadcast and
 927          * multicast destinations within the system.
 928          */
 929         retry_count = 1024;
 930         correlator = 0;
 931         do {
 932                 ret = h_send_logical_lan(adapter->vdev->unit_address,
 933                                              descs[0].desc, descs[1].desc,
 934                                              descs[2].desc, descs[3].desc,
 935                                              descs[4].desc, descs[5].desc,
 936                                              correlator, &correlator);
 937         } while ((ret == H_BUSY) && (retry_count--));
 938
 939         if (ret != H_SUCCESS && ret != H_DROPPED) {
 940                 netdev_err(adapter->netdev, "tx: h_send_logical_lan failed "
 941                            "with rc=%ld\n", ret);
 942                 return 1;
 943         }
 944
 945         return 0;
 946 }
 947
 948 static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 949                                       struct net_device *netdev)
 950 {
 951         struct ibmveth_adapter *adapter = netdev_priv(netdev);
 952         unsigned int desc_flags;
 953         union ibmveth_buf_desc descs[6];
 954         int last, i;
 955         int force_bounce = 0;
 956         dma_addr_t dma_addr;
 957
 958         /*
 959          * veth handles a maximum of 6 segments including the header, so
 960          * we have to linearize the skb if there are more than this.
 961          */
 962         if (skb_shinfo(skb)->nr_frags > 5 && __skb_linearize(skb)) {
 963                 netdev->stats.tx_dropped++;
 964                 goto out;
 965         }
 966
 967         /* veth can't checksum offload UDP */
 968         if (skb->ip_summed == CHECKSUM_PARTIAL &&
 969             ((skb->protocol == htons(ETH_P_IP) &&
 970               ip_hdr(skb)->protocol != IPPROTO_TCP) ||
 971              (skb->protocol == htons(ETH_P_IPV6) &&
 972               ipv6_hdr(skb)->nexthdr != IPPROTO_TCP)) &&
 973             skb_checksum_help(skb)) {
 974
 975                 netdev_err(netdev, "tx: failed to checksum packet\n");
 976                 netdev->stats.tx_dropped++;
 977                 goto out;
 978         }
 979
 980         desc_flags = IBMVETH_BUF_VALID;
 981
 982         if (skb->ip_summed == CHECKSUM_PARTIAL) {
 983                 unsigned char *buf = skb_transport_header(skb) +
 984                                                 skb->csum_offset;
 985
 986                 desc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD);
 987
 988                 /* Need to zero out the checksum */
 989                 buf[0] = 0;
 990                 buf[1] = 0;
 991         }
 992
 993 retry_bounce:
 994         memset(descs, 0, sizeof(descs));
 995
 996         /*
 997          * If a linear packet is below the rx threshold then
 998          * copy it into the static bounce buffer. This avoids the
 999          * cost of a TCE insert and remove.
1000          */
1001         if (force_bounce || (!skb_is_nonlinear(skb) &&
1002                                 (skb->len < tx_copybreak))) {
1003                 skb_copy_from_linear_data(skb, adapter->bounce_buffer,
1004                                           skb->len);
1005
1006                 descs[0].fields.flags_len = desc_flags | skb->len;
1007                 descs[0].fields.address = adapter->bounce_buffer_dma;
1008
1009                 if (ibmveth_send(adapter, descs)) {
1010                         adapter->tx_send_failed++;
1011                         netdev->stats.tx_dropped++;
1012                 } else {
1013                         netdev->stats.tx_packets++;
1014                         netdev->stats.tx_bytes += skb->len;
1015                 }
1016
1017                 goto out;
1018         }
1019
1020         /* Map the header */
1021         dma_addr = dma_map_single(&adapter->vdev->dev, skb->data,
1022                                   skb_headlen(skb), DMA_TO_DEVICE);
1023         if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
1024                 goto map_failed;
1025
1026         descs[0].fields.flags_len = desc_flags | skb_headlen(skb);
1027         descs[0].fields.address = dma_addr;
1028
1029         /* Map the frags */
1030         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1031                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1032
1033                 dma_addr = skb_frag_dma_map(&adapter->vdev->dev, frag, 0,
1034                                             skb_frag_size(frag), DMA_TO_DEVICE);
1035
1036                 if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
1037                         goto map_failed_frags;
1038
1039                 descs[i+1].fields.flags_len = desc_flags | skb_frag_size(frag);
1040                 descs[i+1].fields.address = dma_addr;
1041         }
1042
1043         if (skb_is_gso(skb) && !skb_is_gso_v6(skb)) {
1044                 /* Put -1 in the IP checksum to tell phyp it
1045                  *  is a largesend packet and put the mss in the TCP checksum.
1046                  */
1047                 ip_hdr(skb)->check = 0xffff;
1048                 tcp_hdr(skb)->check = cpu_to_be16(skb_shinfo(skb)->gso_size);
1049                 adapter->tx_large_packets++;
1050         }
1051
1052         if (ibmveth_send(adapter, descs)) {
1053                 adapter->tx_send_failed++;
1054                 netdev->stats.tx_dropped++;
1055         } else {
1056                 netdev->stats.tx_packets++;
1057                 netdev->stats.tx_bytes += skb->len;
1058         }
1059
1060         dma_unmap_single(&adapter->vdev->dev,
1061                          descs[0].fields.address,
1062                          descs[0].fields.flags_len & IBMVETH_BUF_LEN_MASK,
1063                          DMA_TO_DEVICE);
1064
1065         for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++)
1066                 dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address,
1067                                descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK,
1068                                DMA_TO_DEVICE);
1069
1070 out:
1071         dev_consume_skb_any(skb);
1072         return NETDEV_TX_OK;
1073
1074 map_failed_frags:
1075         last = i+1;
1076         for (i = 0; i < last; i++)
1077                 dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address,
1078                                descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK,
1079                                DMA_TO_DEVICE);
1080
1081 map_failed:
1082         if (!firmware_has_feature(FW_FEATURE_CMO))
1083                 netdev_err(netdev, "tx: unable to map xmit buffer\n");
1084         adapter->tx_map_failed++;
1085         skb_linearize(skb);
1086         force_bounce = 1;
1087         goto retry_bounce;
1088 }
1089
1090 static int ibmveth_poll(struct napi_struct *napi, int budget)
1091 {
1092         struct ibmveth_adapter *adapter =
1093                         container_of(napi, struct ibmveth_adapter, napi);
1094         struct net_device *netdev = adapter->netdev;
1095         int frames_processed = 0;
1096         unsigned long lpar_rc;
1097
1098 restart_poll:
1099         while (frames_processed < budget) {
1100                 if (!ibmveth_rxq_pending_buffer(adapter))
1101                         break;
1102
1103                 smp_rmb();
1104                 if (!ibmveth_rxq_buffer_valid(adapter)) {
1105                         wmb(); /* suggested by larson1 */
1106                         adapter->rx_invalid_buffer++;
1107                         netdev_dbg(netdev, "recycling invalid buffer\n");
1108                         ibmveth_rxq_recycle_buffer(adapter);
1109                 } else {
1110                         struct sk_buff *skb, *new_skb;
1111                         int length = ibmveth_rxq_frame_length(adapter);
1112                         int offset = ibmveth_rxq_frame_offset(adapter);
1113                         int csum_good = ibmveth_rxq_csum_good(adapter);
1114
1115                         skb = ibmveth_rxq_get_buffer(adapter);
1116
1117                         new_skb = NULL;
1118                         if (length < rx_copybreak)
1119                                 new_skb = netdev_alloc_skb(netdev, length);
1120
1121                         if (new_skb) {
1122                                 skb_copy_to_linear_data(new_skb,
1123                                                         skb->data + offset,
1124                                                         length);
1125                                 if (rx_flush)
1126                                         ibmveth_flush_buffer(skb->data,
1127                                                 length + offset);
1128                                 if (!ibmveth_rxq_recycle_buffer(adapter))
1129                                         kfree_skb(skb);
1130                                 skb = new_skb;
1131                         } else {
1132                                 ibmveth_rxq_harvest_buffer(adapter);
1133                                 skb_reserve(skb, offset);
1134                         }
1135
1136                         skb_put(skb, length);
1137                         skb->protocol = eth_type_trans(skb, netdev);
1138
1139                         if (csum_good)
1140                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1141
1142                         napi_gro_receive(napi, skb);    /* send it up */
1143
1144                         netdev->stats.rx_packets++;
1145                         netdev->stats.rx_bytes += length;
1146                         frames_processed++;
1147                 }
1148         }
1149
1150         ibmveth_replenish_task(adapter);
1151
1152         if (frames_processed < budget) {
1153                 napi_complete(napi);
1154
1155                 /* We think we are done - reenable interrupts,
1156                  * then check once more to make sure we are done.
1157                  */
1158                 lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1159                                        VIO_IRQ_ENABLE);
1160
1161                 BUG_ON(lpar_rc != H_SUCCESS);
1162
1163                 if (ibmveth_rxq_pending_buffer(adapter) &&
1164                     napi_reschedule(napi)) {
1165                         lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1166                                                VIO_IRQ_DISABLE);
1167                         goto restart_poll;
1168                 }
1169         }
1170
1171         return frames_processed;
1172 }
1173
1174 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance)
1175 {
1176         struct net_device *netdev = dev_instance;
1177         struct ibmveth_adapter *adapter = netdev_priv(netdev);
1178         unsigned long lpar_rc;
1179
1180         if (napi_schedule_prep(&adapter->napi)) {
1181                 lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1182                                        VIO_IRQ_DISABLE);
1183                 BUG_ON(lpar_rc != H_SUCCESS);
1184                 __napi_schedule(&adapter->napi);
1185         }
1186         return IRQ_HANDLED;
1187 }
1188
1189 static void ibmveth_set_multicast_list(struct net_device *netdev)
1190 {
1191         struct ibmveth_adapter *adapter = netdev_priv(netdev);
1192         unsigned long lpar_rc;
1193
1194         if ((netdev->flags & IFF_PROMISC) ||
1195             (netdev_mc_count(netdev) > adapter->mcastFilterSize)) {
1196                 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1197                                            IbmVethMcastEnableRecv |
1198                                            IbmVethMcastDisableFiltering,
1199                                            0);
1200                 if (lpar_rc != H_SUCCESS) {
1201                         netdev_err(netdev, "h_multicast_ctrl rc=%ld when "
1202                                    "entering promisc mode\n", lpar_rc);
1203                 }
1204         } else {
1205                 struct netdev_hw_addr *ha;
1206                 /* clear the filter table & disable filtering */
1207                 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1208                                            IbmVethMcastEnableRecv |
1209                                            IbmVethMcastDisableFiltering |
1210                                            IbmVethMcastClearFilterTable,
1211                                            0);
1212                 if (lpar_rc != H_SUCCESS) {
1213                         netdev_err(netdev, "h_multicast_ctrl rc=%ld when "
1214                                    "attempting to clear filter table\n",
1215                                    lpar_rc);
1216                 }
1217                 /* add the addresses to the filter table */
1218                 netdev_for_each_mc_addr(ha, netdev) {
1219                         /* add the multicast address to the filter table */
1220                         u64 mcast_addr;
1221                         mcast_addr = ibmveth_encode_mac_addr(ha->addr);
1222                         lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1223                                                    IbmVethMcastAddFilter,
1224                                                    mcast_addr);
1225                         if (lpar_rc != H_SUCCESS) {
1226                                 netdev_err(netdev, "h_multicast_ctrl rc=%ld "
1227                                            "when adding an entry to the filter "
1228                                            "table\n", lpar_rc);
1229                         }
1230                 }
1231
1232                 /* re-enable filtering */
1233                 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1234                                            IbmVethMcastEnableFiltering,
1235                                            0);
1236                 if (lpar_rc != H_SUCCESS) {
1237                         netdev_err(netdev, "h_multicast_ctrl rc=%ld when "
1238                                    "enabling filtering\n", lpar_rc);
1239                 }
1240         }
1241 }
1242
1243 static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)
1244 {
1245         struct ibmveth_adapter *adapter = netdev_priv(dev);
1246         struct vio_dev *viodev = adapter->vdev;
1247         int new_mtu_oh = new_mtu + IBMVETH_BUFF_OH;
1248         int i, rc;
1249         int need_restart = 0;
1250
1251         if (new_mtu < IBMVETH_MIN_MTU)
1252                 return -EINVAL;
1253
1254         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
1255                 if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size)
1256                         break;
1257
1258         if (i == IBMVETH_NUM_BUFF_POOLS)
1259                 return -EINVAL;
1260
1261         /* Deactivate all the buffer pools so that the next loop can activate
1262            only the buffer pools necessary to hold the new MTU */
1263         if (netif_running(adapter->netdev)) {
1264                 need_restart = 1;
1265                 adapter->pool_config = 1;
1266                 ibmveth_close(adapter->netdev);
1267                 adapter->pool_config = 0;
1268         }
1269
1270         /* Look for an active buffer pool that can hold the new MTU */
1271         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1272                 adapter->rx_buff_pool[i].active = 1;
1273
1274                 if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) {
1275                         dev->mtu = new_mtu;
1276                         vio_cmo_set_dev_desired(viodev,
1277                                                 ibmveth_get_desired_dma
1278                                                 (viodev));
1279                         if (need_restart) {
1280                                 return ibmveth_open(adapter->netdev);
1281                         }
1282                         return 0;
1283                 }
1284         }
1285
1286         if (need_restart && (rc = ibmveth_open(adapter->netdev)))
1287                 return rc;
1288
1289         return -EINVAL;
1290 }
1291
1292 #ifdef CONFIG_NET_POLL_CONTROLLER
1293 static void ibmveth_poll_controller(struct net_device *dev)
1294 {
1295         ibmveth_replenish_task(netdev_priv(dev));
1296         ibmveth_interrupt(dev->irq, dev);
1297 }
1298 #endif
1299
1300 /**
1301  * ibmveth_get_desired_dma - Calculate IO memory desired by the driver
1302  *
1303  * @vdev: struct vio_dev for the device whose desired IO mem is to be returned
1304  *
1305  * Return value:
1306  *      Number of bytes of IO data the driver will need to perform well.
1307  */
1308 static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)
1309 {
1310         struct net_device *netdev = dev_get_drvdata(&vdev->dev);
1311         struct ibmveth_adapter *adapter;
1312         struct iommu_table *tbl;
1313         unsigned long ret;
1314         int i;
1315         int rxqentries = 1;
1316
1317         tbl = get_iommu_table_base(&vdev->dev);
1318
1319         /* netdev inits at probe time along with the structures we need below*/
1320         if (netdev == NULL)
1321                 return IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl);
1322
1323         adapter = netdev_priv(netdev);
1324
1325         ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE;
1326         ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl);
1327
1328         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1329                 /* add the size of the active receive buffers */
1330                 if (adapter->rx_buff_pool[i].active)
1331                         ret +=
1332                             adapter->rx_buff_pool[i].size *
1333                             IOMMU_PAGE_ALIGN(adapter->rx_buff_pool[i].
1334                                              buff_size, tbl);
1335                 rxqentries += adapter->rx_buff_pool[i].size;
1336         }
1337         /* add the size of the receive queue entries */
1338         ret += IOMMU_PAGE_ALIGN(
1339                 rxqentries * sizeof(struct ibmveth_rx_q_entry), tbl);
1340
1341         return ret;
1342 }
1343
1344 static int ibmveth_set_mac_addr(struct net_device *dev, void *p)
1345 {
1346         struct ibmveth_adapter *adapter = netdev_priv(dev);
1347         struct sockaddr *addr = p;
1348         u64 mac_address;
1349         int rc;
1350
1351         if (!is_valid_ether_addr(addr->sa_data))
1352                 return -EADDRNOTAVAIL;
1353
1354         mac_address = ibmveth_encode_mac_addr(addr->sa_data);
1355         rc = h_change_logical_lan_mac(adapter->vdev->unit_address, mac_address);
1356         if (rc) {
1357                 netdev_err(adapter->netdev, "h_change_logical_lan_mac failed with rc=%d\n", rc);
1358                 return rc;
1359         }
1360
1361         ether_addr_copy(dev->dev_addr, addr->sa_data);
1362
1363         return 0;
1364 }
1365
1366 static const struct net_device_ops ibmveth_netdev_ops = {
1367         .ndo_open               = ibmveth_open,
1368         .ndo_stop               = ibmveth_close,
1369         .ndo_start_xmit         = ibmveth_start_xmit,
1370         .ndo_set_rx_mode        = ibmveth_set_multicast_list,
1371         .ndo_do_ioctl           = ibmveth_ioctl,
1372         .ndo_change_mtu         = ibmveth_change_mtu,
1373         .ndo_fix_features       = ibmveth_fix_features,
1374         .ndo_set_features       = ibmveth_set_features,
1375         .ndo_validate_addr      = eth_validate_addr,
1376         .ndo_set_mac_address    = ibmveth_set_mac_addr,
1377 #ifdef CONFIG_NET_POLL_CONTROLLER
1378         .ndo_poll_controller    = ibmveth_poll_controller,
1379 #endif
1380 };
1381
1382 static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
1383 {
1384         int rc, i, mac_len;
1385         struct net_device *netdev;
1386         struct ibmveth_adapter *adapter;
1387         unsigned char *mac_addr_p;
1388         unsigned int *mcastFilterSize_p;
1389
1390         dev_dbg(&dev->dev, "entering ibmveth_probe for UA 0x%x\n",
1391                 dev->unit_address);
1392
1393         mac_addr_p = (unsigned char *)vio_get_attribute(dev, VETH_MAC_ADDR,
1394                                                         &mac_len);
1395         if (!mac_addr_p) {
1396                 dev_err(&dev->dev, "Can't find VETH_MAC_ADDR attribute\n");
1397                 return -EINVAL;
1398         }
1399         /* Workaround for old/broken pHyp */
1400         if (mac_len == 8)
1401                 mac_addr_p += 2;
1402         else if (mac_len != 6) {
1403                 dev_err(&dev->dev, "VETH_MAC_ADDR attribute wrong len %d\n",
1404                         mac_len);
1405                 return -EINVAL;
1406         }
1407
1408         mcastFilterSize_p = (unsigned int *)vio_get_attribute(dev,
1409                                                 VETH_MCAST_FILTER_SIZE, NULL);
1410         if (!mcastFilterSize_p) {
1411                 dev_err(&dev->dev, "Can't find VETH_MCAST_FILTER_SIZE "
1412                         "attribute\n");
1413                 return -EINVAL;
1414         }
1415
1416         netdev = alloc_etherdev(sizeof(struct ibmveth_adapter));
1417
1418         if (!netdev)
1419                 return -ENOMEM;
1420
1421         adapter = netdev_priv(netdev);
1422         dev_set_drvdata(&dev->dev, netdev);
1423
1424         adapter->vdev = dev;
1425         adapter->netdev = netdev;
1426         adapter->mcastFilterSize = *mcastFilterSize_p;
1427         adapter->pool_config = 0;
1428
1429         netif_napi_add(netdev, &adapter->napi, ibmveth_poll, 16);
1430
1431         netdev->irq = dev->irq;
1432         netdev->netdev_ops = &ibmveth_netdev_ops;
1433         netdev->ethtool_ops = &netdev_ethtool_ops;
1434         SET_NETDEV_DEV(netdev, &dev->dev);
1435         netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
1436                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
1437         netdev->features |= netdev->hw_features;
1438
1439         /* TSO is disabled by default */
1440         netdev->hw_features |= NETIF_F_TSO;
1441
1442         memcpy(netdev->dev_addr, mac_addr_p, ETH_ALEN);
1443
1444         if (firmware_has_feature(FW_FEATURE_CMO))
1445                 memcpy(pool_count, pool_count_cmo, sizeof(pool_count));
1446
1447         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1448                 struct kobject *kobj = &adapter->rx_buff_pool[i].kobj;
1449                 int error;
1450
1451                 ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i,
1452                                          pool_count[i], pool_size[i],
1453                                          pool_active[i]);
1454                 error = kobject_init_and_add(kobj, &ktype_veth_pool,
1455                                              &dev->dev.kobj, "pool%d", i);
1456                 if (!error)
1457                         kobject_uevent(kobj, KOBJ_ADD);
1458         }
1459
1460         netdev_dbg(netdev, "adapter @ 0x%p\n", adapter);
1461
1462         adapter->buffer_list_dma = DMA_ERROR_CODE;
1463         adapter->filter_list_dma = DMA_ERROR_CODE;
1464         adapter->rx_queue.queue_dma = DMA_ERROR_CODE;
1465
1466         netdev_dbg(netdev, "registering netdev...\n");
1467
1468         ibmveth_set_features(netdev, netdev->features);
1469
1470         rc = register_netdev(netdev);
1471
1472         if (rc) {
1473                 netdev_dbg(netdev, "failed to register netdev rc=%d\n", rc);
1474                 free_netdev(netdev);
1475                 return rc;
1476         }
1477
1478         netdev_dbg(netdev, "registered\n");
1479
1480         return 0;
1481 }
1482
1483 static int ibmveth_remove(struct vio_dev *dev)
1484 {
1485         struct net_device *netdev = dev_get_drvdata(&dev->dev);
1486         struct ibmveth_adapter *adapter = netdev_priv(netdev);
1487         int i;
1488
1489         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
1490                 kobject_put(&adapter->rx_buff_pool[i].kobj);
1491
1492         unregister_netdev(netdev);
1493
1494         free_netdev(netdev);
1495         dev_set_drvdata(&dev->dev, NULL);
1496
1497         return 0;
1498 }
1499
1500 static struct attribute veth_active_attr;
1501 static struct attribute veth_num_attr;
1502 static struct attribute veth_size_attr;
1503
1504 static ssize_t veth_pool_show(struct kobject *kobj,
1505                               struct attribute *attr, char *buf)
1506 {
1507         struct ibmveth_buff_pool *pool = container_of(kobj,
1508                                                       struct ibmveth_buff_pool,
1509                                                       kobj);
1510
1511         if (attr == &veth_active_attr)
1512                 return sprintf(buf, "%d\n", pool->active);
1513         else if (attr == &veth_num_attr)
1514                 return sprintf(buf, "%d\n", pool->size);
1515         else if (attr == &veth_size_attr)
1516                 return sprintf(buf, "%d\n", pool->buff_size);
1517         return 0;
1518 }
1519
1520 static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,
1521                                const char *buf, size_t count)
1522 {
1523         struct ibmveth_buff_pool *pool = container_of(kobj,
1524                                                       struct ibmveth_buff_pool,
1525                                                       kobj);
1526         struct net_device *netdev = dev_get_drvdata(
1527             container_of(kobj->parent, struct device, kobj));
1528         struct ibmveth_adapter *adapter = netdev_priv(netdev);
1529         long value = simple_strtol(buf, NULL, 10);
1530         long rc;
1531
1532         if (attr == &veth_active_attr) {
1533                 if (value && !pool->active) {
1534                         if (netif_running(netdev)) {
1535                                 if (ibmveth_alloc_buffer_pool(pool)) {
1536                                         netdev_err(netdev,
1537                                                    "unable to alloc pool\n");
1538                                         return -ENOMEM;
1539                                 }
1540                                 pool->active = 1;
1541                                 adapter->pool_config = 1;
1542                                 ibmveth_close(netdev);
1543                                 adapter->pool_config = 0;
1544                                 if ((rc = ibmveth_open(netdev)))
1545                                         return rc;
1546                         } else {
1547                                 pool->active = 1;
1548                         }
1549                 } else if (!value && pool->active) {
1550                         int mtu = netdev->mtu + IBMVETH_BUFF_OH;
1551                         int i;
1552                         /* Make sure there is a buffer pool with buffers that
1553                            can hold a packet of the size of the MTU */
1554                         for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1555                                 if (pool == &adapter->rx_buff_pool[i])
1556                                         continue;
1557                                 if (!adapter->rx_buff_pool[i].active)
1558                                         continue;
1559                                 if (mtu <= adapter->rx_buff_pool[i].buff_size)
1560                                         break;
1561                         }
1562
1563                         if (i == IBMVETH_NUM_BUFF_POOLS) {
1564                                 netdev_err(netdev, "no active pool >= MTU\n");
1565                                 return -EPERM;
1566                         }
1567
1568                         if (netif_running(netdev)) {
1569                                 adapter->pool_config = 1;
1570                                 ibmveth_close(netdev);
1571                                 pool->active = 0;
1572                                 adapter->pool_config = 0;
1573                                 if ((rc = ibmveth_open(netdev)))
1574                                         return rc;
1575                         }
1576                         pool->active = 0;
1577                 }
1578         } else if (attr == &veth_num_attr) {
1579                 if (value <= 0 || value > IBMVETH_MAX_POOL_COUNT) {
1580                         return -EINVAL;
1581                 } else {
1582                         if (netif_running(netdev)) {
1583                                 adapter->pool_config = 1;
1584                                 ibmveth_close(netdev);
1585                                 adapter->pool_config = 0;
1586                                 pool->size = value;
1587                                 if ((rc = ibmveth_open(netdev)))
1588                                         return rc;
1589                         } else {
1590                                 pool->size = value;
1591                         }
1592                 }
1593         } else if (attr == &veth_size_attr) {
1594                 if (value <= IBMVETH_BUFF_OH || value > IBMVETH_MAX_BUF_SIZE) {
1595                         return -EINVAL;
1596                 } else {
1597                         if (netif_running(netdev)) {
1598                                 adapter->pool_config = 1;
1599                                 ibmveth_close(netdev);
1600                                 adapter->pool_config = 0;
1601                                 pool->buff_size = value;
1602                                 if ((rc = ibmveth_open(netdev)))
1603                                         return rc;
1604                         } else {
1605                                 pool->buff_size = value;
1606                         }
1607                 }
1608         }
1609
1610         /* kick the interrupt handler to allocate/deallocate pools */
1611         ibmveth_interrupt(netdev->irq, netdev);
1612         return count;
1613 }
1614
1615
1616 #define ATTR(_name, _mode)                              \
1617         struct attribute veth_##_name##_attr = {        \
1618         .name = __stringify(_name), .mode = _mode,      \
1619         };
1620
1621 static ATTR(active, 0644);
1622 static ATTR(num, 0644);
1623 static ATTR(size, 0644);
1624
1625 static struct attribute *veth_pool_attrs[] = {
1626         &veth_active_attr,
1627         &veth_num_attr,
1628         &veth_size_attr,
1629         NULL,
1630 };
1631
1632 static const struct sysfs_ops veth_pool_ops = {
1633         .show   = veth_pool_show,
1634         .store  = veth_pool_store,
1635 };
1636
1637 static struct kobj_type ktype_veth_pool = {
1638         .release        = NULL,
1639         .sysfs_ops      = &veth_pool_ops,
1640         .default_attrs  = veth_pool_attrs,
1641 };
1642
1643 static int ibmveth_resume(struct device *dev)
1644 {
1645         struct net_device *netdev = dev_get_drvdata(dev);
1646         ibmveth_interrupt(netdev->irq, netdev);
1647         return 0;
1648 }
1649
1650 static struct vio_device_id ibmveth_device_table[] = {
1651         { "network", "IBM,l-lan"},
1652         { "", "" }
1653 };
1654 MODULE_DEVICE_TABLE(vio, ibmveth_device_table);
1655
1656 static struct dev_pm_ops ibmveth_pm_ops = {
1657         .resume = ibmveth_resume
1658 };
1659
1660 static struct vio_driver ibmveth_driver = {
1661         .id_table       = ibmveth_device_table,
1662         .probe          = ibmveth_probe,
1663         .remove         = ibmveth_remove,
1664         .get_desired_dma = ibmveth_get_desired_dma,
1665         .name           = ibmveth_driver_name,
1666         .pm             = &ibmveth_pm_ops,
1667 };
1668
1669 static int __init ibmveth_module_init(void)
1670 {
1671         printk(KERN_DEBUG "%s: %s %s\n", ibmveth_driver_name,
1672                ibmveth_driver_string, ibmveth_driver_version);
1673
1674         return vio_register_driver(&ibmveth_driver);
1675 }
1676
1677 static void __exit ibmveth_module_exit(void)
1678 {
1679         vio_unregister_driver(&ibmveth_driver);
1680 }
1681
1682 module_init(ibmveth_module_init);
1683 module_exit(ibmveth_module_exit);