drivers/rapidio/devices/tsi721.c

   1 /*
   2  * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
   3  *
   4  * Copyright 2011 Integrated Device Technology, Inc.
   5  * Alexandre Bounine <alexandre.bounine@idt.com>
   6  * Chul Kim <chul.kim@idt.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along with
  19  * this program; if not, write to the Free Software Foundation, Inc., 59
  20  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  21  */
  22
  23 #include <linux/io.h>
  24 #include <linux/errno.h>
  25 #include <linux/init.h>
  26 #include <linux/ioport.h>
  27 #include <linux/kernel.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/rio.h>
  31 #include <linux/rio_drv.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/kfifo.h>
  35 #include <linux/delay.h>
  36
  37 #include "tsi721.h"
  38
  39 #ifdef DEBUG
  40 u32 dbg_level = DBG_INIT | DBG_EXIT;
  41 module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
  42 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
  43 #endif
  44
  45 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
  46 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
  47
  48 /**
  49  * tsi721_lcread - read from local SREP config space
  50  * @mport: RapidIO master port info
  51  * @index: ID of RapdiIO interface
  52  * @offset: Offset into configuration space
  53  * @len: Length (in bytes) of the maintenance transaction
  54  * @data: Value to be read into
  55  *
  56  * Generates a local SREP space read. Returns %0 on
  57  * success or %-EINVAL on failure.
  58  */
  59 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
  60                          int len, u32 *data)
  61 {
  62         struct tsi721_device *priv = mport->priv;
  63
  64         if (len != sizeof(u32))
  65                 return -EINVAL; /* only 32-bit access is supported */
  66
  67         *data = ioread32(priv->regs + offset);
  68
  69         return 0;
  70 }
  71
  72 /**
  73  * tsi721_lcwrite - write into local SREP config space
  74  * @mport: RapidIO master port info
  75  * @index: ID of RapdiIO interface
  76  * @offset: Offset into configuration space
  77  * @len: Length (in bytes) of the maintenance transaction
  78  * @data: Value to be written
  79  *
  80  * Generates a local write into SREP configuration space. Returns %0 on
  81  * success or %-EINVAL on failure.
  82  */
  83 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
  84                           int len, u32 data)
  85 {
  86         struct tsi721_device *priv = mport->priv;
  87
  88         if (len != sizeof(u32))
  89                 return -EINVAL; /* only 32-bit access is supported */
  90
  91         iowrite32(data, priv->regs + offset);
  92
  93         return 0;
  94 }
  95
  96 /**
  97  * tsi721_maint_dma - Helper function to generate RapidIO maintenance
  98  *                    transactions using designated Tsi721 DMA channel.
  99  * @priv: pointer to tsi721 private data
 100  * @sys_size: RapdiIO transport system size
 101  * @destid: Destination ID of transaction
 102  * @hopcount: Number of hops to target device
 103  * @offset: Offset into configuration space
 104  * @len: Length (in bytes) of the maintenance transaction
 105  * @data: Location to be read from or write into
 106  * @do_wr: Operation flag (1 == MAINT_WR)
 107  *
 108  * Generates a RapidIO maintenance transaction (Read or Write).
 109  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 110  */
 111 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
 112                         u16 destid, u8 hopcount, u32 offset, int len,
 113                         u32 *data, int do_wr)
 114 {
 115         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
 116         struct tsi721_dma_desc *bd_ptr;
 117         u32 rd_count, swr_ptr, ch_stat;
 118         int i, err = 0;
 119         u32 op = do_wr ? MAINT_WR : MAINT_RD;
 120
 121         if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
 122                 return -EINVAL;
 123
 124         bd_ptr = priv->mdma.bd_base;
 125
 126         rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
 127
 128         /* Initialize DMA descriptor */
 129         bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
 130         bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
 131         bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
 132         bd_ptr[0].raddr_hi = 0;
 133         if (do_wr)
 134                 bd_ptr[0].data[0] = cpu_to_be32p(data);
 135         else
 136                 bd_ptr[0].data[0] = 0xffffffff;
 137
 138         mb();
 139
 140         /* Start DMA operation */
 141         iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
 142         ioread32(regs + TSI721_DMAC_DWRCNT);
 143         i = 0;
 144
 145         /* Wait until DMA transfer is finished */
 146         while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
 147                                                         & TSI721_DMAC_STS_RUN) {
 148                 udelay(1);
 149                 if (++i >= 5000000) {
 150                         tsi_debug(MAINT, &priv->pdev->dev,
 151                                 "DMA[%d] read timeout ch_status=%x",
 152                                 priv->mdma.ch_id, ch_stat);
 153                         if (!do_wr)
 154                                 *data = 0xffffffff;
 155                         err = -EIO;
 156                         goto err_out;
 157                 }
 158         }
 159
 160         if (ch_stat & TSI721_DMAC_STS_ABORT) {
 161                 /* If DMA operation aborted due to error,
 162                  * reinitialize DMA channel
 163                  */
 164                 tsi_debug(MAINT, &priv->pdev->dev, "DMA ABORT ch_stat=%x",
 165                           ch_stat);
 166                 tsi_debug(MAINT, &priv->pdev->dev,
 167                           "OP=%d : destid=%x hc=%x off=%x",
 168                           do_wr ? MAINT_WR : MAINT_RD,
 169                           destid, hopcount, offset);
 170                 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
 171                 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
 172                 udelay(10);
 173                 iowrite32(0, regs + TSI721_DMAC_DWRCNT);
 174                 udelay(1);
 175                 if (!do_wr)
 176                         *data = 0xffffffff;
 177                 err = -EIO;
 178                 goto err_out;
 179         }
 180
 181         if (!do_wr)
 182                 *data = be32_to_cpu(bd_ptr[0].data[0]);
 183
 184         /*
 185          * Update descriptor status FIFO RD pointer.
 186          * NOTE: Skipping check and clear FIFO entries because we are waiting
 187          * for transfer to be completed.
 188          */
 189         swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
 190         iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
 191 err_out:
 192
 193         return err;
 194 }
 195
 196 /**
 197  * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
 198  *                    using Tsi721 BDMA engine.
 199  * @mport: RapidIO master port control structure
 200  * @index: ID of RapdiIO interface
 201  * @destid: Destination ID of transaction
 202  * @hopcount: Number of hops to target device
 203  * @offset: Offset into configuration space
 204  * @len: Length (in bytes) of the maintenance transaction
 205  * @val: Location to be read into
 206  *
 207  * Generates a RapidIO maintenance read transaction.
 208  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 209  */
 210 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
 211                         u8 hopcount, u32 offset, int len, u32 *data)
 212 {
 213         struct tsi721_device *priv = mport->priv;
 214
 215         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
 216                                 offset, len, data, 0);
 217 }
 218
 219 /**
 220  * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
 221  *                     using Tsi721 BDMA engine
 222  * @mport: RapidIO master port control structure
 223  * @index: ID of RapdiIO interface
 224  * @destid: Destination ID of transaction
 225  * @hopcount: Number of hops to target device
 226  * @offset: Offset into configuration space
 227  * @len: Length (in bytes) of the maintenance transaction
 228  * @val: Value to be written
 229  *
 230  * Generates a RapidIO maintenance write transaction.
 231  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 232  */
 233 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
 234                          u8 hopcount, u32 offset, int len, u32 data)
 235 {
 236         struct tsi721_device *priv = mport->priv;
 237         u32 temp = data;
 238
 239         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
 240                                 offset, len, &temp, 1);
 241 }
 242
 243 /**
 244  * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
 245  * @priv:  tsi721 device private structure
 246  *
 247  * Handles inbound port-write interrupts. Copies PW message from an internal
 248  * buffer into PW message FIFO and schedules deferred routine to process
 249  * queued messages.
 250  */
 251 static int
 252 tsi721_pw_handler(struct tsi721_device *priv)
 253 {
 254         u32 pw_stat;
 255         u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
 256
 257
 258         pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
 259
 260         if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
 261                 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
 262                 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
 263                 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
 264                 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
 265
 266                 /* Queue PW message (if there is room in FIFO),
 267                  * otherwise discard it.
 268                  */
 269                 spin_lock(&priv->pw_fifo_lock);
 270                 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
 271                         kfifo_in(&priv->pw_fifo, pw_buf,
 272                                                 TSI721_RIO_PW_MSG_SIZE);
 273                 else
 274                         priv->pw_discard_count++;
 275                 spin_unlock(&priv->pw_fifo_lock);
 276         }
 277
 278         /* Clear pending PW interrupts */
 279         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
 280                   priv->regs + TSI721_RIO_PW_RX_STAT);
 281
 282         schedule_work(&priv->pw_work);
 283
 284         return 0;
 285 }
 286
 287 static void tsi721_pw_dpc(struct work_struct *work)
 288 {
 289         struct tsi721_device *priv = container_of(work, struct tsi721_device,
 290                                                     pw_work);
 291         union rio_pw_msg pwmsg;
 292
 293         /*
 294          * Process port-write messages
 295          */
 296         while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)&pwmsg,
 297                          TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
 298                 /* Pass the port-write message to RIO core for processing */
 299                 rio_inb_pwrite_handler(&priv->mport, &pwmsg);
 300         }
 301 }
 302
 303 /**
 304  * tsi721_pw_enable - enable/disable port-write interface init
 305  * @mport: Master port implementing the port write unit
 306  * @enable:    1=enable; 0=disable port-write message handling
 307  */
 308 static int tsi721_pw_enable(struct rio_mport *mport, int enable)
 309 {
 310         struct tsi721_device *priv = mport->priv;
 311         u32 rval;
 312
 313         rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
 314
 315         if (enable)
 316                 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
 317         else
 318                 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
 319
 320         /* Clear pending PW interrupts */
 321         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
 322                   priv->regs + TSI721_RIO_PW_RX_STAT);
 323         /* Update enable bits */
 324         iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
 325
 326         return 0;
 327 }
 328
 329 /**
 330  * tsi721_dsend - Send a RapidIO doorbell
 331  * @mport: RapidIO master port info
 332  * @index: ID of RapidIO interface
 333  * @destid: Destination ID of target device
 334  * @data: 16-bit info field of RapidIO doorbell
 335  *
 336  * Sends a RapidIO doorbell message. Always returns %0.
 337  */
 338 static int tsi721_dsend(struct rio_mport *mport, int index,
 339                         u16 destid, u16 data)
 340 {
 341         struct tsi721_device *priv = mport->priv;
 342         u32 offset;
 343
 344         offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
 345                  (destid << 2);
 346
 347         tsi_debug(DBELL, &priv->pdev->dev,
 348                   "Send Doorbell 0x%04x to destID 0x%x", data, destid);
 349         iowrite16be(data, priv->odb_base + offset);
 350
 351         return 0;
 352 }
 353
 354 /**
 355  * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
 356  * @priv: tsi721 device-specific data structure
 357  *
 358  * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
 359  * buffer into DB message FIFO and schedules deferred  routine to process
 360  * queued DBs.
 361  */
 362 static int
 363 tsi721_dbell_handler(struct tsi721_device *priv)
 364 {
 365         u32 regval;
 366
 367         /* Disable IDB interrupts */
 368         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 369         regval &= ~TSI721_SR_CHINT_IDBQRCV;
 370         iowrite32(regval,
 371                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 372
 373         schedule_work(&priv->idb_work);
 374
 375         return 0;
 376 }
 377
 378 static void tsi721_db_dpc(struct work_struct *work)
 379 {
 380         struct tsi721_device *priv = container_of(work, struct tsi721_device,
 381                                                     idb_work);
 382         struct rio_mport *mport;
 383         struct rio_dbell *dbell;
 384         int found = 0;
 385         u32 wr_ptr, rd_ptr;
 386         u64 *idb_entry;
 387         u32 regval;
 388         union {
 389                 u64 msg;
 390                 u8  bytes[8];
 391         } idb;
 392
 393         /*
 394          * Process queued inbound doorbells
 395          */
 396         mport = &priv->mport;
 397
 398         wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 399         rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
 400
 401         while (wr_ptr != rd_ptr) {
 402                 idb_entry = (u64 *)(priv->idb_base +
 403                                         (TSI721_IDB_ENTRY_SIZE * rd_ptr));
 404                 rd_ptr++;
 405                 rd_ptr %= IDB_QSIZE;
 406                 idb.msg = *idb_entry;
 407                 *idb_entry = 0;
 408
 409                 /* Process one doorbell */
 410                 list_for_each_entry(dbell, &mport->dbells, node) {
 411                         if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
 412                             (dbell->res->end >= DBELL_INF(idb.bytes))) {
 413                                 found = 1;
 414                                 break;
 415                         }
 416                 }
 417
 418                 if (found) {
 419                         dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
 420                                     DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
 421                 } else {
 422                         tsi_debug(DBELL, &priv->pdev->dev,
 423                                   "spurious IDB sid %2.2x tid %2.2x info %4.4x",
 424                                   DBELL_SID(idb.bytes), DBELL_TID(idb.bytes),
 425                                   DBELL_INF(idb.bytes));
 426                 }
 427
 428                 wr_ptr = ioread32(priv->regs +
 429                                   TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 430         }
 431
 432         iowrite32(rd_ptr & (IDB_QSIZE - 1),
 433                 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
 434
 435         /* Re-enable IDB interrupts */
 436         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 437         regval |= TSI721_SR_CHINT_IDBQRCV;
 438         iowrite32(regval,
 439                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 440
 441         wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 442         if (wr_ptr != rd_ptr)
 443                 schedule_work(&priv->idb_work);
 444 }
 445
 446 /**
 447  * tsi721_irqhandler - Tsi721 interrupt handler
 448  * @irq: Linux interrupt number
 449  * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
 450  *
 451  * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
 452  * interrupt events and calls an event-specific handler(s).
 453  */
 454 static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
 455 {
 456         struct tsi721_device *priv = (struct tsi721_device *)ptr;
 457         u32 dev_int;
 458         u32 dev_ch_int;
 459         u32 intval;
 460         u32 ch_inte;
 461
 462         /* For MSI mode disable all device-level interrupts */
 463         if (priv->flags & TSI721_USING_MSI)
 464                 iowrite32(0, priv->regs + TSI721_DEV_INTE);
 465
 466         dev_int = ioread32(priv->regs + TSI721_DEV_INT);
 467         if (!dev_int)
 468                 return IRQ_NONE;
 469
 470         dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
 471
 472         if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
 473                 /* Service SR2PC Channel interrupts */
 474                 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
 475                         /* Service Inbound Doorbell interrupt */
 476                         intval = ioread32(priv->regs +
 477                                                 TSI721_SR_CHINT(IDB_QUEUE));
 478                         if (intval & TSI721_SR_CHINT_IDBQRCV)
 479                                 tsi721_dbell_handler(priv);
 480                         else
 481                                 tsi_info(&priv->pdev->dev,
 482                                         "Unsupported SR_CH_INT %x", intval);
 483
 484                         /* Clear interrupts */
 485                         iowrite32(intval,
 486                                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 487                         ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 488                 }
 489         }
 490
 491         if (dev_int & TSI721_DEV_INT_SMSG_CH) {
 492                 int ch;
 493
 494                 /*
 495                  * Service channel interrupts from Messaging Engine
 496                  */
 497
 498                 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
 499                         /* Disable signaled OB MSG Channel interrupts */
 500                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
 501                         ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
 502                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
 503
 504                         /*
 505                          * Process Inbound Message interrupt for each MBOX
 506                          */
 507                         for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
 508                                 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
 509                                         continue;
 510                                 tsi721_imsg_handler(priv, ch);
 511                         }
 512                 }
 513
 514                 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
 515                         /* Disable signaled OB MSG Channel interrupts */
 516                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
 517                         ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
 518                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
 519
 520                         /*
 521                          * Process Outbound Message interrupts for each MBOX
 522                          */
 523
 524                         for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
 525                                 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
 526                                         continue;
 527                                 tsi721_omsg_handler(priv, ch);
 528                         }
 529                 }
 530         }
 531
 532         if (dev_int & TSI721_DEV_INT_SRIO) {
 533                 /* Service SRIO MAC interrupts */
 534                 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
 535                 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
 536                         tsi721_pw_handler(priv);
 537         }
 538
 539 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 540         if (dev_int & TSI721_DEV_INT_BDMA_CH) {
 541                 int ch;
 542
 543                 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
 544                         tsi_debug(DMA, &priv->pdev->dev,
 545                                   "IRQ from DMA channel 0x%08x", dev_ch_int);
 546
 547                         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
 548                                 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
 549                                         continue;
 550                                 tsi721_bdma_handler(&priv->bdma[ch]);
 551                         }
 552                 }
 553         }
 554 #endif
 555
 556         /* For MSI mode re-enable device-level interrupts */
 557         if (priv->flags & TSI721_USING_MSI) {
 558                 dev_int = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
 559                         TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
 560                 iowrite32(dev_int, priv->regs + TSI721_DEV_INTE);
 561         }
 562
 563         return IRQ_HANDLED;
 564 }
 565
 566 static void tsi721_interrupts_init(struct tsi721_device *priv)
 567 {
 568         u32 intr;
 569
 570         /* Enable IDB interrupts */
 571         iowrite32(TSI721_SR_CHINT_ALL,
 572                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 573         iowrite32(TSI721_SR_CHINT_IDBQRCV,
 574                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 575
 576         /* Enable SRIO MAC interrupts */
 577         iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
 578                 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
 579
 580         /* Enable interrupts from channels in use */
 581 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 582         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
 583                 (TSI721_INT_BDMA_CHAN_M &
 584                  ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
 585 #else
 586         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
 587 #endif
 588         iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
 589
 590         if (priv->flags & TSI721_USING_MSIX)
 591                 intr = TSI721_DEV_INT_SRIO;
 592         else
 593                 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
 594                         TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
 595
 596         iowrite32(intr, priv->regs + TSI721_DEV_INTE);
 597         ioread32(priv->regs + TSI721_DEV_INTE);
 598 }
 599
 600 #ifdef CONFIG_PCI_MSI
 601 /**
 602  * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
 603  * @irq: Linux interrupt number
 604  * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
 605  *
 606  * Handles outbound messaging interrupts signaled using MSI-X.
 607  */
 608 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
 609 {
 610         struct tsi721_device *priv = (struct tsi721_device *)ptr;
 611         int mbox;
 612
 613         mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
 614         tsi721_omsg_handler(priv, mbox);
 615         return IRQ_HANDLED;
 616 }
 617
 618 /**
 619  * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
 620  * @irq: Linux interrupt number
 621  * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
 622  *
 623  * Handles inbound messaging interrupts signaled using MSI-X.
 624  */
 625 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
 626 {
 627         struct tsi721_device *priv = (struct tsi721_device *)ptr;
 628         int mbox;
 629
 630         mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
 631         tsi721_imsg_handler(priv, mbox + 4);
 632         return IRQ_HANDLED;
 633 }
 634
 635 /**
 636  * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
 637  * @irq: Linux interrupt number
 638  * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
 639  *
 640  * Handles Tsi721 interrupts from SRIO MAC.
 641  */
 642 static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
 643 {
 644         struct tsi721_device *priv = (struct tsi721_device *)ptr;
 645         u32 srio_int;
 646
 647         /* Service SRIO MAC interrupts */
 648         srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
 649         if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
 650                 tsi721_pw_handler(priv);
 651
 652         return IRQ_HANDLED;
 653 }
 654
 655 /**
 656  * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
 657  * @irq: Linux interrupt number
 658  * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
 659  *
 660  * Handles Tsi721 interrupts from SR2PC Channel.
 661  * NOTE: At this moment services only one SR2PC channel associated with inbound
 662  * doorbells.
 663  */
 664 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
 665 {
 666         struct tsi721_device *priv = (struct tsi721_device *)ptr;
 667         u32 sr_ch_int;
 668
 669         /* Service Inbound DB interrupt from SR2PC channel */
 670         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 671         if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
 672                 tsi721_dbell_handler(priv);
 673
 674         /* Clear interrupts */
 675         iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 676         /* Read back to ensure that interrupt was cleared */
 677         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 678
 679         return IRQ_HANDLED;
 680 }
 681
 682 /**
 683  * tsi721_request_msix - register interrupt service for MSI-X mode.
 684  * @priv: tsi721 device-specific data structure
 685  *
 686  * Registers MSI-X interrupt service routines for interrupts that are active
 687  * immediately after mport initialization. Messaging interrupt service routines
 688  * should be registered during corresponding open requests.
 689  */
 690 static int tsi721_request_msix(struct tsi721_device *priv)
 691 {
 692         int err = 0;
 693
 694         err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
 695                         tsi721_sr2pc_ch_msix, 0,
 696                         priv->msix[TSI721_VECT_IDB].irq_name, (void *)priv);
 697         if (err)
 698                 return err;
 699
 700         err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
 701                         tsi721_srio_msix, 0,
 702                         priv->msix[TSI721_VECT_PWRX].irq_name, (void *)priv);
 703         if (err) {
 704                 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
 705                 return err;
 706         }
 707
 708         return 0;
 709 }
 710
 711 /**
 712  * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
 713  * @priv: pointer to tsi721 private data
 714  *
 715  * Configures MSI-X support for Tsi721. Supports only an exact number
 716  * of requested vectors.
 717  */
 718 static int tsi721_enable_msix(struct tsi721_device *priv)
 719 {
 720         struct msix_entry entries[TSI721_VECT_MAX];
 721         int err;
 722         int i;
 723
 724         entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
 725         entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
 726
 727         /*
 728          * Initialize MSI-X entries for Messaging Engine:
 729          * this driver supports four RIO mailboxes (inbound and outbound)
 730          * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
 731          * offset +4 is added to IB MBOX number.
 732          */
 733         for (i = 0; i < RIO_MAX_MBOX; i++) {
 734                 entries[TSI721_VECT_IMB0_RCV + i].entry =
 735                                         TSI721_MSIX_IMSG_DQ_RCV(i + 4);
 736                 entries[TSI721_VECT_IMB0_INT + i].entry =
 737                                         TSI721_MSIX_IMSG_INT(i + 4);
 738                 entries[TSI721_VECT_OMB0_DONE + i].entry =
 739                                         TSI721_MSIX_OMSG_DONE(i);
 740                 entries[TSI721_VECT_OMB0_INT + i].entry =
 741                                         TSI721_MSIX_OMSG_INT(i);
 742         }
 743
 744 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 745         /*
 746          * Initialize MSI-X entries for Block DMA Engine:
 747          * this driver supports XXX DMA channels
 748          * (one is reserved for SRIO maintenance transactions)
 749          */
 750         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
 751                 entries[TSI721_VECT_DMA0_DONE + i].entry =
 752                                         TSI721_MSIX_DMACH_DONE(i);
 753                 entries[TSI721_VECT_DMA0_INT + i].entry =
 754                                         TSI721_MSIX_DMACH_INT(i);
 755         }
 756 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
 757
 758         err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries));
 759         if (err) {
 760                 tsi_err(&priv->pdev->dev,
 761                         "Failed to enable MSI-X (err=%d)", err);
 762                 return err;
 763         }
 764
 765         /*
 766          * Copy MSI-X vector information into tsi721 private structure
 767          */
 768         priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
 769         snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
 770                  DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
 771         priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
 772         snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
 773                  DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
 774
 775         for (i = 0; i < RIO_MAX_MBOX; i++) {
 776                 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
 777                                 entries[TSI721_VECT_IMB0_RCV + i].vector;
 778                 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
 779                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
 780                          i, pci_name(priv->pdev));
 781
 782                 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
 783                                 entries[TSI721_VECT_IMB0_INT + i].vector;
 784                 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
 785                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
 786                          i, pci_name(priv->pdev));
 787
 788                 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
 789                                 entries[TSI721_VECT_OMB0_DONE + i].vector;
 790                 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
 791                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
 792                          i, pci_name(priv->pdev));
 793
 794                 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
 795                                 entries[TSI721_VECT_OMB0_INT + i].vector;
 796                 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
 797                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
 798                          i, pci_name(priv->pdev));
 799         }
 800
 801 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 802         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
 803                 priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
 804                                 entries[TSI721_VECT_DMA0_DONE + i].vector;
 805                 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
 806                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
 807                          i, pci_name(priv->pdev));
 808
 809                 priv->msix[TSI721_VECT_DMA0_INT + i].vector =
 810                                 entries[TSI721_VECT_DMA0_INT + i].vector;
 811                 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
 812                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
 813                          i, pci_name(priv->pdev));
 814         }
 815 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
 816
 817         return 0;
 818 }
 819 #endif /* CONFIG_PCI_MSI */
 820
 821 static int tsi721_request_irq(struct tsi721_device *priv)
 822 {
 823         int err;
 824
 825 #ifdef CONFIG_PCI_MSI
 826         if (priv->flags & TSI721_USING_MSIX)
 827                 err = tsi721_request_msix(priv);
 828         else
 829 #endif
 830                 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
 831                           (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
 832                           DRV_NAME, (void *)priv);
 833
 834         if (err)
 835                 tsi_err(&priv->pdev->dev,
 836                         "Unable to allocate interrupt, err=%d", err);
 837
 838         return err;
 839 }
 840
 841 static void tsi721_free_irq(struct tsi721_device *priv)
 842 {
 843 #ifdef CONFIG_PCI_MSI
 844         if (priv->flags & TSI721_USING_MSIX) {
 845                 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
 846                 free_irq(priv->msix[TSI721_VECT_PWRX].vector, (void *)priv);
 847         } else
 848 #endif
 849         free_irq(priv->pdev->irq, (void *)priv);
 850 }
 851
 852 static int
 853 tsi721_obw_alloc(struct tsi721_device *priv, struct tsi721_obw_bar *pbar,
 854                  u32 size, int *win_id)
 855 {
 856         u64 win_base;
 857         u64 bar_base;
 858         u64 bar_end;
 859         u32 align;
 860         struct tsi721_ob_win *win;
 861         struct tsi721_ob_win *new_win = NULL;
 862         int new_win_idx = -1;
 863         int i = 0;
 864
 865         bar_base = pbar->base;
 866         bar_end =  bar_base + pbar->size;
 867         win_base = bar_base;
 868         align = size/TSI721_PC2SR_ZONES;
 869
 870         while (i < TSI721_IBWIN_NUM) {
 871                 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
 872                         if (!priv->ob_win[i].active) {
 873                                 if (new_win == NULL) {
 874                                         new_win = &priv->ob_win[i];
 875                                         new_win_idx = i;
 876                                 }
 877                                 continue;
 878                         }
 879
 880                         /*
 881                          * If this window belongs to the current BAR check it
 882                          * for overlap
 883                          */
 884                         win = &priv->ob_win[i];
 885
 886                         if (win->base >= bar_base && win->base < bar_end) {
 887                                 if (win_base < (win->base + win->size) &&
 888                                                 (win_base + size) > win->base) {
 889                                         /* Overlap detected */
 890                                         win_base = win->base + win->size;
 891                                         win_base = ALIGN(win_base, align);
 892                                         break;
 893                                 }
 894                         }
 895                 }
 896         }
 897
 898         if (win_base + size > bar_end)
 899                 return -ENOMEM;
 900
 901         if (!new_win) {
 902                 tsi_err(&priv->pdev->dev, "OBW count tracking failed");
 903                 return -EIO;
 904         }
 905
 906         new_win->active = true;
 907         new_win->base = win_base;
 908         new_win->size = size;
 909         new_win->pbar = pbar;
 910         priv->obwin_cnt--;
 911         pbar->free -= size;
 912         *win_id = new_win_idx;
 913         return 0;
 914 }
 915
 916 static int tsi721_map_outb_win(struct rio_mport *mport, u16 destid, u64 rstart,
 917                         u32 size, u32 flags, dma_addr_t *laddr)
 918 {
 919         struct tsi721_device *priv = mport->priv;
 920         int i;
 921         struct tsi721_obw_bar *pbar;
 922         struct tsi721_ob_win *ob_win;
 923         int obw = -1;
 924         u32 rval;
 925         u64 rio_addr;
 926         u32 zsize;
 927         int ret = -ENOMEM;
 928
 929         tsi_debug(OBW, &priv->pdev->dev,
 930                   "did=%d ra=0x%llx sz=0x%x", destid, rstart, size);
 931
 932         if (!is_power_of_2(size) || (size < 0x8000) || (rstart & (size - 1)))
 933                 return -EINVAL;
 934
 935         if (priv->obwin_cnt == 0)
 936                 return -EBUSY;
 937
 938         for (i = 0; i < 2; i++) {
 939                 if (priv->p2r_bar[i].free >= size) {
 940                         pbar = &priv->p2r_bar[i];
 941                         ret = tsi721_obw_alloc(priv, pbar, size, &obw);
 942                         if (!ret)
 943                                 break;
 944                 }
 945         }
 946
 947         if (ret)
 948                 return ret;
 949
 950         WARN_ON(obw == -1);
 951         ob_win = &priv->ob_win[obw];
 952         ob_win->destid = destid;
 953         ob_win->rstart = rstart;
 954         tsi_debug(OBW, &priv->pdev->dev,
 955                   "allocated OBW%d @%llx", obw, ob_win->base);
 956
 957         /*
 958          * Configure Outbound Window
 959          */
 960
 961         zsize = size/TSI721_PC2SR_ZONES;
 962         rio_addr = rstart;
 963
 964         /*
 965          * Program Address Translation Zones:
 966          *  This implementation uses all 8 zones associated wit window.
 967          */
 968         for (i = 0; i < TSI721_PC2SR_ZONES; i++) {
 969
 970                 while (ioread32(priv->regs + TSI721_ZONE_SEL) &
 971                         TSI721_ZONE_SEL_GO) {
 972                         udelay(1);
 973                 }
 974
 975                 rval = (u32)(rio_addr & TSI721_LUT_DATA0_ADD) |
 976                         TSI721_LUT_DATA0_NREAD | TSI721_LUT_DATA0_NWR;
 977                 iowrite32(rval, priv->regs + TSI721_LUT_DATA0);
 978                 rval = (u32)(rio_addr >> 32);
 979                 iowrite32(rval, priv->regs + TSI721_LUT_DATA1);
 980                 rval = destid;
 981                 iowrite32(rval, priv->regs + TSI721_LUT_DATA2);
 982
 983                 rval = TSI721_ZONE_SEL_GO | (obw << 3) | i;
 984                 iowrite32(rval, priv->regs + TSI721_ZONE_SEL);
 985
 986                 rio_addr += zsize;
 987         }
 988
 989         iowrite32(TSI721_OBWIN_SIZE(size) << 8,
 990                   priv->regs + TSI721_OBWINSZ(obw));
 991         iowrite32((u32)(ob_win->base >> 32), priv->regs + TSI721_OBWINUB(obw));
 992         iowrite32((u32)(ob_win->base & TSI721_OBWINLB_BA) | TSI721_OBWINLB_WEN,
 993                   priv->regs + TSI721_OBWINLB(obw));
 994
 995         *laddr = ob_win->base;
 996         return 0;
 997 }
 998
 999 static void tsi721_unmap_outb_win(struct rio_mport *mport,
1000                                   u16 destid, u64 rstart)
1001 {
1002         struct tsi721_device *priv = mport->priv;
1003         struct tsi721_ob_win *ob_win;
1004         int i;
1005
1006         tsi_debug(OBW, &priv->pdev->dev, "did=%d ra=0x%llx", destid, rstart);
1007
1008         for (i = 0; i < TSI721_OBWIN_NUM; i++) {
1009                 ob_win = &priv->ob_win[i];
1010
1011                 if (ob_win->active &&
1012                     ob_win->destid == destid && ob_win->rstart == rstart) {
1013                         tsi_debug(OBW, &priv->pdev->dev,
1014                                   "free OBW%d @%llx", i, ob_win->base);
1015                         ob_win->active = false;
1016                         iowrite32(0, priv->regs + TSI721_OBWINLB(i));
1017                         ob_win->pbar->free += ob_win->size;
1018                         priv->obwin_cnt++;
1019                         break;
1020                 }
1021         }
1022 }
1023
1024 /**
1025  * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
1026  * translation regions.
1027  * @priv: pointer to tsi721 private data
1028  *
1029  * Disables SREP translation regions.
1030  */
1031 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
1032 {
1033         int i, z;
1034         u32 rval;
1035
1036         /* Disable all PC2SR translation windows */
1037         for (i = 0; i < TSI721_OBWIN_NUM; i++)
1038                 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
1039
1040         /* Initialize zone lookup tables to avoid ECC errors on reads */
1041         iowrite32(0, priv->regs + TSI721_LUT_DATA0);
1042         iowrite32(0, priv->regs + TSI721_LUT_DATA1);
1043         iowrite32(0, priv->regs + TSI721_LUT_DATA2);
1044
1045         for (i = 0; i < TSI721_OBWIN_NUM; i++) {
1046                 for (z = 0; z < TSI721_PC2SR_ZONES; z++) {
1047                         while (ioread32(priv->regs + TSI721_ZONE_SEL) &
1048                                 TSI721_ZONE_SEL_GO) {
1049                                 udelay(1);
1050                         }
1051                         rval = TSI721_ZONE_SEL_GO | (i << 3) | z;
1052                         iowrite32(rval, priv->regs + TSI721_ZONE_SEL);
1053                 }
1054         }
1055
1056         if (priv->p2r_bar[0].size == 0 && priv->p2r_bar[1].size == 0) {
1057                 priv->obwin_cnt = 0;
1058                 return;
1059         }
1060
1061         priv->p2r_bar[0].free = priv->p2r_bar[0].size;
1062         priv->p2r_bar[1].free = priv->p2r_bar[1].size;
1063
1064         for (i = 0; i < TSI721_OBWIN_NUM; i++)
1065                 priv->ob_win[i].active = false;
1066
1067         priv->obwin_cnt = TSI721_OBWIN_NUM;
1068 }
1069
1070 /**
1071  * tsi721_rio_map_inb_mem -- Mapping inbound memory region.
1072  * @mport: RapidIO master port
1073  * @lstart: Local memory space start address.
1074  * @rstart: RapidIO space start address.
1075  * @size: The mapping region size.
1076  * @flags: Flags for mapping. 0 for using default flags.
1077  *
1078  * Return: 0 -- Success.
1079  *
1080  * This function will create the inbound mapping
1081  * from rstart to lstart.
1082  */
1083 static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
1084                 u64 rstart, u32 size, u32 flags)
1085 {
1086         struct tsi721_device *priv = mport->priv;
1087         int i, avail = -1;
1088         u32 regval;
1089         struct tsi721_ib_win *ib_win;
1090         bool direct = (lstart == rstart);
1091         u64 ibw_size;
1092         dma_addr_t loc_start;
1093         u64 ibw_start;
1094         struct tsi721_ib_win_mapping *map = NULL;
1095         int ret = -EBUSY;
1096
1097         if (direct) {
1098                 /* Calculate minimal acceptable window size and base address */
1099
1100                 ibw_size = roundup_pow_of_two(size);
1101                 ibw_start = lstart & ~(ibw_size - 1);
1102
1103                 tsi_debug(IBW, &priv->pdev->dev,
1104                         "Direct (RIO_0x%llx -> PCIe_%pad), size=0x%x, ibw_start = 0x%llx",
1105                         rstart, &lstart, size, ibw_start);
1106
1107                 while ((lstart + size) > (ibw_start + ibw_size)) {
1108                         ibw_size *= 2;
1109                         ibw_start = lstart & ~(ibw_size - 1);
1110                         if (ibw_size > 0x80000000) { /* Limit max size to 2GB */
1111                                 return -EBUSY;
1112                         }
1113                 }
1114
1115                 loc_start = ibw_start;
1116
1117                 map = kzalloc(sizeof(struct tsi721_ib_win_mapping), GFP_ATOMIC);
1118                 if (map == NULL)
1119                         return -ENOMEM;
1120
1121         } else {
1122                 tsi_debug(IBW, &priv->pdev->dev,
1123                         "Translated (RIO_0x%llx -> PCIe_%pad), size=0x%x",
1124                         rstart, &lstart, size);
1125
1126                 if (!is_power_of_2(size) || size < 0x1000 ||
1127                     ((u64)lstart & (size - 1)) || (rstart & (size - 1)))
1128                         return -EINVAL;
1129                 if (priv->ibwin_cnt == 0)
1130                         return -EBUSY;
1131                 ibw_start = rstart;
1132                 ibw_size = size;
1133                 loc_start = lstart;
1134         }
1135
1136         /*
1137          * Scan for overlapping with active regions and mark the first available
1138          * IB window at the same time.
1139          */
1140         for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1141                 ib_win = &priv->ib_win[i];
1142
1143                 if (!ib_win->active) {
1144                         if (avail == -1) {
1145                                 avail = i;
1146                                 ret = 0;
1147                         }
1148                 } else if (ibw_start < (ib_win->rstart + ib_win->size) &&
1149                            (ibw_start + ibw_size) > ib_win->rstart) {
1150                         /* Return error if address translation involved */
1151                         if (direct && ib_win->xlat) {
1152                                 ret = -EFAULT;
1153                                 break;
1154                         }
1155
1156                         /*
1157                          * Direct mappings usually are larger than originally
1158                          * requested fragments - check if this new request fits
1159                          * into it.
1160                          */
1161                         if (rstart >= ib_win->rstart &&
1162                             (rstart + size) <= (ib_win->rstart +
1163                                                         ib_win->size)) {
1164                                 /* We are in - no further mapping required */
1165                                 map->lstart = lstart;
1166                                 list_add_tail(&map->node, &ib_win->mappings);
1167                                 return 0;
1168                         }
1169
1170                         ret = -EFAULT;
1171                         break;
1172                 }
1173         }
1174
1175         if (ret)
1176                 goto out;
1177         i = avail;
1178
1179         /* Sanity check: available IB window must be disabled at this point */
1180         regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
1181         if (WARN_ON(regval & TSI721_IBWIN_LB_WEN)) {
1182                 ret = -EIO;
1183                 goto out;
1184         }
1185
1186         ib_win = &priv->ib_win[i];
1187         ib_win->active = true;
1188         ib_win->rstart = ibw_start;
1189         ib_win->lstart = loc_start;
1190         ib_win->size = ibw_size;
1191         ib_win->xlat = (lstart != rstart);
1192         INIT_LIST_HEAD(&ib_win->mappings);
1193
1194         /*
1195          * When using direct IBW mapping and have larger than requested IBW size
1196          * we can have multiple local memory blocks mapped through the same IBW
1197          * To handle this situation we maintain list of "clients" for such IBWs.
1198          */
1199         if (direct) {
1200                 map->lstart = lstart;
1201                 list_add_tail(&map->node, &ib_win->mappings);
1202         }
1203
1204         iowrite32(TSI721_IBWIN_SIZE(ibw_size) << 8,
1205                         priv->regs + TSI721_IBWIN_SZ(i));
1206
1207         iowrite32(((u64)loc_start >> 32), priv->regs + TSI721_IBWIN_TUA(i));
1208         iowrite32(((u64)loc_start & TSI721_IBWIN_TLA_ADD),
1209                   priv->regs + TSI721_IBWIN_TLA(i));
1210
1211         iowrite32(ibw_start >> 32, priv->regs + TSI721_IBWIN_UB(i));
1212         iowrite32((ibw_start & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN,
1213                 priv->regs + TSI721_IBWIN_LB(i));
1214
1215         priv->ibwin_cnt--;
1216
1217         tsi_debug(IBW, &priv->pdev->dev,
1218                 "Configured IBWIN%d (RIO_0x%llx -> PCIe_%pad), size=0x%llx",
1219                 i, ibw_start, &loc_start, ibw_size);
1220
1221         return 0;
1222 out:
1223         kfree(map);
1224         return ret;
1225 }
1226
1227 /**
1228  * tsi721_rio_unmap_inb_mem -- Unmapping inbound memory region.
1229  * @mport: RapidIO master port
1230  * @lstart: Local memory space start address.
1231  */
1232 static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport,
1233                                 dma_addr_t lstart)
1234 {
1235         struct tsi721_device *priv = mport->priv;
1236         struct tsi721_ib_win *ib_win;
1237         int i;
1238
1239         tsi_debug(IBW, &priv->pdev->dev,
1240                 "Unmap IBW mapped to PCIe_%pad", &lstart);
1241
1242         /* Search for matching active inbound translation window */
1243         for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1244                 ib_win = &priv->ib_win[i];
1245
1246                 /* Address translating IBWs must to be an exact march */
1247                 if (!ib_win->active ||
1248                     (ib_win->xlat && lstart != ib_win->lstart))
1249                         continue;
1250
1251                 if (lstart >= ib_win->lstart &&
1252                     lstart < (ib_win->lstart + ib_win->size)) {
1253
1254                         if (!ib_win->xlat) {
1255                                 struct tsi721_ib_win_mapping *map;
1256                                 int found = 0;
1257
1258                                 list_for_each_entry(map,
1259                                                     &ib_win->mappings, node) {
1260                                         if (map->lstart == lstart) {
1261                                                 list_del(&map->node);
1262                                                 kfree(map);
1263                                                 found = 1;
1264                                                 break;
1265                                         }
1266                                 }
1267
1268                                 if (!found)
1269                                         continue;
1270
1271                                 if (!list_empty(&ib_win->mappings))
1272                                         break;
1273                         }
1274
1275                         tsi_debug(IBW, &priv->pdev->dev, "Disable IBWIN_%d", i);
1276                         iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1277                         ib_win->active = false;
1278                         priv->ibwin_cnt++;
1279                         break;
1280                 }
1281         }
1282
1283         if (i == TSI721_IBWIN_NUM)
1284                 tsi_debug(IBW, &priv->pdev->dev,
1285                         "IB window mapped to %pad not found", &lstart);
1286 }
1287
1288 /**
1289  * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
1290  * translation regions.
1291  * @priv: pointer to tsi721 private data
1292  *
1293  * Disables inbound windows.
1294  */
1295 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
1296 {
1297         int i;
1298
1299         /* Disable all SR2PC inbound windows */
1300         for (i = 0; i < TSI721_IBWIN_NUM; i++)
1301                 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1302         priv->ibwin_cnt = TSI721_IBWIN_NUM;
1303 }
1304
1305 /*
1306  * tsi721_close_sr2pc_mapping - closes all active inbound (SRIO->PCIe)
1307  * translation regions.
1308  * @priv: pointer to tsi721 device private data
1309  */
1310 static void tsi721_close_sr2pc_mapping(struct tsi721_device *priv)
1311 {
1312         struct tsi721_ib_win *ib_win;
1313         int i;
1314
1315         /* Disable all active SR2PC inbound windows */
1316         for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1317                 ib_win = &priv->ib_win[i];
1318                 if (ib_win->active) {
1319                         iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1320                         ib_win->active = false;
1321                 }
1322         }
1323 }
1324
1325 /**
1326  * tsi721_port_write_init - Inbound port write interface init
1327  * @priv: pointer to tsi721 private data
1328  *
1329  * Initializes inbound port write handler.
1330  * Returns %0 on success or %-ENOMEM on failure.
1331  */
1332 static int tsi721_port_write_init(struct tsi721_device *priv)
1333 {
1334         priv->pw_discard_count = 0;
1335         INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
1336         spin_lock_init(&priv->pw_fifo_lock);
1337         if (kfifo_alloc(&priv->pw_fifo,
1338                         TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
1339                 tsi_err(&priv->pdev->dev, "PW FIFO allocation failed");
1340                 return -ENOMEM;
1341         }
1342
1343         /* Use reliable port-write capture mode */
1344         iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
1345         return 0;
1346 }
1347
1348 static void tsi721_port_write_free(struct tsi721_device *priv)
1349 {
1350         kfifo_free(&priv->pw_fifo);
1351 }
1352
1353 static int tsi721_doorbell_init(struct tsi721_device *priv)
1354 {
1355         /* Outbound Doorbells do not require any setup.
1356          * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
1357          * That BAR1 was mapped during the probe routine.
1358          */
1359
1360         /* Initialize Inbound Doorbell processing DPC and queue */
1361         priv->db_discard_count = 0;
1362         INIT_WORK(&priv->idb_work, tsi721_db_dpc);
1363
1364         /* Allocate buffer for inbound doorbells queue */
1365         priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
1366                                 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1367                                 &priv->idb_dma, GFP_KERNEL);
1368         if (!priv->idb_base)
1369                 return -ENOMEM;
1370
1371         tsi_debug(DBELL, &priv->pdev->dev,
1372                   "Allocated IDB buffer @ %p (phys = %pad)",
1373                   priv->idb_base, &priv->idb_dma);
1374
1375         iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
1376                 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
1377         iowrite32(((u64)priv->idb_dma >> 32),
1378                 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
1379         iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
1380                 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
1381         /* Enable accepting all inbound doorbells */
1382         iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
1383
1384         iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
1385
1386         iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
1387
1388         return 0;
1389 }
1390
1391 static void tsi721_doorbell_free(struct tsi721_device *priv)
1392 {
1393         if (priv->idb_base == NULL)
1394                 return;
1395
1396         /* Free buffer allocated for inbound doorbell queue */
1397         dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1398                           priv->idb_base, priv->idb_dma);
1399         priv->idb_base = NULL;
1400 }
1401
1402 /**
1403  * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
1404  * @priv: pointer to tsi721 private data
1405  *
1406  * Initialize BDMA channel allocated for RapidIO maintenance read/write
1407  * request generation
1408  * Returns %0 on success or %-ENOMEM on failure.
1409  */
1410 static int tsi721_bdma_maint_init(struct tsi721_device *priv)
1411 {
1412         struct tsi721_dma_desc *bd_ptr;
1413         u64             *sts_ptr;
1414         dma_addr_t      bd_phys, sts_phys;
1415         int             sts_size;
1416         int             bd_num = 2;
1417         void __iomem    *regs;
1418
1419         tsi_debug(MAINT, &priv->pdev->dev,
1420                   "Init BDMA_%d Maintenance requests", TSI721_DMACH_MAINT);
1421
1422         /*
1423          * Initialize DMA channel for maintenance requests
1424          */
1425
1426         priv->mdma.ch_id = TSI721_DMACH_MAINT;
1427         regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
1428
1429         /* Allocate space for DMA descriptors */
1430         bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
1431                                         bd_num * sizeof(struct tsi721_dma_desc),
1432                                         &bd_phys, GFP_KERNEL);
1433         if (!bd_ptr)
1434                 return -ENOMEM;
1435
1436         priv->mdma.bd_num = bd_num;
1437         priv->mdma.bd_phys = bd_phys;
1438         priv->mdma.bd_base = bd_ptr;
1439
1440         tsi_debug(MAINT, &priv->pdev->dev, "DMA descriptors @ %p (phys = %pad)",
1441                   bd_ptr, &bd_phys);
1442
1443         /* Allocate space for descriptor status FIFO */
1444         sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
1445                                         bd_num : TSI721_DMA_MINSTSSZ;
1446         sts_size = roundup_pow_of_two(sts_size);
1447         sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
1448                                      sts_size * sizeof(struct tsi721_dma_sts),
1449                                      &sts_phys, GFP_KERNEL);
1450         if (!sts_ptr) {
1451                 /* Free space allocated for DMA descriptors */
1452                 dma_free_coherent(&priv->pdev->dev,
1453                                   bd_num * sizeof(struct tsi721_dma_desc),
1454                                   bd_ptr, bd_phys);
1455                 priv->mdma.bd_base = NULL;
1456                 return -ENOMEM;
1457         }
1458
1459         priv->mdma.sts_phys = sts_phys;
1460         priv->mdma.sts_base = sts_ptr;
1461         priv->mdma.sts_size = sts_size;
1462
1463         tsi_debug(MAINT, &priv->pdev->dev,
1464                 "desc status FIFO @ %p (phys = %pad) size=0x%x",
1465                 sts_ptr, &sts_phys, sts_size);
1466
1467         /* Initialize DMA descriptors ring */
1468         bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
1469         bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
1470                                                  TSI721_DMAC_DPTRL_MASK);
1471         bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
1472
1473         /* Setup DMA descriptor pointers */
1474         iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
1475         iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
1476                 regs + TSI721_DMAC_DPTRL);
1477
1478         /* Setup descriptor status FIFO */
1479         iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
1480         iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
1481                 regs + TSI721_DMAC_DSBL);
1482         iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
1483                 regs + TSI721_DMAC_DSSZ);
1484
1485         /* Clear interrupt bits */
1486         iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
1487
1488         ioread32(regs + TSI721_DMAC_INT);
1489
1490         /* Toggle DMA channel initialization */
1491         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1492         ioread32(regs + TSI721_DMAC_CTL);
1493         udelay(10);
1494
1495         return 0;
1496 }
1497
1498 static int tsi721_bdma_maint_free(struct tsi721_device *priv)
1499 {
1500         u32 ch_stat;
1501         struct tsi721_bdma_maint *mdma = &priv->mdma;
1502         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
1503
1504         if (mdma->bd_base == NULL)
1505                 return 0;
1506
1507         /* Check if DMA channel still running */
1508         ch_stat = ioread32(regs + TSI721_DMAC_STS);
1509         if (ch_stat & TSI721_DMAC_STS_RUN)
1510                 return -EFAULT;
1511
1512         /* Put DMA channel into init state */
1513         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1514
1515         /* Free space allocated for DMA descriptors */
1516         dma_free_coherent(&priv->pdev->dev,
1517                 mdma->bd_num * sizeof(struct tsi721_dma_desc),
1518                 mdma->bd_base, mdma->bd_phys);
1519         mdma->bd_base = NULL;
1520
1521         /* Free space allocated for status FIFO */
1522         dma_free_coherent(&priv->pdev->dev,
1523                 mdma->sts_size * sizeof(struct tsi721_dma_sts),
1524                 mdma->sts_base, mdma->sts_phys);
1525         mdma->sts_base = NULL;
1526         return 0;
1527 }
1528
1529 /* Enable Inbound Messaging Interrupts */
1530 static void
1531 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1532                                   u32 inte_mask)
1533 {
1534         u32 rval;
1535
1536         if (!inte_mask)
1537                 return;
1538
1539         /* Clear pending Inbound Messaging interrupts */
1540         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1541
1542         /* Enable Inbound Messaging interrupts */
1543         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1544         iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1545
1546         if (priv->flags & TSI721_USING_MSIX)
1547                 return; /* Finished if we are in MSI-X mode */
1548
1549         /*
1550          * For MSI and INTA interrupt signalling we need to enable next levels
1551          */
1552
1553         /* Enable Device Channel Interrupt */
1554         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1555         iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1556                   priv->regs + TSI721_DEV_CHAN_INTE);
1557 }
1558
1559 /* Disable Inbound Messaging Interrupts */
1560 static void
1561 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1562                                    u32 inte_mask)
1563 {
1564         u32 rval;
1565
1566         if (!inte_mask)
1567                 return;
1568
1569         /* Clear pending Inbound Messaging interrupts */
1570         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1571
1572         /* Disable Inbound Messaging interrupts */
1573         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1574         rval &= ~inte_mask;
1575         iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1576
1577         if (priv->flags & TSI721_USING_MSIX)
1578                 return; /* Finished if we are in MSI-X mode */
1579
1580         /*
1581          * For MSI and INTA interrupt signalling we need to disable next levels
1582          */
1583
1584         /* Disable Device Channel Interrupt */
1585         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1586         rval &= ~TSI721_INT_IMSG_CHAN(ch);
1587         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1588 }
1589
1590 /* Enable Outbound Messaging interrupts */
1591 static void
1592 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1593                                   u32 inte_mask)
1594 {
1595         u32 rval;
1596
1597         if (!inte_mask)
1598                 return;
1599
1600         /* Clear pending Outbound Messaging interrupts */
1601         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1602
1603         /* Enable Outbound Messaging channel interrupts */
1604         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1605         iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1606
1607         if (priv->flags & TSI721_USING_MSIX)
1608                 return; /* Finished if we are in MSI-X mode */
1609
1610         /*
1611          * For MSI and INTA interrupt signalling we need to enable next levels
1612          */
1613
1614         /* Enable Device Channel Interrupt */
1615         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1616         iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1617                   priv->regs + TSI721_DEV_CHAN_INTE);
1618 }
1619
1620 /* Disable Outbound Messaging interrupts */
1621 static void
1622 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1623                                    u32 inte_mask)
1624 {
1625         u32 rval;
1626
1627         if (!inte_mask)
1628                 return;
1629
1630         /* Clear pending Outbound Messaging interrupts */
1631         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1632
1633         /* Disable Outbound Messaging interrupts */
1634         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1635         rval &= ~inte_mask;
1636         iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1637
1638         if (priv->flags & TSI721_USING_MSIX)
1639                 return; /* Finished if we are in MSI-X mode */
1640
1641         /*
1642          * For MSI and INTA interrupt signalling we need to disable next levels
1643          */
1644
1645         /* Disable Device Channel Interrupt */
1646         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1647         rval &= ~TSI721_INT_OMSG_CHAN(ch);
1648         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1649 }
1650
1651 /**
1652  * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1653  * @mport: Master port with outbound message queue
1654  * @rdev: Target of outbound message
1655  * @mbox: Outbound mailbox
1656  * @buffer: Message to add to outbound queue
1657  * @len: Length of message
1658  */
1659 static int
1660 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1661                         void *buffer, size_t len)
1662 {
1663         struct tsi721_device *priv = mport->priv;
1664         struct tsi721_omsg_desc *desc;
1665         u32 tx_slot;
1666         unsigned long flags;
1667
1668         if (!priv->omsg_init[mbox] ||
1669             len > TSI721_MSG_MAX_SIZE || len < 8)
1670                 return -EINVAL;
1671
1672         spin_lock_irqsave(&priv->omsg_ring[mbox].lock, flags);
1673
1674         tx_slot = priv->omsg_ring[mbox].tx_slot;
1675
1676         /* Copy copy message into transfer buffer */
1677         memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1678
1679         if (len & 0x7)
1680                 len += 8;
1681
1682         /* Build descriptor associated with buffer */
1683         desc = priv->omsg_ring[mbox].omd_base;
1684         desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
1685 #ifdef TSI721_OMSG_DESC_INT
1686         /* Request IOF_DONE interrupt generation for each N-th frame in queue */
1687         if (tx_slot % 4 == 0)
1688                 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
1689 #endif
1690         desc[tx_slot].msg_info =
1691                 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1692                             (0xe << 12) | (len & 0xff8));
1693         desc[tx_slot].bufptr_lo =
1694                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1695                             0xffffffff);
1696         desc[tx_slot].bufptr_hi =
1697                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1698
1699         priv->omsg_ring[mbox].wr_count++;
1700
1701         /* Go to next descriptor */
1702         if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1703                 priv->omsg_ring[mbox].tx_slot = 0;
1704                 /* Move through the ring link descriptor at the end */
1705                 priv->omsg_ring[mbox].wr_count++;
1706         }
1707
1708         mb();
1709
1710         /* Set new write count value */
1711         iowrite32(priv->omsg_ring[mbox].wr_count,
1712                 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1713         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1714
1715         spin_unlock_irqrestore(&priv->omsg_ring[mbox].lock, flags);
1716
1717         return 0;
1718 }
1719
1720 /**
1721  * tsi721_omsg_handler - Outbound Message Interrupt Handler
1722  * @priv: pointer to tsi721 private data
1723  * @ch:   number of OB MSG channel to service
1724  *
1725  * Services channel interrupts from outbound messaging engine.
1726  */
1727 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1728 {
1729         u32 omsg_int;
1730         struct rio_mport *mport = &priv->mport;
1731         void *dev_id = NULL;
1732         u32 tx_slot = 0xffffffff;
1733         int do_callback = 0;
1734
1735         spin_lock(&priv->omsg_ring[ch].lock);
1736
1737         omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1738
1739         if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1740                 tsi_info(&priv->pdev->dev,
1741                         "OB MBOX%d: Status FIFO is full", ch);
1742
1743         if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1744                 u32 srd_ptr;
1745                 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1746                 int i, j;
1747
1748                 /*
1749                  * Find last successfully processed descriptor
1750                  */
1751
1752                 /* Check and clear descriptor status FIFO entries */
1753                 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1754                 sts_ptr = priv->omsg_ring[ch].sts_base;
1755                 j = srd_ptr * 8;
1756                 while (sts_ptr[j]) {
1757                         for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1758                                 prev_ptr = last_ptr;
1759                                 last_ptr = le64_to_cpu(sts_ptr[j]);
1760                                 sts_ptr[j] = 0;
1761                         }
1762
1763                         ++srd_ptr;
1764                         srd_ptr %= priv->omsg_ring[ch].sts_size;
1765                         j = srd_ptr * 8;
1766                 }
1767
1768                 if (last_ptr == 0)
1769                         goto no_sts_update;
1770
1771                 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1772                 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1773
1774                 if (!mport->outb_msg[ch].mcback)
1775                         goto no_sts_update;
1776
1777                 /* Inform upper layer about transfer completion */
1778
1779                 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1780                                                 sizeof(struct tsi721_omsg_desc);
1781
1782                 /*
1783                  * Check if this is a Link Descriptor (LD).
1784                  * If yes, ignore LD and use descriptor processed
1785                  * before LD.
1786                  */
1787                 if (tx_slot == priv->omsg_ring[ch].size) {
1788                         if (prev_ptr)
1789                                 tx_slot = (prev_ptr -
1790                                         (u64)priv->omsg_ring[ch].omd_phys)/
1791                                                 sizeof(struct tsi721_omsg_desc);
1792                         else
1793                                 goto no_sts_update;
1794                 }
1795
1796                 if (tx_slot >= priv->omsg_ring[ch].size)
1797                         tsi_debug(OMSG, &priv->pdev->dev,
1798                                   "OB_MSG tx_slot=%x > size=%x",
1799                                   tx_slot, priv->omsg_ring[ch].size);
1800                 WARN_ON(tx_slot >= priv->omsg_ring[ch].size);
1801
1802                 /* Move slot index to the next message to be sent */
1803                 ++tx_slot;
1804                 if (tx_slot == priv->omsg_ring[ch].size)
1805                         tx_slot = 0;
1806
1807                 dev_id = priv->omsg_ring[ch].dev_id;
1808                 do_callback = 1;
1809         }
1810
1811 no_sts_update:
1812
1813         if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1814                 /*
1815                 * Outbound message operation aborted due to error,
1816                 * reinitialize OB MSG channel
1817                 */
1818
1819                 tsi_debug(OMSG, &priv->pdev->dev, "OB MSG ABORT ch_stat=%x",
1820                           ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1821
1822                 iowrite32(TSI721_OBDMAC_INT_ERROR,
1823                                 priv->regs + TSI721_OBDMAC_INT(ch));
1824                 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
1825                                 priv->regs + TSI721_OBDMAC_CTL(ch));
1826                 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1827
1828                 /* Inform upper level to clear all pending tx slots */
1829                 dev_id = priv->omsg_ring[ch].dev_id;
1830                 tx_slot = priv->omsg_ring[ch].tx_slot;
1831                 do_callback = 1;
1832
1833                 /* Synch tx_slot tracking */
1834                 iowrite32(priv->omsg_ring[ch].tx_slot,
1835                         priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1836                 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1837                 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1838                 priv->omsg_ring[ch].sts_rdptr = 0;
1839         }
1840
1841         /* Clear channel interrupts */
1842         iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1843
1844         if (!(priv->flags & TSI721_USING_MSIX)) {
1845                 u32 ch_inte;
1846
1847                 /* Re-enable channel interrupts */
1848                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1849                 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1850                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1851         }
1852
1853         spin_unlock(&priv->omsg_ring[ch].lock);
1854
1855         if (mport->outb_msg[ch].mcback && do_callback)
1856                 mport->outb_msg[ch].mcback(mport, dev_id, ch, tx_slot);
1857 }
1858
1859 /**
1860  * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1861  * @mport: Master port implementing Outbound Messaging Engine
1862  * @dev_id: Device specific pointer to pass on event
1863  * @mbox: Mailbox to open
1864  * @entries: Number of entries in the outbound mailbox ring
1865  */
1866 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1867                                  int mbox, int entries)
1868 {
1869         struct tsi721_device *priv = mport->priv;
1870         struct tsi721_omsg_desc *bd_ptr;
1871         int i, rc = 0;
1872
1873         if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1874             (entries > (TSI721_OMSGD_RING_SIZE)) ||
1875             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1876                 rc = -EINVAL;
1877                 goto out;
1878         }
1879
1880         priv->omsg_ring[mbox].dev_id = dev_id;
1881         priv->omsg_ring[mbox].size = entries;
1882         priv->omsg_ring[mbox].sts_rdptr = 0;
1883         spin_lock_init(&priv->omsg_ring[mbox].lock);
1884
1885         /* Outbound Msg Buffer allocation based on
1886            the number of maximum descriptor entries */
1887         for (i = 0; i < entries; i++) {
1888                 priv->omsg_ring[mbox].omq_base[i] =
1889                         dma_alloc_coherent(
1890                                 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1891                                 &priv->omsg_ring[mbox].omq_phys[i],
1892                                 GFP_KERNEL);
1893                 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1894                         tsi_debug(OMSG, &priv->pdev->dev,
1895                                   "ENOMEM for OB_MSG_%d data buffer", mbox);
1896                         rc = -ENOMEM;
1897                         goto out_buf;
1898                 }
1899         }
1900
1901         /* Outbound message descriptor allocation */
1902         priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1903                                 &priv->pdev->dev,
1904                                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1905                                 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1906         if (priv->omsg_ring[mbox].omd_base == NULL) {
1907                 tsi_debug(OMSG, &priv->pdev->dev,
1908                         "ENOMEM for OB_MSG_%d descriptor memory", mbox);
1909                 rc = -ENOMEM;
1910                 goto out_buf;
1911         }
1912
1913         priv->omsg_ring[mbox].tx_slot = 0;
1914
1915         /* Outbound message descriptor status FIFO allocation */
1916         priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
1917         priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
1918                         priv->omsg_ring[mbox].sts_size *
1919                                                 sizeof(struct tsi721_dma_sts),
1920                         &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1921         if (priv->omsg_ring[mbox].sts_base == NULL) {
1922                 tsi_debug(OMSG, &priv->pdev->dev,
1923                         "ENOMEM for OB_MSG_%d status FIFO", mbox);
1924                 rc = -ENOMEM;
1925                 goto out_desc;
1926         }
1927
1928         /*
1929          * Configure Outbound Messaging Engine
1930          */
1931
1932         /* Setup Outbound Message descriptor pointer */
1933         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1934                         priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1935         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1936                                         TSI721_OBDMAC_DPTRL_MASK),
1937                         priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1938
1939         /* Setup Outbound Message descriptor status FIFO */
1940         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1941                         priv->regs + TSI721_OBDMAC_DSBH(mbox));
1942         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1943                                         TSI721_OBDMAC_DSBL_MASK),
1944                         priv->regs + TSI721_OBDMAC_DSBL(mbox));
1945         iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1946                 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1947
1948         /* Enable interrupts */
1949
1950 #ifdef CONFIG_PCI_MSI
1951         if (priv->flags & TSI721_USING_MSIX) {
1952                 int idx = TSI721_VECT_OMB0_DONE + mbox;
1953
1954                 /* Request interrupt service if we are in MSI-X mode */
1955                 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
1956                                  priv->msix[idx].irq_name, (void *)priv);
1957
1958                 if (rc) {
1959                         tsi_debug(OMSG, &priv->pdev->dev,
1960                                 "Unable to get MSI-X IRQ for OBOX%d-DONE",
1961                                 mbox);
1962                         goto out_stat;
1963                 }
1964
1965                 idx = TSI721_VECT_OMB0_INT + mbox;
1966                 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
1967                                  priv->msix[idx].irq_name, (void *)priv);
1968
1969                 if (rc) {
1970                         tsi_debug(OMSG, &priv->pdev->dev,
1971                                 "Unable to get MSI-X IRQ for MBOX%d-INT", mbox);
1972                         idx = TSI721_VECT_OMB0_DONE + mbox;
1973                         free_irq(priv->msix[idx].vector, (void *)priv);
1974                         goto out_stat;
1975                 }
1976         }
1977 #endif /* CONFIG_PCI_MSI */
1978
1979         tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1980
1981         /* Initialize Outbound Message descriptors ring */
1982         bd_ptr = priv->omsg_ring[mbox].omd_base;
1983         bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1984         bd_ptr[entries].msg_info = 0;
1985         bd_ptr[entries].next_lo =
1986                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1987                 TSI721_OBDMAC_DPTRL_MASK);
1988         bd_ptr[entries].next_hi =
1989                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1990         priv->omsg_ring[mbox].wr_count = 0;
1991         mb();
1992
1993         /* Initialize Outbound Message engine */
1994         iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
1995                   priv->regs + TSI721_OBDMAC_CTL(mbox));
1996         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1997         udelay(10);
1998
1999         priv->omsg_init[mbox] = 1;
2000
2001         return 0;
2002
2003 #ifdef CONFIG_PCI_MSI
2004 out_stat:
2005         dma_free_coherent(&priv->pdev->dev,
2006                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
2007                 priv->omsg_ring[mbox].sts_base,
2008                 priv->omsg_ring[mbox].sts_phys);
2009
2010         priv->omsg_ring[mbox].sts_base = NULL;
2011 #endif /* CONFIG_PCI_MSI */
2012
2013 out_desc:
2014         dma_free_coherent(&priv->pdev->dev,
2015                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
2016                 priv->omsg_ring[mbox].omd_base,
2017                 priv->omsg_ring[mbox].omd_phys);
2018
2019         priv->omsg_ring[mbox].omd_base = NULL;
2020
2021 out_buf:
2022         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
2023                 if (priv->omsg_ring[mbox].omq_base[i]) {
2024                         dma_free_coherent(&priv->pdev->dev,
2025                                 TSI721_MSG_BUFFER_SIZE,
2026                                 priv->omsg_ring[mbox].omq_base[i],
2027                                 priv->omsg_ring[mbox].omq_phys[i]);
2028
2029                         priv->omsg_ring[mbox].omq_base[i] = NULL;
2030                 }
2031         }
2032
2033 out:
2034         return rc;
2035 }
2036
2037 /**
2038  * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
2039  * @mport: Master port implementing the outbound message unit
2040  * @mbox: Mailbox to close
2041  */
2042 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
2043 {
2044         struct tsi721_device *priv = mport->priv;
2045         u32 i;
2046
2047         if (!priv->omsg_init[mbox])
2048                 return;
2049         priv->omsg_init[mbox] = 0;
2050
2051         /* Disable Interrupts */
2052
2053         tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
2054
2055 #ifdef CONFIG_PCI_MSI
2056         if (priv->flags & TSI721_USING_MSIX) {
2057                 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
2058                          (void *)priv);
2059                 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
2060                          (void *)priv);
2061         }
2062 #endif /* CONFIG_PCI_MSI */
2063
2064         /* Free OMSG Descriptor Status FIFO */
2065         dma_free_coherent(&priv->pdev->dev,
2066                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
2067                 priv->omsg_ring[mbox].sts_base,
2068                 priv->omsg_ring[mbox].sts_phys);
2069
2070         priv->omsg_ring[mbox].sts_base = NULL;
2071
2072         /* Free OMSG descriptors */
2073         dma_free_coherent(&priv->pdev->dev,
2074                 (priv->omsg_ring[mbox].size + 1) *
2075                         sizeof(struct tsi721_omsg_desc),
2076                 priv->omsg_ring[mbox].omd_base,
2077                 priv->omsg_ring[mbox].omd_phys);
2078
2079         priv->omsg_ring[mbox].omd_base = NULL;
2080
2081         /* Free message buffers */
2082         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
2083                 if (priv->omsg_ring[mbox].omq_base[i]) {
2084                         dma_free_coherent(&priv->pdev->dev,
2085                                 TSI721_MSG_BUFFER_SIZE,
2086                                 priv->omsg_ring[mbox].omq_base[i],
2087                                 priv->omsg_ring[mbox].omq_phys[i]);
2088
2089                         priv->omsg_ring[mbox].omq_base[i] = NULL;
2090                 }
2091         }
2092 }
2093
2094 /**
2095  * tsi721_imsg_handler - Inbound Message Interrupt Handler
2096  * @priv: pointer to tsi721 private data
2097  * @ch: inbound message channel number to service
2098  *
2099  * Services channel interrupts from inbound messaging engine.
2100  */
2101 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
2102 {
2103         u32 mbox = ch - 4;
2104         u32 imsg_int;
2105         struct rio_mport *mport = &priv->mport;
2106
2107         spin_lock(&priv->imsg_ring[mbox].lock);
2108
2109         imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
2110
2111         if (imsg_int & TSI721_IBDMAC_INT_SRTO)
2112                 tsi_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout", mbox);
2113
2114         if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
2115                 tsi_info(&priv->pdev->dev, "IB MBOX%d PCIe error", mbox);
2116
2117         if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
2118                 tsi_info(&priv->pdev->dev, "IB MBOX%d IB free queue low", mbox);
2119
2120         /* Clear IB channel interrupts */
2121         iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
2122
2123         /* If an IB Msg is received notify the upper layer */
2124         if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
2125                 mport->inb_msg[mbox].mcback)
2126                 mport->inb_msg[mbox].mcback(mport,
2127                                 priv->imsg_ring[mbox].dev_id, mbox, -1);
2128
2129         if (!(priv->flags & TSI721_USING_MSIX)) {
2130                 u32 ch_inte;
2131
2132                 /* Re-enable channel interrupts */
2133                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
2134                 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
2135                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
2136         }
2137
2138         spin_unlock(&priv->imsg_ring[mbox].lock);
2139 }
2140
2141 /**
2142  * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
2143  * @mport: Master port implementing the Inbound Messaging Engine
2144  * @dev_id: Device specific pointer to pass on event
2145  * @mbox: Mailbox to open
2146  * @entries: Number of entries in the inbound mailbox ring
2147  */
2148 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
2149                                 int mbox, int entries)
2150 {
2151         struct tsi721_device *priv = mport->priv;
2152         int ch = mbox + 4;
2153         int i;
2154         u64 *free_ptr;
2155         int rc = 0;
2156
2157         if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
2158             (entries > TSI721_IMSGD_RING_SIZE) ||
2159             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
2160                 rc = -EINVAL;
2161                 goto out;
2162         }
2163
2164         /* Initialize IB Messaging Ring */
2165         priv->imsg_ring[mbox].dev_id = dev_id;
2166         priv->imsg_ring[mbox].size = entries;
2167         priv->imsg_ring[mbox].rx_slot = 0;
2168         priv->imsg_ring[mbox].desc_rdptr = 0;
2169         priv->imsg_ring[mbox].fq_wrptr = 0;
2170         for (i = 0; i < priv->imsg_ring[mbox].size; i++)
2171                 priv->imsg_ring[mbox].imq_base[i] = NULL;
2172         spin_lock_init(&priv->imsg_ring[mbox].lock);
2173
2174         /* Allocate buffers for incoming messages */
2175         priv->imsg_ring[mbox].buf_base =
2176                 dma_alloc_coherent(&priv->pdev->dev,
2177                                    entries * TSI721_MSG_BUFFER_SIZE,
2178                                    &priv->imsg_ring[mbox].buf_phys,
2179                                    GFP_KERNEL);
2180
2181         if (priv->imsg_ring[mbox].buf_base == NULL) {
2182                 tsi_err(&priv->pdev->dev,
2183                         "Failed to allocate buffers for IB MBOX%d", mbox);
2184                 rc = -ENOMEM;
2185                 goto out;
2186         }
2187
2188         /* Allocate memory for circular free list */
2189         priv->imsg_ring[mbox].imfq_base =
2190                 dma_alloc_coherent(&priv->pdev->dev,
2191                                    entries * 8,
2192                                    &priv->imsg_ring[mbox].imfq_phys,
2193                                    GFP_KERNEL);
2194
2195         if (priv->imsg_ring[mbox].imfq_base == NULL) {
2196                 tsi_err(&priv->pdev->dev,
2197                         "Failed to allocate free queue for IB MBOX%d", mbox);
2198                 rc = -ENOMEM;
2199                 goto out_buf;
2200         }
2201
2202         /* Allocate memory for Inbound message descriptors */
2203         priv->imsg_ring[mbox].imd_base =
2204                 dma_alloc_coherent(&priv->pdev->dev,
2205                                    entries * sizeof(struct tsi721_imsg_desc),
2206                                    &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
2207
2208         if (priv->imsg_ring[mbox].imd_base == NULL) {
2209                 tsi_err(&priv->pdev->dev,
2210                         "Failed to allocate descriptor memory for IB MBOX%d",
2211                         mbox);
2212                 rc = -ENOMEM;
2213                 goto out_dma;
2214         }
2215
2216         /* Fill free buffer pointer list */
2217         free_ptr = priv->imsg_ring[mbox].imfq_base;
2218         for (i = 0; i < entries; i++)
2219                 free_ptr[i] = cpu_to_le64(
2220                                 (u64)(priv->imsg_ring[mbox].buf_phys) +
2221                                 i * 0x1000);
2222
2223         mb();
2224
2225         /*
2226          * For mapping of inbound SRIO Messages into appropriate queues we need
2227          * to set Inbound Device ID register in the messaging engine. We do it
2228          * once when first inbound mailbox is requested.
2229          */
2230         if (!(priv->flags & TSI721_IMSGID_SET)) {
2231                 iowrite32((u32)priv->mport.host_deviceid,
2232                         priv->regs + TSI721_IB_DEVID);
2233                 priv->flags |= TSI721_IMSGID_SET;
2234         }
2235
2236         /*
2237          * Configure Inbound Messaging channel (ch = mbox + 4)
2238          */
2239
2240         /* Setup Inbound Message free queue */
2241         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
2242                 priv->regs + TSI721_IBDMAC_FQBH(ch));
2243         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
2244                         TSI721_IBDMAC_FQBL_MASK),
2245                 priv->regs+TSI721_IBDMAC_FQBL(ch));
2246         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
2247                 priv->regs + TSI721_IBDMAC_FQSZ(ch));
2248
2249         /* Setup Inbound Message descriptor queue */
2250         iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
2251                 priv->regs + TSI721_IBDMAC_DQBH(ch));
2252         iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
2253                    (u32)TSI721_IBDMAC_DQBL_MASK),
2254                 priv->regs+TSI721_IBDMAC_DQBL(ch));
2255         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
2256                 priv->regs + TSI721_IBDMAC_DQSZ(ch));
2257
2258         /* Enable interrupts */
2259
2260 #ifdef CONFIG_PCI_MSI
2261         if (priv->flags & TSI721_USING_MSIX) {
2262                 int idx = TSI721_VECT_IMB0_RCV + mbox;
2263
2264                 /* Request interrupt service if we are in MSI-X mode */
2265                 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
2266                                  priv->msix[idx].irq_name, (void *)priv);
2267
2268                 if (rc) {
2269                         tsi_debug(IMSG, &priv->pdev->dev,
2270                                 "Unable to get MSI-X IRQ for IBOX%d-DONE",
2271                                 mbox);
2272                         goto out_desc;
2273                 }
2274
2275                 idx = TSI721_VECT_IMB0_INT + mbox;
2276                 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
2277                                  priv->msix[idx].irq_name, (void *)priv);
2278
2279                 if (rc) {
2280                         tsi_debug(IMSG, &priv->pdev->dev,
2281                                 "Unable to get MSI-X IRQ for IBOX%d-INT", mbox);
2282                         free_irq(
2283                                 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
2284                                 (void *)priv);
2285                         goto out_desc;
2286                 }
2287         }
2288 #endif /* CONFIG_PCI_MSI */
2289
2290         tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
2291
2292         /* Initialize Inbound Message Engine */
2293         iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
2294         ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
2295         udelay(10);
2296         priv->imsg_ring[mbox].fq_wrptr = entries - 1;
2297         iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
2298
2299         priv->imsg_init[mbox] = 1;
2300         return 0;
2301
2302 #ifdef CONFIG_PCI_MSI
2303 out_desc:
2304         dma_free_coherent(&priv->pdev->dev,
2305                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2306                 priv->imsg_ring[mbox].imd_base,
2307                 priv->imsg_ring[mbox].imd_phys);
2308
2309         priv->imsg_ring[mbox].imd_base = NULL;
2310 #endif /* CONFIG_PCI_MSI */
2311
2312 out_dma:
2313         dma_free_coherent(&priv->pdev->dev,
2314                 priv->imsg_ring[mbox].size * 8,
2315                 priv->imsg_ring[mbox].imfq_base,
2316                 priv->imsg_ring[mbox].imfq_phys);
2317
2318         priv->imsg_ring[mbox].imfq_base = NULL;
2319
2320 out_buf:
2321         dma_free_coherent(&priv->pdev->dev,
2322                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
2323                 priv->imsg_ring[mbox].buf_base,
2324                 priv->imsg_ring[mbox].buf_phys);
2325
2326         priv->imsg_ring[mbox].buf_base = NULL;
2327
2328 out:
2329         return rc;
2330 }
2331
2332 /**
2333  * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
2334  * @mport: Master port implementing the Inbound Messaging Engine
2335  * @mbox: Mailbox to close
2336  */
2337 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
2338 {
2339         struct tsi721_device *priv = mport->priv;
2340         u32 rx_slot;
2341         int ch = mbox + 4;
2342
2343         if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
2344                 return;
2345         priv->imsg_init[mbox] = 0;
2346
2347         /* Disable Inbound Messaging Engine */
2348
2349         /* Disable Interrupts */
2350         tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
2351
2352 #ifdef CONFIG_PCI_MSI
2353         if (priv->flags & TSI721_USING_MSIX) {
2354                 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
2355                                 (void *)priv);
2356                 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
2357                                 (void *)priv);
2358         }
2359 #endif /* CONFIG_PCI_MSI */
2360
2361         /* Clear Inbound Buffer Queue */
2362         for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
2363                 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2364
2365         /* Free memory allocated for message buffers */
2366         dma_free_coherent(&priv->pdev->dev,
2367                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
2368                 priv->imsg_ring[mbox].buf_base,
2369                 priv->imsg_ring[mbox].buf_phys);
2370
2371         priv->imsg_ring[mbox].buf_base = NULL;
2372
2373         /* Free memory allocated for free pointr list */
2374         dma_free_coherent(&priv->pdev->dev,
2375                 priv->imsg_ring[mbox].size * 8,
2376                 priv->imsg_ring[mbox].imfq_base,
2377                 priv->imsg_ring[mbox].imfq_phys);
2378
2379         priv->imsg_ring[mbox].imfq_base = NULL;
2380
2381         /* Free memory allocated for RX descriptors */
2382         dma_free_coherent(&priv->pdev->dev,
2383                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2384                 priv->imsg_ring[mbox].imd_base,
2385                 priv->imsg_ring[mbox].imd_phys);
2386
2387         priv->imsg_ring[mbox].imd_base = NULL;
2388 }
2389
2390 /**
2391  * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
2392  * @mport: Master port implementing the Inbound Messaging Engine
2393  * @mbox: Inbound mailbox number
2394  * @buf: Buffer to add to inbound queue
2395  */
2396 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
2397 {
2398         struct tsi721_device *priv = mport->priv;
2399         u32 rx_slot;
2400         int rc = 0;
2401
2402         rx_slot = priv->imsg_ring[mbox].rx_slot;
2403         if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
2404                 tsi_err(&priv->pdev->dev,
2405                         "Error adding inbound buffer %d, buffer exists",
2406                         rx_slot);
2407                 rc = -EINVAL;
2408                 goto out;
2409         }
2410
2411         priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
2412
2413         if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
2414                 priv->imsg_ring[mbox].rx_slot = 0;
2415
2416 out:
2417         return rc;
2418 }
2419
2420 /**
2421  * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
2422  * @mport: Master port implementing the Inbound Messaging Engine
2423  * @mbox: Inbound mailbox number
2424  *
2425  * Returns pointer to the message on success or NULL on failure.
2426  */
2427 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
2428 {
2429         struct tsi721_device *priv = mport->priv;
2430         struct tsi721_imsg_desc *desc;
2431         u32 rx_slot;
2432         void *rx_virt = NULL;
2433         u64 rx_phys;
2434         void *buf = NULL;
2435         u64 *free_ptr;
2436         int ch = mbox + 4;
2437         int msg_size;
2438
2439         if (!priv->imsg_init[mbox])
2440                 return NULL;
2441
2442         desc = priv->imsg_ring[mbox].imd_base;
2443         desc += priv->imsg_ring[mbox].desc_rdptr;
2444
2445         if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
2446                 goto out;
2447
2448         rx_slot = priv->imsg_ring[mbox].rx_slot;
2449         while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
2450                 if (++rx_slot == priv->imsg_ring[mbox].size)
2451                         rx_slot = 0;
2452         }
2453
2454         rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
2455                         le32_to_cpu(desc->bufptr_lo);
2456
2457         rx_virt = priv->imsg_ring[mbox].buf_base +
2458                   (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
2459
2460         buf = priv->imsg_ring[mbox].imq_base[rx_slot];
2461         msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
2462         if (msg_size == 0)
2463                 msg_size = RIO_MAX_MSG_SIZE;
2464
2465         memcpy(buf, rx_virt, msg_size);
2466         priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2467
2468         desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
2469         if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
2470                 priv->imsg_ring[mbox].desc_rdptr = 0;
2471
2472         iowrite32(priv->imsg_ring[mbox].desc_rdptr,
2473                 priv->regs + TSI721_IBDMAC_DQRP(ch));
2474
2475         /* Return free buffer into the pointer list */
2476         free_ptr = priv->imsg_ring[mbox].imfq_base;
2477         free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
2478
2479         if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
2480                 priv->imsg_ring[mbox].fq_wrptr = 0;
2481
2482         iowrite32(priv->imsg_ring[mbox].fq_wrptr,
2483                 priv->regs + TSI721_IBDMAC_FQWP(ch));
2484 out:
2485         return buf;
2486 }
2487
2488 /**
2489  * tsi721_messages_init - Initialization of Messaging Engine
2490  * @priv: pointer to tsi721 private data
2491  *
2492  * Configures Tsi721 messaging engine.
2493  */
2494 static int tsi721_messages_init(struct tsi721_device *priv)
2495 {
2496         int     ch;
2497
2498         iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2499         iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2500         iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2501
2502         /* Set SRIO Message Request/Response Timeout */
2503         iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2504
2505         /* Initialize Inbound Messaging Engine Registers */
2506         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2507                 /* Clear interrupt bits */
2508                 iowrite32(TSI721_IBDMAC_INT_MASK,
2509                         priv->regs + TSI721_IBDMAC_INT(ch));
2510                 /* Clear Status */
2511                 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2512
2513                 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2514                                 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2515                 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2516                                 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2517         }
2518
2519         return 0;
2520 }
2521
2522 /**
2523  * tsi721_query_mport - Fetch inbound message from the Tsi721 MSG Queue
2524  * @mport: Master port implementing the Inbound Messaging Engine
2525  * @mbox: Inbound mailbox number
2526  *
2527  * Returns pointer to the message on success or NULL on failure.
2528  */
2529 static int tsi721_query_mport(struct rio_mport *mport,
2530                               struct rio_mport_attr *attr)
2531 {
2532         struct tsi721_device *priv = mport->priv;
2533         u32 rval;
2534
2535         rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_ERR_STS_CSR(0)));
2536         if (rval & RIO_PORT_N_ERR_STS_PORT_OK) {
2537                 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_CTL2_CSR(0)));
2538                 attr->link_speed = (rval & RIO_PORT_N_CTL2_SEL_BAUD) >> 28;
2539                 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_CTL_CSR(0)));
2540                 attr->link_width = (rval & RIO_PORT_N_CTL_IPW) >> 27;
2541         } else
2542                 attr->link_speed = RIO_LINK_DOWN;
2543
2544 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2545         attr->flags = RIO_MPORT_DMA | RIO_MPORT_DMA_SG;
2546         attr->dma_max_sge = 0;
2547         attr->dma_max_size = TSI721_BDMA_MAX_BCOUNT;
2548         attr->dma_align = 0;
2549 #else
2550         attr->flags = 0;
2551 #endif
2552         return 0;
2553 }
2554
2555 /**
2556  * tsi721_disable_ints - disables all device interrupts
2557  * @priv: pointer to tsi721 private data
2558  */
2559 static void tsi721_disable_ints(struct tsi721_device *priv)
2560 {
2561         int ch;
2562
2563         /* Disable all device level interrupts */
2564         iowrite32(0, priv->regs + TSI721_DEV_INTE);
2565
2566         /* Disable all Device Channel interrupts */
2567         iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2568
2569         /* Disable all Inbound Msg Channel interrupts */
2570         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2571                 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2572
2573         /* Disable all Outbound Msg Channel interrupts */
2574         for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2575                 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2576
2577         /* Disable all general messaging interrupts */
2578         iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2579
2580         /* Disable all BDMA Channel interrupts */
2581         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
2582                 iowrite32(0,
2583                         priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
2584
2585         /* Disable all general BDMA interrupts */
2586         iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2587
2588         /* Disable all SRIO Channel interrupts */
2589         for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2590                 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2591
2592         /* Disable all general SR2PC interrupts */
2593         iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2594
2595         /* Disable all PC2SR interrupts */
2596         iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2597
2598         /* Disable all I2C interrupts */
2599         iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2600
2601         /* Disable SRIO MAC interrupts */
2602         iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2603         iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2604 }
2605
2606 static struct rio_ops tsi721_rio_ops = {
2607         .lcread                 = tsi721_lcread,
2608         .lcwrite                = tsi721_lcwrite,
2609         .cread                  = tsi721_cread_dma,
2610         .cwrite                 = tsi721_cwrite_dma,
2611         .dsend                  = tsi721_dsend,
2612         .open_inb_mbox          = tsi721_open_inb_mbox,
2613         .close_inb_mbox         = tsi721_close_inb_mbox,
2614         .open_outb_mbox         = tsi721_open_outb_mbox,
2615         .close_outb_mbox        = tsi721_close_outb_mbox,
2616         .add_outb_message       = tsi721_add_outb_message,
2617         .add_inb_buffer         = tsi721_add_inb_buffer,
2618         .get_inb_message        = tsi721_get_inb_message,
2619         .map_inb                = tsi721_rio_map_inb_mem,
2620         .unmap_inb              = tsi721_rio_unmap_inb_mem,
2621         .pwenable               = tsi721_pw_enable,
2622         .query_mport            = tsi721_query_mport,
2623         .map_outb               = tsi721_map_outb_win,
2624         .unmap_outb             = tsi721_unmap_outb_win,
2625 };
2626
2627 static void tsi721_mport_release(struct device *dev)
2628 {
2629         struct rio_mport *mport = to_rio_mport(dev);
2630
2631         tsi_debug(EXIT, dev, "%s id=%d", mport->name, mport->id);
2632 }
2633
2634 /**
2635  * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2636  * @priv: pointer to tsi721 private data
2637  *
2638  * Configures Tsi721 as RapidIO master port.
2639  */
2640 static int tsi721_setup_mport(struct tsi721_device *priv)
2641 {
2642         struct pci_dev *pdev = priv->pdev;
2643         int err = 0;
2644         struct rio_mport *mport = &priv->mport;
2645
2646         err = rio_mport_initialize(mport);
2647         if (err)
2648                 return err;
2649
2650         mport->ops = &tsi721_rio_ops;
2651         mport->index = 0;
2652         mport->sys_size = 0; /* small system */
2653         mport->phy_type = RIO_PHY_SERIAL;
2654         mport->priv = (void *)priv;
2655         mport->phys_efptr = 0x100;
2656         mport->dev.parent = &pdev->dev;
2657         mport->dev.release = tsi721_mport_release;
2658
2659         INIT_LIST_HEAD(&mport->dbells);
2660
2661         rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
2662         rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2663         rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
2664         snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)",
2665                  dev_driver_string(&pdev->dev), dev_name(&pdev->dev));
2666
2667         /* Hook up interrupt handler */
2668
2669 #ifdef CONFIG_PCI_MSI
2670         if (!tsi721_enable_msix(priv))
2671                 priv->flags |= TSI721_USING_MSIX;
2672         else if (!pci_enable_msi(pdev))
2673                 priv->flags |= TSI721_USING_MSI;
2674         else
2675                 tsi_debug(MPORT, &pdev->dev,
2676                          "MSI/MSI-X is not available. Using legacy INTx.");
2677 #endif /* CONFIG_PCI_MSI */
2678
2679         err = tsi721_request_irq(priv);
2680
2681         if (err) {
2682                 tsi_err(&pdev->dev, "Unable to get PCI IRQ %02X (err=0x%x)",
2683                         pdev->irq, err);
2684                 return err;
2685         }
2686
2687 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2688         err = tsi721_register_dma(priv);
2689         if (err)
2690                 goto err_exit;
2691 #endif
2692         /* Enable SRIO link */
2693         iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2694                   TSI721_DEVCTL_SRBOOT_CMPL,
2695                   priv->regs + TSI721_DEVCTL);
2696
2697         if (mport->host_deviceid >= 0)
2698                 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2699                           RIO_PORT_GEN_DISCOVERED,
2700                           priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2701         else
2702                 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2703
2704         err = rio_register_mport(mport);
2705         if (err) {
2706                 tsi721_unregister_dma(priv);
2707                 goto err_exit;
2708         }
2709
2710         return 0;
2711
2712 err_exit:
2713         tsi721_free_irq(priv);
2714         return err;
2715 }
2716
2717 static int tsi721_probe(struct pci_dev *pdev,
2718                                   const struct pci_device_id *id)
2719 {
2720         struct tsi721_device *priv;
2721         int err;
2722
2723         priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2724         if (!priv) {
2725                 err = -ENOMEM;
2726                 goto err_exit;
2727         }
2728
2729         err = pci_enable_device(pdev);
2730         if (err) {
2731                 tsi_err(&pdev->dev, "Failed to enable PCI device");
2732                 goto err_clean;
2733         }
2734
2735         priv->pdev = pdev;
2736
2737 #ifdef DEBUG
2738         {
2739                 int i;
2740
2741                 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2742                         tsi_debug(INIT, &pdev->dev, "res%d %pR",
2743                                   i, &pdev->resource[i]);
2744                 }
2745         }
2746 #endif
2747         /*
2748          * Verify BAR configuration
2749          */
2750
2751         /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2752         if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2753             pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2754             pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2755                 tsi_err(&pdev->dev, "Missing or misconfigured CSR BAR0");
2756                 err = -ENODEV;
2757                 goto err_disable_pdev;
2758         }
2759
2760         /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2761         if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2762             pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2763             pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2764                 tsi_err(&pdev->dev, "Missing or misconfigured Doorbell BAR1");
2765                 err = -ENODEV;
2766                 goto err_disable_pdev;
2767         }
2768
2769         /*
2770          * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2771          * space.
2772          * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2773          * It may be a good idea to keep them disabled using HW configuration
2774          * to save PCI memory space.
2775          */
2776
2777         priv->p2r_bar[0].size = priv->p2r_bar[1].size = 0;
2778
2779         if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64) {
2780                 if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_PREFETCH)
2781                         tsi_debug(INIT, &pdev->dev,
2782                                  "Prefetchable OBW BAR2 will not be used");
2783                 else {
2784                         priv->p2r_bar[0].base = pci_resource_start(pdev, BAR_2);
2785                         priv->p2r_bar[0].size = pci_resource_len(pdev, BAR_2);
2786                 }
2787         }
2788
2789         if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64) {
2790                 if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_PREFETCH)
2791                         tsi_debug(INIT, &pdev->dev,
2792                                  "Prefetchable OBW BAR4 will not be used");
2793                 else {
2794                         priv->p2r_bar[1].base = pci_resource_start(pdev, BAR_4);
2795                         priv->p2r_bar[1].size = pci_resource_len(pdev, BAR_4);
2796                 }
2797         }
2798
2799         err = pci_request_regions(pdev, DRV_NAME);
2800         if (err) {
2801                 tsi_err(&pdev->dev, "Unable to obtain PCI resources");
2802                 goto err_disable_pdev;
2803         }
2804
2805         pci_set_master(pdev);
2806
2807         priv->regs = pci_ioremap_bar(pdev, BAR_0);
2808         if (!priv->regs) {
2809                 tsi_err(&pdev->dev, "Unable to map device registers space");
2810                 err = -ENOMEM;
2811                 goto err_free_res;
2812         }
2813
2814         priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2815         if (!priv->odb_base) {
2816                 tsi_err(&pdev->dev, "Unable to map outbound doorbells space");
2817                 err = -ENOMEM;
2818                 goto err_unmap_bars;
2819         }
2820
2821         /* Configure DMA attributes. */
2822         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2823                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2824                 if (err) {
2825                         tsi_err(&pdev->dev, "Unable to set DMA mask");
2826                         goto err_unmap_bars;
2827                 }
2828
2829                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2830                         tsi_info(&pdev->dev, "Unable to set consistent DMA mask");
2831         } else {
2832                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2833                 if (err)
2834                         tsi_info(&pdev->dev, "Unable to set consistent DMA mask");
2835         }
2836
2837         BUG_ON(!pci_is_pcie(pdev));
2838
2839         /* Clear "no snoop" and "relaxed ordering" bits. */
2840         pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
2841                 PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN, 0);
2842
2843         /* Adjust PCIe completion timeout. */
2844         pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, 0xf, 0x2);
2845
2846         /*
2847          * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2848          */
2849         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2850         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2851                                                 TSI721_MSIXTBL_OFFSET);
2852         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2853                                                 TSI721_MSIXPBA_OFFSET);
2854         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2855         /* End of FIXUP */
2856
2857         tsi721_disable_ints(priv);
2858
2859         tsi721_init_pc2sr_mapping(priv);
2860         tsi721_init_sr2pc_mapping(priv);
2861
2862         if (tsi721_bdma_maint_init(priv)) {
2863                 tsi_err(&pdev->dev, "BDMA initialization failed");
2864                 err = -ENOMEM;
2865                 goto err_unmap_bars;
2866         }
2867
2868         err = tsi721_doorbell_init(priv);
2869         if (err)
2870                 goto err_free_bdma;
2871
2872         tsi721_port_write_init(priv);
2873
2874         err = tsi721_messages_init(priv);
2875         if (err)
2876                 goto err_free_consistent;
2877
2878         err = tsi721_setup_mport(priv);
2879         if (err)
2880                 goto err_free_consistent;
2881
2882         pci_set_drvdata(pdev, priv);
2883         tsi721_interrupts_init(priv);
2884
2885         return 0;
2886
2887 err_free_consistent:
2888         tsi721_port_write_free(priv);
2889         tsi721_doorbell_free(priv);
2890 err_free_bdma:
2891         tsi721_bdma_maint_free(priv);
2892 err_unmap_bars:
2893         if (priv->regs)
2894                 iounmap(priv->regs);
2895         if (priv->odb_base)
2896                 iounmap(priv->odb_base);
2897 err_free_res:
2898         pci_release_regions(pdev);
2899         pci_clear_master(pdev);
2900 err_disable_pdev:
2901         pci_disable_device(pdev);
2902 err_clean:
2903         kfree(priv);
2904 err_exit:
2905         return err;
2906 }
2907
2908 static void tsi721_remove(struct pci_dev *pdev)
2909 {
2910         struct tsi721_device *priv = pci_get_drvdata(pdev);
2911
2912         tsi_debug(EXIT, &pdev->dev, "enter");
2913
2914         tsi721_disable_ints(priv);
2915         tsi721_free_irq(priv);
2916         flush_scheduled_work();
2917         rio_unregister_mport(&priv->mport);
2918
2919         tsi721_unregister_dma(priv);
2920         tsi721_bdma_maint_free(priv);
2921         tsi721_doorbell_free(priv);
2922         tsi721_port_write_free(priv);
2923         tsi721_close_sr2pc_mapping(priv);
2924
2925         if (priv->regs)
2926                 iounmap(priv->regs);
2927         if (priv->odb_base)
2928                 iounmap(priv->odb_base);
2929 #ifdef CONFIG_PCI_MSI
2930         if (priv->flags & TSI721_USING_MSIX)
2931                 pci_disable_msix(priv->pdev);
2932         else if (priv->flags & TSI721_USING_MSI)
2933                 pci_disable_msi(priv->pdev);
2934 #endif
2935         pci_release_regions(pdev);
2936         pci_clear_master(pdev);
2937         pci_disable_device(pdev);
2938         pci_set_drvdata(pdev, NULL);
2939         kfree(priv);
2940         tsi_debug(EXIT, &pdev->dev, "exit");
2941 }
2942
2943 static void tsi721_shutdown(struct pci_dev *pdev)
2944 {
2945         struct tsi721_device *priv = pci_get_drvdata(pdev);
2946
2947         tsi_debug(EXIT, &pdev->dev, "enter");
2948
2949         tsi721_disable_ints(priv);
2950         tsi721_dma_stop_all(priv);
2951         pci_clear_master(pdev);
2952         pci_disable_device(pdev);
2953 }
2954
2955 static const struct pci_device_id tsi721_pci_tbl[] = {
2956         { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2957         { 0, }  /* terminate list */
2958 };
2959
2960 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2961
2962 static struct pci_driver tsi721_driver = {
2963         .name           = "tsi721",
2964         .id_table       = tsi721_pci_tbl,
2965         .probe          = tsi721_probe,
2966         .remove         = tsi721_remove,
2967         .shutdown       = tsi721_shutdown,
2968 };
2969
2970 module_pci_driver(tsi721_driver);
2971
2972 MODULE_DESCRIPTION("IDT Tsi721 PCIExpress-to-SRIO bridge driver");
2973 MODULE_AUTHOR("Integrated Device Technology, Inc.");
2974 MODULE_LICENSE("GPL");