drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/nand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_RANDOM_EN           BIT(9)
 149 #define NFC_RANDOM_DIRECTION    BIT(10)
 150 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 151 #define NFC_ECC_MODE(x)         ((x) << 12)
 152 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 153 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 154
 155 /* define bit use in NFC_ECC_ST */
 156 #define NFC_ECC_ERR(x)          BIT(x)
 157 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 158 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 159 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 160
 161 #define NFC_DEFAULT_TIMEOUT_MS  1000
 162
 163 #define NFC_SRAM_SIZE           1024
 164
 165 #define NFC_MAX_CS              7
 166
 167 /*
 168  * Ready/Busy detection type: describes the Ready/Busy detection modes
 169  *
 170  * @RB_NONE:    no external detection available, rely on STATUS command
 171  *              and software timeouts
 172  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 173  *              pin of the NAND flash chip must be connected to one of the
 174  *              native NAND R/B pins (those which can be muxed to the NAND
 175  *              Controller)
 176  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 177  *              pin of the NAND flash chip must be connected to a GPIO capable
 178  *              pin.
 179  */
 180 enum sunxi_nand_rb_type {
 181         RB_NONE,
 182         RB_NATIVE,
 183         RB_GPIO,
 184 };
 185
 186 /*
 187  * Ready/Busy structure: stores information related to Ready/Busy detection
 188  *
 189  * @type:       the Ready/Busy detection mode
 190  * @info:       information related to the R/B detection mode. Either a gpio
 191  *              id or a native R/B id (those supported by the NAND controller).
 192  */
 193 struct sunxi_nand_rb {
 194         enum sunxi_nand_rb_type type;
 195         union {
 196                 int gpio;
 197                 int nativeid;
 198         } info;
 199 };
 200
 201 /*
 202  * Chip Select structure: stores information related to NAND Chip Select
 203  *
 204  * @cs:         the NAND CS id used to communicate with a NAND Chip
 205  * @rb:         the Ready/Busy description
 206  */
 207 struct sunxi_nand_chip_sel {
 208         u8 cs;
 209         struct sunxi_nand_rb rb;
 210 };
 211
 212 /*
 213  * sunxi HW ECC infos: stores information related to HW ECC support
 214  *
 215  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 216  */
 217 struct sunxi_nand_hw_ecc {
 218         int mode;
 219 };
 220
 221 /*
 222  * NAND chip structure: stores NAND chip device related information
 223  *
 224  * @node:               used to store NAND chips into a list
 225  * @nand:               base NAND chip structure
 226  * @mtd:                base MTD structure
 227  * @clk_rate:           clk_rate required for this NAND chip
 228  * @timing_cfg          TIMING_CFG register value for this NAND chip
 229  * @selected:           current active CS
 230  * @nsels:              number of CS lines required by the NAND chip
 231  * @sels:               array of CS lines descriptions
 232  */
 233 struct sunxi_nand_chip {
 234         struct list_head node;
 235         struct nand_chip nand;
 236         unsigned long clk_rate;
 237         u32 timing_cfg;
 238         u32 timing_ctl;
 239         int selected;
 240         int addr_cycles;
 241         u32 addr[2];
 242         int cmd_cycles;
 243         u8 cmd[2];
 244         int nsels;
 245         struct sunxi_nand_chip_sel sels[0];
 246 };
 247
 248 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 249 {
 250         return container_of(nand, struct sunxi_nand_chip, nand);
 251 }
 252
 253 /*
 254  * NAND Controller structure: stores sunxi NAND controller information
 255  *
 256  * @controller:         base controller structure
 257  * @dev:                parent device (used to print error messages)
 258  * @regs:               NAND controller registers
 259  * @ahb_clk:            NAND Controller AHB clock
 260  * @mod_clk:            NAND Controller mod clock
 261  * @assigned_cs:        bitmask describing already assigned CS lines
 262  * @clk_rate:           NAND controller current clock rate
 263  * @chips:              a list containing all the NAND chips attached to
 264  *                      this NAND controller
 265  * @complete:           a completion object used to wait for NAND
 266  *                      controller events
 267  */
 268 struct sunxi_nfc {
 269         struct nand_hw_control controller;
 270         struct device *dev;
 271         void __iomem *regs;
 272         struct clk *ahb_clk;
 273         struct clk *mod_clk;
 274         struct reset_control *reset;
 275         unsigned long assigned_cs;
 276         unsigned long clk_rate;
 277         struct list_head chips;
 278         struct completion complete;
 279         struct dma_chan *dmac;
 280 };
 281
 282 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 283 {
 284         return container_of(ctrl, struct sunxi_nfc, controller);
 285 }
 286
 287 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 288 {
 289         struct sunxi_nfc *nfc = dev_id;
 290         u32 st = readl(nfc->regs + NFC_REG_ST);
 291         u32 ien = readl(nfc->regs + NFC_REG_INT);
 292
 293         if (!(ien & st))
 294                 return IRQ_NONE;
 295
 296         if ((ien & st) == ien)
 297                 complete(&nfc->complete);
 298
 299         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 300         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 301
 302         return IRQ_HANDLED;
 303 }
 304
 305 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 306                                  bool use_polling, unsigned int timeout_ms)
 307 {
 308         int ret;
 309
 310         if (events & ~NFC_INT_MASK)
 311                 return -EINVAL;
 312
 313         if (!timeout_ms)
 314                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 315
 316         if (!use_polling) {
 317                 init_completion(&nfc->complete);
 318
 319                 writel(events, nfc->regs + NFC_REG_INT);
 320
 321                 ret = wait_for_completion_timeout(&nfc->complete,
 322                                                 msecs_to_jiffies(timeout_ms));
 323
 324                 writel(0, nfc->regs + NFC_REG_INT);
 325         } else {
 326                 u32 status;
 327
 328                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 329                                          (status & events) == events, 1,
 330                                          timeout_ms * 1000);
 331         }
 332
 333         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 334
 335         if (ret)
 336                 dev_err(nfc->dev, "wait interrupt timedout\n");
 337
 338         return ret;
 339 }
 340
 341 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 342 {
 343         u32 status;
 344         int ret;
 345
 346         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 347                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 348                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 349         if (ret)
 350                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 351
 352         return ret;
 353 }
 354
 355 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 356 {
 357         u32 ctl;
 358         int ret;
 359
 360         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 361         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 362
 363         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 364                                  !(ctl & NFC_RESET), 1,
 365                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 366         if (ret)
 367                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 368
 369         return ret;
 370 }
 371
 372 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 373                                     int chunksize, int nchunks,
 374                                     enum dma_data_direction ddir,
 375                                     struct scatterlist *sg)
 376 {
 377         struct nand_chip *nand = mtd_to_nand(mtd);
 378         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 379         struct dma_async_tx_descriptor *dmad;
 380         enum dma_transfer_direction tdir;
 381         dma_cookie_t dmat;
 382         int ret;
 383
 384         if (ddir == DMA_FROM_DEVICE)
 385                 tdir = DMA_DEV_TO_MEM;
 386         else
 387                 tdir = DMA_MEM_TO_DEV;
 388
 389         sg_init_one(sg, buf, nchunks * chunksize);
 390         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 391         if (!ret)
 392                 return -ENOMEM;
 393
 394         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 395         if (!dmad) {
 396                 ret = -EINVAL;
 397                 goto err_unmap_buf;
 398         }
 399
 400         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 401                nfc->regs + NFC_REG_CTL);
 402         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 403         writel(chunksize, nfc->regs + NFC_REG_CNT);
 404         dmat = dmaengine_submit(dmad);
 405
 406         ret = dma_submit_error(dmat);
 407         if (ret)
 408                 goto err_clr_dma_flag;
 409
 410         return 0;
 411
 412 err_clr_dma_flag:
 413         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 414                nfc->regs + NFC_REG_CTL);
 415
 416 err_unmap_buf:
 417         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 418         return ret;
 419 }
 420
 421 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 422                                      enum dma_data_direction ddir,
 423                                      struct scatterlist *sg)
 424 {
 425         struct nand_chip *nand = mtd_to_nand(mtd);
 426         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 427
 428         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 429         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 430                nfc->regs + NFC_REG_CTL);
 431 }
 432
 433 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 434 {
 435         struct nand_chip *nand = mtd_to_nand(mtd);
 436         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 437         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 438         struct sunxi_nand_rb *rb;
 439         int ret;
 440
 441         if (sunxi_nand->selected < 0)
 442                 return 0;
 443
 444         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 445
 446         switch (rb->type) {
 447         case RB_NATIVE:
 448                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 449                          NFC_RB_STATE(rb->info.nativeid));
 450                 break;
 451         case RB_GPIO:
 452                 ret = gpio_get_value(rb->info.gpio);
 453                 break;
 454         case RB_NONE:
 455         default:
 456                 ret = 0;
 457                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 458                 break;
 459         }
 460
 461         return ret;
 462 }
 463
 464 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 465 {
 466         struct nand_chip *nand = mtd_to_nand(mtd);
 467         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 468         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 469         struct sunxi_nand_chip_sel *sel;
 470         u32 ctl;
 471
 472         if (chip > 0 && chip >= sunxi_nand->nsels)
 473                 return;
 474
 475         if (chip == sunxi_nand->selected)
 476                 return;
 477
 478         ctl = readl(nfc->regs + NFC_REG_CTL) &
 479               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 480
 481         if (chip >= 0) {
 482                 sel = &sunxi_nand->sels[chip];
 483
 484                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 485                        NFC_PAGE_SHIFT(nand->page_shift);
 486                 if (sel->rb.type == RB_NONE) {
 487                         nand->dev_ready = NULL;
 488                 } else {
 489                         nand->dev_ready = sunxi_nfc_dev_ready;
 490                         if (sel->rb.type == RB_NATIVE)
 491                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 492                 }
 493
 494                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 495
 496                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 497                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 498                         nfc->clk_rate = sunxi_nand->clk_rate;
 499                 }
 500         }
 501
 502         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 503         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 504         writel(ctl, nfc->regs + NFC_REG_CTL);
 505
 506         sunxi_nand->selected = chip;
 507 }
 508
 509 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 510 {
 511         struct nand_chip *nand = mtd_to_nand(mtd);
 512         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 513         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 514         int ret;
 515         int cnt;
 516         int offs = 0;
 517         u32 tmp;
 518
 519         while (len > offs) {
 520                 cnt = min(len - offs, NFC_SRAM_SIZE);
 521
 522                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 523                 if (ret)
 524                         break;
 525
 526                 writel(cnt, nfc->regs + NFC_REG_CNT);
 527                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 528                 writel(tmp, nfc->regs + NFC_REG_CMD);
 529
 530                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 531                 if (ret)
 532                         break;
 533
 534                 if (buf)
 535                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 536                                       cnt);
 537                 offs += cnt;
 538         }
 539 }
 540
 541 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 542                                 int len)
 543 {
 544         struct nand_chip *nand = mtd_to_nand(mtd);
 545         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 546         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 547         int ret;
 548         int cnt;
 549         int offs = 0;
 550         u32 tmp;
 551
 552         while (len > offs) {
 553                 cnt = min(len - offs, NFC_SRAM_SIZE);
 554
 555                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 556                 if (ret)
 557                         break;
 558
 559                 writel(cnt, nfc->regs + NFC_REG_CNT);
 560                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 561                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 562                       NFC_ACCESS_DIR;
 563                 writel(tmp, nfc->regs + NFC_REG_CMD);
 564
 565                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 566                 if (ret)
 567                         break;
 568
 569                 offs += cnt;
 570         }
 571 }
 572
 573 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 574 {
 575         uint8_t ret;
 576
 577         sunxi_nfc_read_buf(mtd, &ret, 1);
 578
 579         return ret;
 580 }
 581
 582 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 583                                unsigned int ctrl)
 584 {
 585         struct nand_chip *nand = mtd_to_nand(mtd);
 586         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 587         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 588         int ret;
 589
 590         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 591         if (ret)
 592                 return;
 593
 594         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 595             !(ctrl & (NAND_CLE | NAND_ALE))) {
 596                 u32 cmd = 0;
 597
 598                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 599                         return;
 600
 601                 if (sunxi_nand->cmd_cycles--)
 602                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 603
 604                 if (sunxi_nand->cmd_cycles--) {
 605                         cmd |= NFC_SEND_CMD2;
 606                         writel(sunxi_nand->cmd[1],
 607                                nfc->regs + NFC_REG_RCMD_SET);
 608                 }
 609
 610                 sunxi_nand->cmd_cycles = 0;
 611
 612                 if (sunxi_nand->addr_cycles) {
 613                         cmd |= NFC_SEND_ADR |
 614                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 615                         writel(sunxi_nand->addr[0],
 616                                nfc->regs + NFC_REG_ADDR_LOW);
 617                 }
 618
 619                 if (sunxi_nand->addr_cycles > 4)
 620                         writel(sunxi_nand->addr[1],
 621                                nfc->regs + NFC_REG_ADDR_HIGH);
 622
 623                 writel(cmd, nfc->regs + NFC_REG_CMD);
 624                 sunxi_nand->addr[0] = 0;
 625                 sunxi_nand->addr[1] = 0;
 626                 sunxi_nand->addr_cycles = 0;
 627                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 628         }
 629
 630         if (ctrl & NAND_CLE) {
 631                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 632         } else if (ctrl & NAND_ALE) {
 633                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 634                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 635                 sunxi_nand->addr_cycles++;
 636         }
 637 }
 638
 639 /* These seed values have been extracted from Allwinner's BSP */
 640 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 641         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 642         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 643         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 644         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 645         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 646         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 647         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 648         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 649         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 650         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 651         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 652         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 653         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 654         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 655         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 656         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 657 };
 658
 659 /*
 660  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 661  * have been generated using
 662  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 663  * the randomizer engine does internally before de/scrambling OOB data.
 664  *
 665  * Those tables are statically defined to avoid calculating randomizer state
 666  * at runtime.
 667  */
 668 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 669         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 670         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 671         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 672         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 673         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 674         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 675         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 676         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 677         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 678         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 679         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 680         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 681         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 682         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 683         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 684         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 685 };
 686
 687 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 688         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 689         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 690         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 691         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 692         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 693         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 694         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 695         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 696         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 697         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 698         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 699         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 700         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 701         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 702         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 703         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 704 };
 705
 706 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 707 {
 708         state &= 0x7fff;
 709
 710         /*
 711          * This loop is just a simple implementation of a Fibonacci LFSR using
 712          * the x16 + x15 + 1 polynomial.
 713          */
 714         while (count--)
 715                 state = ((state >> 1) |
 716                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 717
 718         return state;
 719 }
 720
 721 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 722 {
 723         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 724         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 725
 726         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 727                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 728
 729         if (ecc) {
 730                 if (mtd->ecc_step_size == 512)
 731                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 732                 else
 733                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 734         }
 735
 736         return seeds[page % mod];
 737 }
 738
 739 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 740                                         int page, bool ecc)
 741 {
 742         struct nand_chip *nand = mtd_to_nand(mtd);
 743         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 744         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 745         u16 state;
 746
 747         if (!(nand->options & NAND_NEED_SCRAMBLING))
 748                 return;
 749
 750         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 751         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 752         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 753         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 754 }
 755
 756 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 757 {
 758         struct nand_chip *nand = mtd_to_nand(mtd);
 759         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 760
 761         if (!(nand->options & NAND_NEED_SCRAMBLING))
 762                 return;
 763
 764         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 765                nfc->regs + NFC_REG_ECC_CTL);
 766 }
 767
 768 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 769 {
 770         struct nand_chip *nand = mtd_to_nand(mtd);
 771         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 772
 773         if (!(nand->options & NAND_NEED_SCRAMBLING))
 774                 return;
 775
 776         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 777                nfc->regs + NFC_REG_ECC_CTL);
 778 }
 779
 780 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 781 {
 782         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 783
 784         bbm[0] ^= state;
 785         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 786 }
 787
 788 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 789                                            const uint8_t *buf, int len,
 790                                            bool ecc, int page)
 791 {
 792         sunxi_nfc_randomizer_config(mtd, page, ecc);
 793         sunxi_nfc_randomizer_enable(mtd);
 794         sunxi_nfc_write_buf(mtd, buf, len);
 795         sunxi_nfc_randomizer_disable(mtd);
 796 }
 797
 798 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 799                                           int len, bool ecc, int page)
 800 {
 801         sunxi_nfc_randomizer_config(mtd, page, ecc);
 802         sunxi_nfc_randomizer_enable(mtd);
 803         sunxi_nfc_read_buf(mtd, buf, len);
 804         sunxi_nfc_randomizer_disable(mtd);
 805 }
 806
 807 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 808 {
 809         struct nand_chip *nand = mtd_to_nand(mtd);
 810         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 811         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 812         u32 ecc_ctl;
 813
 814         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 815         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 816                      NFC_ECC_BLOCK_SIZE_MSK);
 817         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 818                    NFC_ECC_PIPELINE;
 819
 820         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 821 }
 822
 823 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 824 {
 825         struct nand_chip *nand = mtd_to_nand(mtd);
 826         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 827
 828         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 829                nfc->regs + NFC_REG_ECC_CTL);
 830 }
 831
 832 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 833 {
 834         buf[0] = user_data;
 835         buf[1] = user_data >> 8;
 836         buf[2] = user_data >> 16;
 837         buf[3] = user_data >> 24;
 838 }
 839
 840 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 841 {
 842         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 843 }
 844
 845 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 846                                                 int step, bool bbm, int page)
 847 {
 848         struct nand_chip *nand = mtd_to_nand(mtd);
 849         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 850
 851         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 852                                    oob);
 853
 854         /* De-randomize the Bad Block Marker. */
 855         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 856                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 857 }
 858
 859 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 860                                                 const u8 *oob, int step,
 861                                                 bool bbm, int page)
 862 {
 863         struct nand_chip *nand = mtd_to_nand(mtd);
 864         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 865         u8 user_data[4];
 866
 867         /* Randomize the Bad Block Marker. */
 868         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 869                 memcpy(user_data, oob, sizeof(user_data));
 870                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 871                 oob = user_data;
 872         }
 873
 874         writel(sunxi_nfc_buf_to_user_data(oob),
 875                nfc->regs + NFC_REG_USER_DATA(step));
 876 }
 877
 878 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 879                                           unsigned int *max_bitflips, int ret)
 880 {
 881         if (ret < 0) {
 882                 mtd->ecc_stats.failed++;
 883         } else {
 884                 mtd->ecc_stats.corrected += ret;
 885                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 886         }
 887 }
 888
 889 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 890                                     int step, u32 status, bool *erased)
 891 {
 892         struct nand_chip *nand = mtd_to_nand(mtd);
 893         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 894         struct nand_ecc_ctrl *ecc = &nand->ecc;
 895         u32 tmp;
 896
 897         *erased = false;
 898
 899         if (status & NFC_ECC_ERR(step))
 900                 return -EBADMSG;
 901
 902         if (status & NFC_ECC_PAT_FOUND(step)) {
 903                 u8 pattern;
 904
 905                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 906                         pattern = 0x0;
 907                 } else {
 908                         pattern = 0xff;
 909                         *erased = true;
 910                 }
 911
 912                 if (data)
 913                         memset(data, pattern, ecc->size);
 914
 915                 if (oob)
 916                         memset(oob, pattern, ecc->bytes + 4);
 917
 918                 return 0;
 919         }
 920
 921         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 922
 923         return NFC_ECC_ERR_CNT(step, tmp);
 924 }
 925
 926 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 927                                        u8 *data, int data_off,
 928                                        u8 *oob, int oob_off,
 929                                        int *cur_off,
 930                                        unsigned int *max_bitflips,
 931                                        bool bbm, bool oob_required, int page)
 932 {
 933         struct nand_chip *nand = mtd_to_nand(mtd);
 934         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 935         struct nand_ecc_ctrl *ecc = &nand->ecc;
 936         int raw_mode = 0;
 937         bool erased;
 938         int ret;
 939
 940         if (*cur_off != data_off)
 941                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 942
 943         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 944
 945         if (data_off + ecc->size != oob_off)
 946                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 947
 948         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 949         if (ret)
 950                 return ret;
 951
 952         sunxi_nfc_randomizer_enable(mtd);
 953         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 954                nfc->regs + NFC_REG_CMD);
 955
 956         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 957         sunxi_nfc_randomizer_disable(mtd);
 958         if (ret)
 959                 return ret;
 960
 961         *cur_off = oob_off + ecc->bytes + 4;
 962
 963         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 964                                        readl(nfc->regs + NFC_REG_ECC_ST),
 965                                        &erased);
 966         if (erased)
 967                 return 1;
 968
 969         if (ret < 0) {
 970                 /*
 971                  * Re-read the data with the randomizer disabled to identify
 972                  * bitflips in erased pages.
 973                  */
 974                 if (nand->options & NAND_NEED_SCRAMBLING) {
 975                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 976                         nand->read_buf(mtd, data, ecc->size);
 977                 } else {
 978                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 979                                       ecc->size);
 980                 }
 981
 982                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 983                 nand->read_buf(mtd, oob, ecc->bytes + 4);
 984
 985                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 986                                                   oob, ecc->bytes + 4,
 987                                                   NULL, 0, ecc->strength);
 988                 if (ret >= 0)
 989                         raw_mode = 1;
 990         } else {
 991                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 992
 993                 if (oob_required) {
 994                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 995                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
 996                                                       true, page);
 997
 998                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
 999                                                             bbm, page);
1000                 }
1001         }
1002
1003         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1004
1005         return raw_mode;
1006 }
1007
1008 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1009                                             u8 *oob, int *cur_off,
1010                                             bool randomize, int page)
1011 {
1012         struct nand_chip *nand = mtd_to_nand(mtd);
1013         struct nand_ecc_ctrl *ecc = &nand->ecc;
1014         int offset = ((ecc->bytes + 4) * ecc->steps);
1015         int len = mtd->oobsize - offset;
1016
1017         if (len <= 0)
1018                 return;
1019
1020         if (!cur_off || *cur_off != offset)
1021                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1022                               offset + mtd->writesize, -1);
1023
1024         if (!randomize)
1025                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1026         else
1027                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1028                                               false, page);
1029
1030         if (cur_off)
1031                 *cur_off = mtd->oobsize + mtd->writesize;
1032 }
1033
1034 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1035                                             int oob_required, int page,
1036                                             int nchunks)
1037 {
1038         struct nand_chip *nand = mtd_to_nand(mtd);
1039         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1040         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1041         struct nand_ecc_ctrl *ecc = &nand->ecc;
1042         unsigned int max_bitflips = 0;
1043         int ret, i, raw_mode = 0;
1044         struct scatterlist sg;
1045         u32 status;
1046
1047         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1048         if (ret)
1049                 return ret;
1050
1051         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1052                                        DMA_FROM_DEVICE, &sg);
1053         if (ret)
1054                 return ret;
1055
1056         sunxi_nfc_hw_ecc_enable(mtd);
1057         sunxi_nfc_randomizer_config(mtd, page, false);
1058         sunxi_nfc_randomizer_enable(mtd);
1059
1060         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1061                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1062
1063         dma_async_issue_pending(nfc->dmac);
1064
1065         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1066                nfc->regs + NFC_REG_CMD);
1067
1068         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1069         if (ret)
1070                 dmaengine_terminate_all(nfc->dmac);
1071
1072         sunxi_nfc_randomizer_disable(mtd);
1073         sunxi_nfc_hw_ecc_disable(mtd);
1074
1075         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1076
1077         if (ret)
1078                 return ret;
1079
1080         status = readl(nfc->regs + NFC_REG_ECC_ST);
1081
1082         for (i = 0; i < nchunks; i++) {
1083                 int data_off = i * ecc->size;
1084                 int oob_off = i * (ecc->bytes + 4);
1085                 u8 *data = buf + data_off;
1086                 u8 *oob = nand->oob_poi + oob_off;
1087                 bool erased;
1088
1089                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1090                                                oob_required ? oob : NULL,
1091                                                i, status, &erased);
1092
1093                 /* ECC errors are handled in the second loop. */
1094                 if (ret < 0)
1095                         continue;
1096
1097                 if (oob_required && !erased) {
1098                         /* TODO: use DMA to retrieve OOB */
1099                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1100                                       mtd->writesize + oob_off, -1);
1101                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1102
1103                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1104                                                             !i, page);
1105                 }
1106
1107                 if (erased)
1108                         raw_mode = 1;
1109
1110                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1111         }
1112
1113         if (status & NFC_ECC_ERR_MSK) {
1114                 for (i = 0; i < nchunks; i++) {
1115                         int data_off = i * ecc->size;
1116                         int oob_off = i * (ecc->bytes + 4);
1117                         u8 *data = buf + data_off;
1118                         u8 *oob = nand->oob_poi + oob_off;
1119
1120                         if (!(status & NFC_ECC_ERR(i)))
1121                                 continue;
1122
1123                         /*
1124                          * Re-read the data with the randomizer disabled to
1125                          * identify bitflips in erased pages.
1126                          */
1127                         if (randomized) {
1128                                 /* TODO: use DMA to read page in raw mode */
1129                                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1130                                               data_off, -1);
1131                                 nand->read_buf(mtd, data, ecc->size);
1132                         }
1133
1134                         /* TODO: use DMA to retrieve OOB */
1135                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1136                                       mtd->writesize + oob_off, -1);
1137                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1138
1139                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1140                                                           oob, ecc->bytes + 4,
1141                                                           NULL, 0,
1142                                                           ecc->strength);
1143                         if (ret >= 0)
1144                                 raw_mode = 1;
1145
1146                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1147                 }
1148         }
1149
1150         if (oob_required)
1151                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1152                                                 NULL, !raw_mode,
1153                                                 page);
1154
1155         return max_bitflips;
1156 }
1157
1158 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1159                                         const u8 *data, int data_off,
1160                                         const u8 *oob, int oob_off,
1161                                         int *cur_off, bool bbm,
1162                                         int page)
1163 {
1164         struct nand_chip *nand = mtd_to_nand(mtd);
1165         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1166         struct nand_ecc_ctrl *ecc = &nand->ecc;
1167         int ret;
1168
1169         if (data_off != *cur_off)
1170                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1171
1172         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1173
1174         if (data_off + ecc->size != oob_off)
1175                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1176
1177         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1178         if (ret)
1179                 return ret;
1180
1181         sunxi_nfc_randomizer_enable(mtd);
1182         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1183
1184         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1185                NFC_ACCESS_DIR | NFC_ECC_OP,
1186                nfc->regs + NFC_REG_CMD);
1187
1188         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1189         sunxi_nfc_randomizer_disable(mtd);
1190         if (ret)
1191                 return ret;
1192
1193         *cur_off = oob_off + ecc->bytes + 4;
1194
1195         return 0;
1196 }
1197
1198 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1199                                              u8 *oob, int *cur_off,
1200                                              int page)
1201 {
1202         struct nand_chip *nand = mtd_to_nand(mtd);
1203         struct nand_ecc_ctrl *ecc = &nand->ecc;
1204         int offset = ((ecc->bytes + 4) * ecc->steps);
1205         int len = mtd->oobsize - offset;
1206
1207         if (len <= 0)
1208                 return;
1209
1210         if (!cur_off || *cur_off != offset)
1211                 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1212                               offset + mtd->writesize, -1);
1213
1214         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1215
1216         if (cur_off)
1217                 *cur_off = mtd->oobsize + mtd->writesize;
1218 }
1219
1220 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1221                                       struct nand_chip *chip, uint8_t *buf,
1222                                       int oob_required, int page)
1223 {
1224         struct nand_ecc_ctrl *ecc = &chip->ecc;
1225         unsigned int max_bitflips = 0;
1226         int ret, i, cur_off = 0;
1227         bool raw_mode = false;
1228
1229         sunxi_nfc_hw_ecc_enable(mtd);
1230
1231         for (i = 0; i < ecc->steps; i++) {
1232                 int data_off = i * ecc->size;
1233                 int oob_off = i * (ecc->bytes + 4);
1234                 u8 *data = buf + data_off;
1235                 u8 *oob = chip->oob_poi + oob_off;
1236
1237                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1238                                                   oob_off + mtd->writesize,
1239                                                   &cur_off, &max_bitflips,
1240                                                   !i, oob_required, page);
1241                 if (ret < 0)
1242                         return ret;
1243                 else if (ret)
1244                         raw_mode = true;
1245         }
1246
1247         if (oob_required)
1248                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1249                                                 !raw_mode, page);
1250
1251         sunxi_nfc_hw_ecc_disable(mtd);
1252
1253         return max_bitflips;
1254 }
1255
1256 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1257                                           struct nand_chip *chip, u8 *buf,
1258                                           int oob_required, int page)
1259 {
1260         int ret;
1261
1262         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1263                                                chip->ecc.steps);
1264         if (ret >= 0)
1265                 return ret;
1266
1267         /* Fallback to PIO mode */
1268         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1269
1270         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1271 }
1272
1273 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1274                                          struct nand_chip *chip,
1275                                          u32 data_offs, u32 readlen,
1276                                          u8 *bufpoi, int page)
1277 {
1278         struct nand_ecc_ctrl *ecc = &chip->ecc;
1279         int ret, i, cur_off = 0;
1280         unsigned int max_bitflips = 0;
1281
1282         sunxi_nfc_hw_ecc_enable(mtd);
1283
1284         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1285         for (i = data_offs / ecc->size;
1286              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1287                 int data_off = i * ecc->size;
1288                 int oob_off = i * (ecc->bytes + 4);
1289                 u8 *data = bufpoi + data_off;
1290                 u8 *oob = chip->oob_poi + oob_off;
1291
1292                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1293                                                   oob,
1294                                                   oob_off + mtd->writesize,
1295                                                   &cur_off, &max_bitflips, !i,
1296                                                   false, page);
1297                 if (ret < 0)
1298                         return ret;
1299         }
1300
1301         sunxi_nfc_hw_ecc_disable(mtd);
1302
1303         return max_bitflips;
1304 }
1305
1306 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1307                                              struct nand_chip *chip,
1308                                              u32 data_offs, u32 readlen,
1309                                              u8 *buf, int page)
1310 {
1311         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1312         int ret;
1313
1314         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1315         if (ret >= 0)
1316                 return ret;
1317
1318         /* Fallback to PIO mode */
1319         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1320
1321         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1322                                              buf, page);
1323 }
1324
1325 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1326                                        struct nand_chip *chip,
1327                                        const uint8_t *buf, int oob_required,
1328                                        int page)
1329 {
1330         struct nand_ecc_ctrl *ecc = &chip->ecc;
1331         int ret, i, cur_off = 0;
1332
1333         sunxi_nfc_hw_ecc_enable(mtd);
1334
1335         for (i = 0; i < ecc->steps; i++) {
1336                 int data_off = i * ecc->size;
1337                 int oob_off = i * (ecc->bytes + 4);
1338                 const u8 *data = buf + data_off;
1339                 const u8 *oob = chip->oob_poi + oob_off;
1340
1341                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1342                                                    oob_off + mtd->writesize,
1343                                                    &cur_off, !i, page);
1344                 if (ret)
1345                         return ret;
1346         }
1347
1348         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1349                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1350                                                  &cur_off, page);
1351
1352         sunxi_nfc_hw_ecc_disable(mtd);
1353
1354         return 0;
1355 }
1356
1357 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1358                                           struct nand_chip *chip,
1359                                           u32 data_offs, u32 data_len,
1360                                           const u8 *buf, int oob_required,
1361                                           int page)
1362 {
1363         struct nand_ecc_ctrl *ecc = &chip->ecc;
1364         int ret, i, cur_off = 0;
1365
1366         sunxi_nfc_hw_ecc_enable(mtd);
1367
1368         for (i = data_offs / ecc->size;
1369              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1370                 int data_off = i * ecc->size;
1371                 int oob_off = i * (ecc->bytes + 4);
1372                 const u8 *data = buf + data_off;
1373                 const u8 *oob = chip->oob_poi + oob_off;
1374
1375                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1376                                                    oob_off + mtd->writesize,
1377                                                    &cur_off, !i, page);
1378                 if (ret)
1379                         return ret;
1380         }
1381
1382         sunxi_nfc_hw_ecc_disable(mtd);
1383
1384         return 0;
1385 }
1386
1387 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1388                                            struct nand_chip *chip,
1389                                            const u8 *buf,
1390                                            int oob_required,
1391                                            int page)
1392 {
1393         struct nand_chip *nand = mtd_to_nand(mtd);
1394         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1395         struct nand_ecc_ctrl *ecc = &nand->ecc;
1396         struct scatterlist sg;
1397         int ret, i;
1398
1399         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1400         if (ret)
1401                 return ret;
1402
1403         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1404                                        DMA_TO_DEVICE, &sg);
1405         if (ret)
1406                 goto pio_fallback;
1407
1408         for (i = 0; i < ecc->steps; i++) {
1409                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1410
1411                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1412         }
1413
1414         sunxi_nfc_hw_ecc_enable(mtd);
1415         sunxi_nfc_randomizer_config(mtd, page, false);
1416         sunxi_nfc_randomizer_enable(mtd);
1417
1418         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1419                nfc->regs + NFC_REG_RCMD_SET);
1420
1421         dma_async_issue_pending(nfc->dmac);
1422
1423         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1424                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1425                nfc->regs + NFC_REG_CMD);
1426
1427         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1428         if (ret)
1429                 dmaengine_terminate_all(nfc->dmac);
1430
1431         sunxi_nfc_randomizer_disable(mtd);
1432         sunxi_nfc_hw_ecc_disable(mtd);
1433
1434         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1435
1436         if (ret)
1437                 return ret;
1438
1439         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1440                 /* TODO: use DMA to transfer extra OOB bytes ? */
1441                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1442                                                  NULL, page);
1443
1444         return 0;
1445
1446 pio_fallback:
1447         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1448 }
1449
1450 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1451                                                struct nand_chip *chip,
1452                                                uint8_t *buf, int oob_required,
1453                                                int page)
1454 {
1455         struct nand_ecc_ctrl *ecc = &chip->ecc;
1456         unsigned int max_bitflips = 0;
1457         int ret, i, cur_off = 0;
1458         bool raw_mode = false;
1459
1460         sunxi_nfc_hw_ecc_enable(mtd);
1461
1462         for (i = 0; i < ecc->steps; i++) {
1463                 int data_off = i * (ecc->size + ecc->bytes + 4);
1464                 int oob_off = data_off + ecc->size;
1465                 u8 *data = buf + (i * ecc->size);
1466                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1467
1468                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1469                                                   oob_off, &cur_off,
1470                                                   &max_bitflips, !i,
1471                                                   oob_required,
1472                                                   page);
1473                 if (ret < 0)
1474                         return ret;
1475                 else if (ret)
1476                         raw_mode = true;
1477         }
1478
1479         if (oob_required)
1480                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1481                                                 !raw_mode, page);
1482
1483         sunxi_nfc_hw_ecc_disable(mtd);
1484
1485         return max_bitflips;
1486 }
1487
1488 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1489                                                 struct nand_chip *chip,
1490                                                 const uint8_t *buf,
1491                                                 int oob_required, int page)
1492 {
1493         struct nand_ecc_ctrl *ecc = &chip->ecc;
1494         int ret, i, cur_off = 0;
1495
1496         sunxi_nfc_hw_ecc_enable(mtd);
1497
1498         for (i = 0; i < ecc->steps; i++) {
1499                 int data_off = i * (ecc->size + ecc->bytes + 4);
1500                 int oob_off = data_off + ecc->size;
1501                 const u8 *data = buf + (i * ecc->size);
1502                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1503
1504                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1505                                                    oob, oob_off, &cur_off,
1506                                                    false, page);
1507                 if (ret)
1508                         return ret;
1509         }
1510
1511         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1512                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1513                                                  &cur_off, page);
1514
1515         sunxi_nfc_hw_ecc_disable(mtd);
1516
1517         return 0;
1518 }
1519
1520 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1521                                             struct nand_chip *chip,
1522                                             int page)
1523 {
1524         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1525
1526         chip->pagebuf = -1;
1527
1528         return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1529 }
1530
1531 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1532                                              struct nand_chip *chip,
1533                                              int page)
1534 {
1535         int ret, status;
1536
1537         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1538
1539         chip->pagebuf = -1;
1540
1541         memset(chip->buffers->databuf, 0xff, mtd->writesize);
1542         ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1543         if (ret)
1544                 return ret;
1545
1546         /* Send command to program the OOB data */
1547         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1548
1549         status = chip->waitfunc(mtd, chip);
1550
1551         return status & NAND_STATUS_FAIL ? -EIO : 0;
1552 }
1553
1554 static const s32 tWB_lut[] = {6, 12, 16, 20};
1555 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1556
1557 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1558                 u32 clk_period)
1559 {
1560         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1561         int i;
1562
1563         for (i = 0; i < lut_size; i++) {
1564                 if (clk_cycles <= lut[i])
1565                         return i;
1566         }
1567
1568         /* Doesn't fit */
1569         return -EINVAL;
1570 }
1571
1572 #define sunxi_nand_lookup_timing(l, p, c) \
1573                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1574
1575 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1576                                         const struct nand_data_interface *conf,
1577                                         bool check_only)
1578 {
1579         struct nand_chip *nand = mtd_to_nand(mtd);
1580         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1581         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1582         const struct nand_sdr_timings *timings;
1583         u32 min_clk_period = 0;
1584         s32 tWB, tADL, tWHR, tRHW, tCAD;
1585         long real_clk_rate;
1586
1587         timings = nand_get_sdr_timings(conf);
1588         if (IS_ERR(timings))
1589                 return -ENOTSUPP;
1590
1591         /* T1 <=> tCLS */
1592         if (timings->tCLS_min > min_clk_period)
1593                 min_clk_period = timings->tCLS_min;
1594
1595         /* T2 <=> tCLH */
1596         if (timings->tCLH_min > min_clk_period)
1597                 min_clk_period = timings->tCLH_min;
1598
1599         /* T3 <=> tCS */
1600         if (timings->tCS_min > min_clk_period)
1601                 min_clk_period = timings->tCS_min;
1602
1603         /* T4 <=> tCH */
1604         if (timings->tCH_min > min_clk_period)
1605                 min_clk_period = timings->tCH_min;
1606
1607         /* T5 <=> tWP */
1608         if (timings->tWP_min > min_clk_period)
1609                 min_clk_period = timings->tWP_min;
1610
1611         /* T6 <=> tWH */
1612         if (timings->tWH_min > min_clk_period)
1613                 min_clk_period = timings->tWH_min;
1614
1615         /* T7 <=> tALS */
1616         if (timings->tALS_min > min_clk_period)
1617                 min_clk_period = timings->tALS_min;
1618
1619         /* T8 <=> tDS */
1620         if (timings->tDS_min > min_clk_period)
1621                 min_clk_period = timings->tDS_min;
1622
1623         /* T9 <=> tDH */
1624         if (timings->tDH_min > min_clk_period)
1625                 min_clk_period = timings->tDH_min;
1626
1627         /* T10 <=> tRR */
1628         if (timings->tRR_min > (min_clk_period * 3))
1629                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1630
1631         /* T11 <=> tALH */
1632         if (timings->tALH_min > min_clk_period)
1633                 min_clk_period = timings->tALH_min;
1634
1635         /* T12 <=> tRP */
1636         if (timings->tRP_min > min_clk_period)
1637                 min_clk_period = timings->tRP_min;
1638
1639         /* T13 <=> tREH */
1640         if (timings->tREH_min > min_clk_period)
1641                 min_clk_period = timings->tREH_min;
1642
1643         /* T14 <=> tRC */
1644         if (timings->tRC_min > (min_clk_period * 2))
1645                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1646
1647         /* T15 <=> tWC */
1648         if (timings->tWC_min > (min_clk_period * 2))
1649                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1650
1651         /* T16 - T19 + tCAD */
1652         if (timings->tWB_max > (min_clk_period * 20))
1653                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1654
1655         if (timings->tADL_min > (min_clk_period * 32))
1656                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1657
1658         if (timings->tWHR_min > (min_clk_period * 32))
1659                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1660
1661         if (timings->tRHW_min > (min_clk_period * 20))
1662                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1663
1664         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1665                                         min_clk_period);
1666         if (tWB < 0) {
1667                 dev_err(nfc->dev, "unsupported tWB\n");
1668                 return tWB;
1669         }
1670
1671         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1672         if (tADL > 3) {
1673                 dev_err(nfc->dev, "unsupported tADL\n");
1674                 return -EINVAL;
1675         }
1676
1677         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1678         if (tWHR > 3) {
1679                 dev_err(nfc->dev, "unsupported tWHR\n");
1680                 return -EINVAL;
1681         }
1682
1683         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1684                                         min_clk_period);
1685         if (tRHW < 0) {
1686                 dev_err(nfc->dev, "unsupported tRHW\n");
1687                 return tRHW;
1688         }
1689
1690         if (check_only)
1691                 return 0;
1692
1693         /*
1694          * TODO: according to ONFI specs this value only applies for DDR NAND,
1695          * but Allwinner seems to set this to 0x7. Mimic them for now.
1696          */
1697         tCAD = 0x7;
1698
1699         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1700         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1701
1702         /* Convert min_clk_period from picoseconds to nanoseconds */
1703         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1704
1705         /*
1706          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1707          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1708          * This new formula was verified with a scope and validated by
1709          * Allwinner engineers.
1710          */
1711         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1712         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1713
1714         /*
1715          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1716          * output cycle timings shall be used if the host drives tRC less than
1717          * 30 ns.
1718          */
1719         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1720         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1721                            NFC_TIMING_CTL_EDO : 0;
1722
1723         return 0;
1724 }
1725
1726 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1727                                     struct mtd_oob_region *oobregion)
1728 {
1729         struct nand_chip *nand = mtd_to_nand(mtd);
1730         struct nand_ecc_ctrl *ecc = &nand->ecc;
1731
1732         if (section >= ecc->steps)
1733                 return -ERANGE;
1734
1735         oobregion->offset = section * (ecc->bytes + 4) + 4;
1736         oobregion->length = ecc->bytes;
1737
1738         return 0;
1739 }
1740
1741 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1742                                      struct mtd_oob_region *oobregion)
1743 {
1744         struct nand_chip *nand = mtd_to_nand(mtd);
1745         struct nand_ecc_ctrl *ecc = &nand->ecc;
1746
1747         if (section > ecc->steps)
1748                 return -ERANGE;
1749
1750         /*
1751          * The first 2 bytes are used for BB markers, hence we
1752          * only have 2 bytes available in the first user data
1753          * section.
1754          */
1755         if (!section && ecc->mode == NAND_ECC_HW) {
1756                 oobregion->offset = 2;
1757                 oobregion->length = 2;
1758
1759                 return 0;
1760         }
1761
1762         oobregion->offset = section * (ecc->bytes + 4);
1763
1764         if (section < ecc->steps)
1765                 oobregion->length = 4;
1766         else
1767                 oobregion->offset = mtd->oobsize - oobregion->offset;
1768
1769         return 0;
1770 }
1771
1772 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1773         .ecc = sunxi_nand_ooblayout_ecc,
1774         .free = sunxi_nand_ooblayout_free,
1775 };
1776
1777 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1778                                               struct nand_ecc_ctrl *ecc,
1779                                               struct device_node *np)
1780 {
1781         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1782         struct nand_chip *nand = mtd_to_nand(mtd);
1783         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1784         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1785         struct sunxi_nand_hw_ecc *data;
1786         int nsectors;
1787         int ret;
1788         int i;
1789
1790         if (ecc->options & NAND_ECC_MAXIMIZE) {
1791                 int bytes;
1792
1793                 ecc->size = 1024;
1794                 nsectors = mtd->writesize / ecc->size;
1795
1796                 /* Reserve 2 bytes for the BBM */
1797                 bytes = (mtd->oobsize - 2) / nsectors;
1798
1799                 /* 4 non-ECC bytes are added before each ECC bytes section */
1800                 bytes -= 4;
1801
1802                 /* and bytes has to be even. */
1803                 if (bytes % 2)
1804                         bytes--;
1805
1806                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1807
1808                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1809                         if (strengths[i] > ecc->strength)
1810                                 break;
1811                 }
1812
1813                 if (!i)
1814                         ecc->strength = 0;
1815                 else
1816                         ecc->strength = strengths[i - 1];
1817         }
1818
1819         if (ecc->size != 512 && ecc->size != 1024)
1820                 return -EINVAL;
1821
1822         data = kzalloc(sizeof(*data), GFP_KERNEL);
1823         if (!data)
1824                 return -ENOMEM;
1825
1826         /* Prefer 1k ECC chunk over 512 ones */
1827         if (ecc->size == 512 && mtd->writesize > 512) {
1828                 ecc->size = 1024;
1829                 ecc->strength *= 2;
1830         }
1831
1832         /* Add ECC info retrieval from DT */
1833         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1834                 if (ecc->strength <= strengths[i])
1835                         break;
1836         }
1837
1838         if (i >= ARRAY_SIZE(strengths)) {
1839                 dev_err(nfc->dev, "unsupported strength\n");
1840                 ret = -ENOTSUPP;
1841                 goto err;
1842         }
1843
1844         data->mode = i;
1845
1846         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1847         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1848
1849         /* HW ECC always work with even numbers of ECC bytes */
1850         ecc->bytes = ALIGN(ecc->bytes, 2);
1851
1852         nsectors = mtd->writesize / ecc->size;
1853
1854         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1855                 ret = -EINVAL;
1856                 goto err;
1857         }
1858
1859         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1860         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1861         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1862         ecc->priv = data;
1863
1864         return 0;
1865
1866 err:
1867         kfree(data);
1868
1869         return ret;
1870 }
1871
1872 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1873 {
1874         kfree(ecc->priv);
1875 }
1876
1877 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1878                                        struct nand_ecc_ctrl *ecc,
1879                                        struct device_node *np)
1880 {
1881         struct nand_chip *nand = mtd_to_nand(mtd);
1882         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1883         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1884         int ret;
1885
1886         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1887         if (ret)
1888                 return ret;
1889
1890         if (nfc->dmac) {
1891                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1892                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1893                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1894                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1895         } else {
1896                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1897                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1898                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1899         }
1900
1901         /* TODO: support DMA for raw accesses and subpage write */
1902         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1903         ecc->read_oob_raw = nand_read_oob_std;
1904         ecc->write_oob_raw = nand_write_oob_std;
1905         ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1906
1907         return 0;
1908 }
1909
1910 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1911                                                 struct nand_ecc_ctrl *ecc,
1912                                                 struct device_node *np)
1913 {
1914         int ret;
1915
1916         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1917         if (ret)
1918                 return ret;
1919
1920         ecc->prepad = 4;
1921         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1922         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1923         ecc->read_oob_raw = nand_read_oob_syndrome;
1924         ecc->write_oob_raw = nand_write_oob_syndrome;
1925
1926         return 0;
1927 }
1928
1929 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1930 {
1931         switch (ecc->mode) {
1932         case NAND_ECC_HW:
1933         case NAND_ECC_HW_SYNDROME:
1934                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1935                 break;
1936         case NAND_ECC_NONE:
1937         default:
1938                 break;
1939         }
1940 }
1941
1942 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1943                                struct device_node *np)
1944 {
1945         struct nand_chip *nand = mtd_to_nand(mtd);
1946         int ret;
1947
1948         if (!ecc->size) {
1949                 ecc->size = nand->ecc_step_ds;
1950                 ecc->strength = nand->ecc_strength_ds;
1951         }
1952
1953         if (!ecc->size || !ecc->strength)
1954                 return -EINVAL;
1955
1956         switch (ecc->mode) {
1957         case NAND_ECC_HW:
1958                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1959                 if (ret)
1960                         return ret;
1961                 break;
1962         case NAND_ECC_HW_SYNDROME:
1963                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1964                 if (ret)
1965                         return ret;
1966                 break;
1967         case NAND_ECC_NONE:
1968         case NAND_ECC_SOFT:
1969                 break;
1970         default:
1971                 return -EINVAL;
1972         }
1973
1974         return 0;
1975 }
1976
1977 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1978                                 struct device_node *np)
1979 {
1980         struct sunxi_nand_chip *chip;
1981         struct mtd_info *mtd;
1982         struct nand_chip *nand;
1983         int nsels;
1984         int ret;
1985         int i;
1986         u32 tmp;
1987
1988         if (!of_get_property(np, "reg", &nsels))
1989                 return -EINVAL;
1990
1991         nsels /= sizeof(u32);
1992         if (!nsels) {
1993                 dev_err(dev, "invalid reg property size\n");
1994                 return -EINVAL;
1995         }
1996
1997         chip = devm_kzalloc(dev,
1998                             sizeof(*chip) +
1999                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2000                             GFP_KERNEL);
2001         if (!chip) {
2002                 dev_err(dev, "could not allocate chip\n");
2003                 return -ENOMEM;
2004         }
2005
2006         chip->nsels = nsels;
2007         chip->selected = -1;
2008
2009         for (i = 0; i < nsels; i++) {
2010                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2011                 if (ret) {
2012                         dev_err(dev, "could not retrieve reg property: %d\n",
2013                                 ret);
2014                         return ret;
2015                 }
2016
2017                 if (tmp > NFC_MAX_CS) {
2018                         dev_err(dev,
2019                                 "invalid reg value: %u (max CS = 7)\n",
2020                                 tmp);
2021                         return -EINVAL;
2022                 }
2023
2024                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2025                         dev_err(dev, "CS %d already assigned\n", tmp);
2026                         return -EINVAL;
2027                 }
2028
2029                 chip->sels[i].cs = tmp;
2030
2031                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2032                     tmp < 2) {
2033                         chip->sels[i].rb.type = RB_NATIVE;
2034                         chip->sels[i].rb.info.nativeid = tmp;
2035                 } else {
2036                         ret = of_get_named_gpio(np, "rb-gpios", i);
2037                         if (ret >= 0) {
2038                                 tmp = ret;
2039                                 chip->sels[i].rb.type = RB_GPIO;
2040                                 chip->sels[i].rb.info.gpio = tmp;
2041                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2042                                 if (ret)
2043                                         return ret;
2044
2045                                 ret = gpio_direction_input(tmp);
2046                                 if (ret)
2047                                         return ret;
2048                         } else {
2049                                 chip->sels[i].rb.type = RB_NONE;
2050                         }
2051                 }
2052         }
2053
2054         nand = &chip->nand;
2055         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2056         nand->chip_delay = 200;
2057         nand->controller = &nfc->controller;
2058         /*
2059          * Set the ECC mode to the default value in case nothing is specified
2060          * in the DT.
2061          */
2062         nand->ecc.mode = NAND_ECC_HW;
2063         nand_set_flash_node(nand, np);
2064         nand->select_chip = sunxi_nfc_select_chip;
2065         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2066         nand->read_buf = sunxi_nfc_read_buf;
2067         nand->write_buf = sunxi_nfc_write_buf;
2068         nand->read_byte = sunxi_nfc_read_byte;
2069         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2070
2071         mtd = nand_to_mtd(nand);
2072         mtd->dev.parent = dev;
2073
2074         ret = nand_scan_ident(mtd, nsels, NULL);
2075         if (ret)
2076                 return ret;
2077
2078         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2079                 nand->bbt_options |= NAND_BBT_NO_OOB;
2080
2081         if (nand->options & NAND_NEED_SCRAMBLING)
2082                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2083
2084         nand->options |= NAND_SUBPAGE_READ;
2085
2086         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2087         if (ret) {
2088                 dev_err(dev, "ECC init failed: %d\n", ret);
2089                 return ret;
2090         }
2091
2092         ret = nand_scan_tail(mtd);
2093         if (ret) {
2094                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2095                 return ret;
2096         }
2097
2098         ret = mtd_device_register(mtd, NULL, 0);
2099         if (ret) {
2100                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2101                 nand_release(mtd);
2102                 return ret;
2103         }
2104
2105         list_add_tail(&chip->node, &nfc->chips);
2106
2107         return 0;
2108 }
2109
2110 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2111 {
2112         struct device_node *np = dev->of_node;
2113         struct device_node *nand_np;
2114         int nchips = of_get_child_count(np);
2115         int ret;
2116
2117         if (nchips > 8) {
2118                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2119                 return -EINVAL;
2120         }
2121
2122         for_each_child_of_node(np, nand_np) {
2123                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2124                 if (ret) {
2125                         of_node_put(nand_np);
2126                         return ret;
2127                 }
2128         }
2129
2130         return 0;
2131 }
2132
2133 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2134 {
2135         struct sunxi_nand_chip *chip;
2136
2137         while (!list_empty(&nfc->chips)) {
2138                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2139                                         node);
2140                 nand_release(nand_to_mtd(&chip->nand));
2141                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2142                 list_del(&chip->node);
2143         }
2144 }
2145
2146 static int sunxi_nfc_probe(struct platform_device *pdev)
2147 {
2148         struct device *dev = &pdev->dev;
2149         struct resource *r;
2150         struct sunxi_nfc *nfc;
2151         int irq;
2152         int ret;
2153
2154         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2155         if (!nfc)
2156                 return -ENOMEM;
2157
2158         nfc->dev = dev;
2159         nand_hw_control_init(&nfc->controller);
2160         INIT_LIST_HEAD(&nfc->chips);
2161
2162         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2163         nfc->regs = devm_ioremap_resource(dev, r);
2164         if (IS_ERR(nfc->regs))
2165                 return PTR_ERR(nfc->regs);
2166
2167         irq = platform_get_irq(pdev, 0);
2168         if (irq < 0) {
2169                 dev_err(dev, "failed to retrieve irq\n");
2170                 return irq;
2171         }
2172
2173         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2174         if (IS_ERR(nfc->ahb_clk)) {
2175                 dev_err(dev, "failed to retrieve ahb clk\n");
2176                 return PTR_ERR(nfc->ahb_clk);
2177         }
2178
2179         ret = clk_prepare_enable(nfc->ahb_clk);
2180         if (ret)
2181                 return ret;
2182
2183         nfc->mod_clk = devm_clk_get(dev, "mod");
2184         if (IS_ERR(nfc->mod_clk)) {
2185                 dev_err(dev, "failed to retrieve mod clk\n");
2186                 ret = PTR_ERR(nfc->mod_clk);
2187                 goto out_ahb_clk_unprepare;
2188         }
2189
2190         ret = clk_prepare_enable(nfc->mod_clk);
2191         if (ret)
2192                 goto out_ahb_clk_unprepare;
2193
2194         nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2195         if (!IS_ERR(nfc->reset)) {
2196                 ret = reset_control_deassert(nfc->reset);
2197                 if (ret) {
2198                         dev_err(dev, "reset err %d\n", ret);
2199                         goto out_mod_clk_unprepare;
2200                 }
2201         } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2202                 ret = PTR_ERR(nfc->reset);
2203                 goto out_mod_clk_unprepare;
2204         }
2205
2206         ret = sunxi_nfc_rst(nfc);
2207         if (ret)
2208                 goto out_ahb_reset_reassert;
2209
2210         writel(0, nfc->regs + NFC_REG_INT);
2211         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2212                                0, "sunxi-nand", nfc);
2213         if (ret)
2214                 goto out_ahb_reset_reassert;
2215
2216         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2217         if (nfc->dmac) {
2218                 struct dma_slave_config dmac_cfg = { };
2219
2220                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2221                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2222                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2223                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2224                 dmac_cfg.src_maxburst = 4;
2225                 dmac_cfg.dst_maxburst = 4;
2226                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2227         } else {
2228                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2229         }
2230
2231         platform_set_drvdata(pdev, nfc);
2232
2233         ret = sunxi_nand_chips_init(dev, nfc);
2234         if (ret) {
2235                 dev_err(dev, "failed to init nand chips\n");
2236                 goto out_release_dmac;
2237         }
2238
2239         return 0;
2240
2241 out_release_dmac:
2242         if (nfc->dmac)
2243                 dma_release_channel(nfc->dmac);
2244 out_ahb_reset_reassert:
2245         if (!IS_ERR(nfc->reset))
2246                 reset_control_assert(nfc->reset);
2247 out_mod_clk_unprepare:
2248         clk_disable_unprepare(nfc->mod_clk);
2249 out_ahb_clk_unprepare:
2250         clk_disable_unprepare(nfc->ahb_clk);
2251
2252         return ret;
2253 }
2254
2255 static int sunxi_nfc_remove(struct platform_device *pdev)
2256 {
2257         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2258
2259         sunxi_nand_chips_cleanup(nfc);
2260
2261         if (!IS_ERR(nfc->reset))
2262                 reset_control_assert(nfc->reset);
2263
2264         if (nfc->dmac)
2265                 dma_release_channel(nfc->dmac);
2266         clk_disable_unprepare(nfc->mod_clk);
2267         clk_disable_unprepare(nfc->ahb_clk);
2268
2269         return 0;
2270 }
2271
2272 static const struct of_device_id sunxi_nfc_ids[] = {
2273         { .compatible = "allwinner,sun4i-a10-nand" },
2274         { /* sentinel */ }
2275 };
2276 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2277
2278 static struct platform_driver sunxi_nfc_driver = {
2279         .driver = {
2280                 .name = "sunxi_nand",
2281                 .of_match_table = sunxi_nfc_ids,
2282         },
2283         .probe = sunxi_nfc_probe,
2284         .remove = sunxi_nfc_remove,
2285 };
2286 module_platform_driver(sunxi_nfc_driver);
2287
2288 MODULE_LICENSE("GPL v2");
2289 MODULE_AUTHOR("Boris BREZILLON");
2290 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2291 MODULE_ALIAS("platform:sunxi_nand");