]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/dma/ste_dma40.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
DMAENGINE: ste_dma40: config checks
[net-next-2.6.git] / drivers / dma / ste_dma40.c
CommitLineData
8d318a50
LW		 1/*
2 * driver/dma/ste_dma40.c
3 *
4 * Copyright (C) ST-Ericsson 2007-2010
5 * License terms: GNU General Public License (GPL) version 2
6 * Author: Per Friden <per.friden@stericsson.com>
7 * Author: Jonas Aaberg <jonas.aberg@stericsson.com>
8 *
9 */
10
11#include <linux/kernel.h>
12#include <linux/slab.h>
13#include <linux/dmaengine.h>
14#include <linux/platform_device.h>
15#include <linux/clk.h>
16#include <linux/delay.h>
17
18#include <plat/ste_dma40.h>
19
20#include "ste_dma40_ll.h"
21
22#define D40_NAME "dma40"
23
24#define D40_PHY_CHAN -1
25
26/* For masking out/in 2 bit channel positions */
27#define D40_CHAN_POS(chan) (2 * (chan / 2))
28#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
29
30/* Maximum iterations taken before giving up suspending a channel */
31#define D40_SUSPEND_MAX_IT 500
32
508849ad
LW		 33/* Hardware requirement on LCLA alignment */
34#define LCLA_ALIGNMENT 0x40000
35/* Attempts before giving up to trying to get pages that are aligned */
36#define MAX_LCLA_ALLOC_ATTEMPTS 256
37
38/* Bit markings for allocation map */
8d318a50
LW		 39#define D40_ALLOC_FREE (1 << 31)
40#define D40_ALLOC_PHY (1 << 30)
41#define D40_ALLOC_LOG_FREE 0
42
8d318a50
LW		 43/* Hardware designer of the block */
44#define D40_PERIPHID2_DESIGNER 0x8
45
46/**
47 * enum 40_command - The different commands and/or statuses.
48 *
49 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
50 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
51 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
52 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
53 */
54enum d40_command {
55 D40_DMA_STOP = 0,
56 D40_DMA_RUN = 1,
57 D40_DMA_SUSPEND_REQ = 2,
58 D40_DMA_SUSPENDED = 3
59};
60
61/**
62 * struct d40_lli_pool - Structure for keeping LLIs in memory
63 *
64 * @base: Pointer to memory area when the pre_alloc_lli's are not large
65 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
66 * pre_alloc_lli is used.
67 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
68 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
69 * one buffer to one buffer.
70 */
71struct d40_lli_pool {
72 void *base;
508849ad 73 int size;
8d318a50 74 /* Space for dst and src, plus an extra for padding */
508849ad 75 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
8d318a50
LW		 76};
77
78/**
79 * struct d40_desc - A descriptor is one DMA job.
80 *
81 * @lli_phy: LLI settings for physical channel. Both src and dst=
82 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
83 * lli_len equals one.
84 * @lli_log: Same as above but for logical channels.
85 * @lli_pool: The pool with two entries pre-allocated.
941b77a3
PF		 86 * @lli_len: Number of llis of current descriptor.
87 * @lli_count: Number of transfered llis.
88 * @lli_tx_len: Max number of LLIs per transfer, there can be
89 * many transfer for one descriptor.
8d318a50
LW		 90 * @txd: DMA engine struct. Used for among other things for communication
91 * during a transfer.
92 * @node: List entry.
93 * @dir: The transfer direction of this job.
94 * @is_in_client_list: true if the client owns this descriptor.
95 *
96 * This descriptor is used for both logical and physical transfers.
97 */
98
99struct d40_desc {
100 /* LLI physical */
101 struct d40_phy_lli_bidir lli_phy;
102 /* LLI logical */
103 struct d40_log_lli_bidir lli_log;
104
105 struct d40_lli_pool lli_pool;
941b77a3
PF		 106 int lli_len;
107 int lli_count;
108 u32 lli_tx_len;
8d318a50
LW		 109
110 struct dma_async_tx_descriptor txd;
111 struct list_head node;
112
113 enum dma_data_direction dir;
114 bool is_in_client_list;
115};
116
117/**
118 * struct d40_lcla_pool - LCLA pool settings and data.
119 *
508849ad
LW		 120 * @base: The virtual address of LCLA. 18 bit aligned.
121 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
122 * This pointer is only there for clean-up on error.
123 * @pages: The number of pages needed for all physical channels.
124 * Only used later for clean-up on error
8d318a50 125 * @lock: Lock to protect the content in this struct.
508849ad 126 * @alloc_map: Bitmap mapping between physical channel and LCLA entries.
8d318a50
LW		 127 * @num_blocks: The number of entries of alloc_map. Equals to the
128 * number of physical channels.
129 */
130struct d40_lcla_pool {
131 void *base;
508849ad
LW		 132 void *base_unaligned;
133 int pages;
8d318a50
LW		 134 spinlock_t lock;
135 u32 *alloc_map;
136 int num_blocks;
137};
138
139/**
140 * struct d40_phy_res - struct for handling eventlines mapped to physical
141 * channels.
142 *
143 * @lock: A lock protection this entity.
144 * @num: The physical channel number of this entity.
145 * @allocated_src: Bit mapped to show which src event line's are mapped to
146 * this physical channel. Can also be free or physically allocated.
147 * @allocated_dst: Same as for src but is dst.
148 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
149 * event line number. Both allocated_src and allocated_dst can not be
150 * allocated to a physical channel, since the interrupt handler has then
151 * no way of figure out which one the interrupt belongs to.
152 */
153struct d40_phy_res {
154 spinlock_t lock;
155 int num;
156 u32 allocated_src;
157 u32 allocated_dst;
158};
159
160struct d40_base;
161
162/**
163 * struct d40_chan - Struct that describes a channel.
164 *
165 * @lock: A spinlock to protect this struct.
166 * @log_num: The logical number, if any of this channel.
167 * @completed: Starts with 1, after first interrupt it is set to dma engine's
168 * current cookie.
169 * @pending_tx: The number of pending transfers. Used between interrupt handler
170 * and tasklet.
171 * @busy: Set to true when transfer is ongoing on this channel.
2a614340
JA		 172 * @phy_chan: Pointer to physical channel which this instance runs on. If this
173 * point is NULL, then the channel is not allocated.
8d318a50
LW		 174 * @chan: DMA engine handle.
175 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
176 * transfer and call client callback.
177 * @client: Cliented owned descriptor list.
178 * @active: Active descriptor.
179 * @queue: Queued jobs.
8d318a50
LW		 180 * @dma_cfg: The client configuration of this dma channel.
181 * @base: Pointer to the device instance struct.
182 * @src_def_cfg: Default cfg register setting for src.
183 * @dst_def_cfg: Default cfg register setting for dst.
184 * @log_def: Default logical channel settings.
185 * @lcla: Space for one dst src pair for logical channel transfers.
186 * @lcpa: Pointer to dst and src lcpa settings.
187 *
188 * This struct can either "be" a logical or a physical channel.
189 */
190struct d40_chan {
191 spinlock_t lock;
192 int log_num;
193 /* ID of the most recent completed transfer */
194 int completed;
195 int pending_tx;
196 bool busy;
197 struct d40_phy_res *phy_chan;
198 struct dma_chan chan;
199 struct tasklet_struct tasklet;
200 struct list_head client;
201 struct list_head active;
202 struct list_head queue;
8d318a50
LW		 203 struct stedma40_chan_cfg dma_cfg;
204 struct d40_base *base;
205 /* Default register configurations */
206 u32 src_def_cfg;
207 u32 dst_def_cfg;
208 struct d40_def_lcsp log_def;
209 struct d40_lcla_elem lcla;
210 struct d40_log_lli_full *lcpa;
95e1400f
LW		 211 /* Runtime reconfiguration */
212 dma_addr_t runtime_addr;
213 enum dma_data_direction runtime_direction;
8d318a50
LW		 214};
215
216/**
217 * struct d40_base - The big global struct, one for each probe'd instance.
218 *
219 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
220 * @execmd_lock: Lock for execute command usage since several channels share
221 * the same physical register.
222 * @dev: The device structure.
223 * @virtbase: The virtual base address of the DMA's register.
f4185592 224 * @rev: silicon revision detected.
8d318a50
LW		 225 * @clk: Pointer to the DMA clock structure.
226 * @phy_start: Physical memory start of the DMA registers.
227 * @phy_size: Size of the DMA register map.
228 * @irq: The IRQ number.
229 * @num_phy_chans: The number of physical channels. Read from HW. This
230 * is the number of available channels for this driver, not counting "Secure
231 * mode" allocated physical channels.
232 * @num_log_chans: The number of logical channels. Calculated from
233 * num_phy_chans.
234 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
235 * @dma_slave: dma_device channels that can do only do slave transfers.
236 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
237 * @phy_chans: Room for all possible physical channels in system.
238 * @log_chans: Room for all possible logical channels in system.
239 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
240 * to log_chans entries.
241 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
242 * to phy_chans entries.
243 * @plat_data: Pointer to provided platform_data which is the driver
244 * configuration.
245 * @phy_res: Vector containing all physical channels.
246 * @lcla_pool: lcla pool settings and data.
247 * @lcpa_base: The virtual mapped address of LCPA.
248 * @phy_lcpa: The physical address of the LCPA.
249 * @lcpa_size: The size of the LCPA area.
c675b1b4 250 * @desc_slab: cache for descriptors.
8d318a50
LW		 251 */
252struct d40_base {
253 spinlock_t interrupt_lock;
254 spinlock_t execmd_lock;
255 struct device *dev;
256 void __iomem *virtbase;
f4185592 257 u8 rev:4;
8d318a50
LW		 258 struct clk *clk;
259 phys_addr_t phy_start;
260 resource_size_t phy_size;
261 int irq;
262 int num_phy_chans;
263 int num_log_chans;
264 struct dma_device dma_both;
265 struct dma_device dma_slave;
266 struct dma_device dma_memcpy;
267 struct d40_chan *phy_chans;
268 struct d40_chan *log_chans;
269 struct d40_chan **lookup_log_chans;
270 struct d40_chan **lookup_phy_chans;
271 struct stedma40_platform_data *plat_data;
272 /* Physical half channels */
273 struct d40_phy_res *phy_res;
274 struct d40_lcla_pool lcla_pool;
275 void *lcpa_base;
276 dma_addr_t phy_lcpa;
277 resource_size_t lcpa_size;
c675b1b4 278 struct kmem_cache *desc_slab;
8d318a50
LW		 279};
280
281/**
282 * struct d40_interrupt_lookup - lookup table for interrupt handler
283 *
284 * @src: Interrupt mask register.
285 * @clr: Interrupt clear register.
286 * @is_error: true if this is an error interrupt.
287 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
288 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
289 */
290struct d40_interrupt_lookup {
291 u32 src;
292 u32 clr;
293 bool is_error;
294 int offset;
295};
296
297/**
298 * struct d40_reg_val - simple lookup struct
299 *
300 * @reg: The register.
301 * @val: The value that belongs to the register in reg.
302 */
303struct d40_reg_val {
304 unsigned int reg;
305 unsigned int val;
306};
307
308static int d40_pool_lli_alloc(struct d40_desc *d40d,
309 int lli_len, bool is_log)
310{
311 u32 align;
312 void *base;
313
314 if (is_log)
315 align = sizeof(struct d40_log_lli);
316 else
317 align = sizeof(struct d40_phy_lli);
318
319 if (lli_len == 1) {
320 base = d40d->lli_pool.pre_alloc_lli;
321 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
322 d40d->lli_pool.base = NULL;
323 } else {
324 d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
325
326 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
327 d40d->lli_pool.base = base;
328
329 if (d40d->lli_pool.base == NULL)
330 return -ENOMEM;
331 }
332
333 if (is_log) {
334 d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
335 align);
336 d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
337 align);
338 } else {
339 d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
340 align);
341 d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
342 align);
343
344 d40d->lli_phy.src_addr = virt_to_phys(d40d->lli_phy.src);
345 d40d->lli_phy.dst_addr = virt_to_phys(d40d->lli_phy.dst);
346 }
347
348 return 0;
349}
350
351static void d40_pool_lli_free(struct d40_desc *d40d)
352{
353 kfree(d40d->lli_pool.base);
354 d40d->lli_pool.base = NULL;
355 d40d->lli_pool.size = 0;
356 d40d->lli_log.src = NULL;
357 d40d->lli_log.dst = NULL;
358 d40d->lli_phy.src = NULL;
359 d40d->lli_phy.dst = NULL;
360 d40d->lli_phy.src_addr = 0;
361 d40d->lli_phy.dst_addr = 0;
362}
363
364static dma_cookie_t d40_assign_cookie(struct d40_chan *d40c,
365 struct d40_desc *desc)
366{
367 dma_cookie_t cookie = d40c->chan.cookie;
368
369 if (++cookie < 0)
370 cookie = 1;
371
372 d40c->chan.cookie = cookie;
373 desc->txd.cookie = cookie;
374
375 return cookie;
376}
377
8d318a50
LW		 378static void d40_desc_remove(struct d40_desc *d40d)
379{
380 list_del(&d40d->node);
381}
382
383static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
384{
8d318a50
LW		 385 struct d40_desc *d;
386 struct d40_desc *_d;
387
388 if (!list_empty(&d40c->client)) {
389 list_for_each_entry_safe(d, _d, &d40c->client, node)
390 if (async_tx_test_ack(&d->txd)) {
391 d40_pool_lli_free(d);
392 d40_desc_remove(d);
c675b1b4 393 break;
8d318a50 394 }
8d318a50 395 } else {
c675b1b4
JA		 396 d = kmem_cache_alloc(d40c->base->desc_slab, GFP_NOWAIT);
397 if (d != NULL) {
398 memset(d, 0, sizeof(struct d40_desc));
399 INIT_LIST_HEAD(&d->node);
400 }
8d318a50 401 }
c675b1b4 402 return d;
8d318a50
LW		 403}
404
405static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
406{
c675b1b4 407 kmem_cache_free(d40c->base->desc_slab, d40d);
8d318a50
LW		 408}
409
410static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
411{
412 list_add_tail(&desc->node, &d40c->active);
413}
414
415static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
416{
417 struct d40_desc *d;
418
419 if (list_empty(&d40c->active))
420 return NULL;
421
422 d = list_first_entry(&d40c->active,
423 struct d40_desc,
424 node);
425 return d;
426}
427
428static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
429{
430 list_add_tail(&desc->node, &d40c->queue);
431}
432
433static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
434{
435 struct d40_desc *d;
436
437 if (list_empty(&d40c->queue))
438 return NULL;
439
440 d = list_first_entry(&d40c->queue,
441 struct d40_desc,
442 node);
443 return d;
444}
445
446/* Support functions for logical channels */
447
508849ad 448static int d40_lcla_id_get(struct d40_chan *d40c)
8d318a50
LW		 449{
450 int src_id = 0;
451 int dst_id = 0;
452 struct d40_log_lli *lcla_lidx_base =
508849ad 453 d40c->base->lcla_pool.base + d40c->phy_chan->num * 1024;
8d318a50
LW		 454 int i;
455 int lli_per_log = d40c->base->plat_data->llis_per_log;
2292b880 456 unsigned long flags;
8d318a50
LW		 457
458 if (d40c->lcla.src_id >= 0 && d40c->lcla.dst_id >= 0)
459 return 0;
460
508849ad 461 if (d40c->base->lcla_pool.num_blocks > 32)
8d318a50
LW		 462 return -EINVAL;
463
508849ad 464 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
8d318a50 465
508849ad
LW		 466 for (i = 0; i < d40c->base->lcla_pool.num_blocks; i++) {
467 if (!(d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &
468 (0x1 << i))) {
469 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] |=
470 (0x1 << i);
8d318a50
LW		 471 break;
472 }
473 }
474 src_id = i;
508849ad 475 if (src_id >= d40c->base->lcla_pool.num_blocks)
8d318a50
LW		 476 goto err;
477
508849ad
LW		 478 for (; i < d40c->base->lcla_pool.num_blocks; i++) {
479 if (!(d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &
480 (0x1 << i))) {
481 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] |=
482 (0x1 << i);
8d318a50
LW		 483 break;
484 }
485 }
486
487 dst_id = i;
488 if (dst_id == src_id)
489 goto err;
490
491 d40c->lcla.src_id = src_id;
492 d40c->lcla.dst_id = dst_id;
493 d40c->lcla.dst = lcla_lidx_base + dst_id * lli_per_log + 1;
494 d40c->lcla.src = lcla_lidx_base + src_id * lli_per_log + 1;
495
508849ad 496 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
8d318a50
LW		 497 return 0;
498err:
508849ad 499 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
8d318a50
LW		 500 return -EINVAL;
501}
502
8d318a50
LW		 503
504static int d40_channel_execute_command(struct d40_chan *d40c,
505 enum d40_command command)
506{
507 int status, i;
508 void __iomem *active_reg;
509 int ret = 0;
510 unsigned long flags;
1d392a7b 511 u32 wmask;
8d318a50
LW		 512
513 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
514
515 if (d40c->phy_chan->num % 2 == 0)
516 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
517 else
518 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
519
520 if (command == D40_DMA_SUSPEND_REQ) {
521 status = (readl(active_reg) &
522 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
523 D40_CHAN_POS(d40c->phy_chan->num);
524
525 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
526 goto done;
527 }
528
1d392a7b
JA		 529 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
530 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
531 active_reg);
8d318a50
LW		 532
533 if (command == D40_DMA_SUSPEND_REQ) {
534
535 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
536 status = (readl(active_reg) &
537 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
538 D40_CHAN_POS(d40c->phy_chan->num);
539
540 cpu_relax();
541 /*
542 * Reduce the number of bus accesses while
543 * waiting for the DMA to suspend.
544 */
545 udelay(3);
546
547 if (status == D40_DMA_STOP ||
548 status == D40_DMA_SUSPENDED)
549 break;
550 }
551
552 if (i == D40_SUSPEND_MAX_IT) {
553 dev_err(&d40c->chan.dev->device,
554 "[%s]: unable to suspend the chl %d (log: %d) status %x\n",
555 __func__, d40c->phy_chan->num, d40c->log_num,
556 status);
557 dump_stack();
558 ret = -EBUSY;
559 }
560
561 }
562done:
563 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
564 return ret;
565}
566
567static void d40_term_all(struct d40_chan *d40c)
568{
569 struct d40_desc *d40d;
508849ad 570 unsigned long flags;
8d318a50
LW		 571
572 /* Release active descriptors */
573 while ((d40d = d40_first_active_get(d40c))) {
574 d40_desc_remove(d40d);
575
576 /* Return desc to free-list */
577 d40_desc_free(d40c, d40d);
578 }
579
580 /* Release queued descriptors waiting for transfer */
581 while ((d40d = d40_first_queued(d40c))) {
582 d40_desc_remove(d40d);
583
584 /* Return desc to free-list */
585 d40_desc_free(d40c, d40d);
586 }
587
508849ad
LW		 588 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
589
590 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &=
591 (~(0x1 << d40c->lcla.dst_id));
592 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &=
593 (~(0x1 << d40c->lcla.src_id));
594
595 d40c->lcla.src_id = -1;
596 d40c->lcla.dst_id = -1;
597
598 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
8d318a50
LW		 599
600 d40c->pending_tx = 0;
601 d40c->busy = false;
602}
603
604static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
605{
606 u32 val;
607 unsigned long flags;
608
0c32269d 609 /* Notice, that disable requires the physical channel to be stopped */
8d318a50
LW		 610 if (do_enable)
611 val = D40_ACTIVATE_EVENTLINE;
612 else
613 val = D40_DEACTIVATE_EVENTLINE;
614
615 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
616
617 /* Enable event line connected to device (or memcpy) */
618 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
619 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
620 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
621
622 writel((val << D40_EVENTLINE_POS(event)) |
623 ~D40_EVENTLINE_MASK(event),
624 d40c->base->virtbase + D40_DREG_PCBASE +
625 d40c->phy_chan->num * D40_DREG_PCDELTA +
626 D40_CHAN_REG_SSLNK);
627 }
628 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
629 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
630
631 writel((val << D40_EVENTLINE_POS(event)) |
632 ~D40_EVENTLINE_MASK(event),
633 d40c->base->virtbase + D40_DREG_PCBASE +
634 d40c->phy_chan->num * D40_DREG_PCDELTA +
635 D40_CHAN_REG_SDLNK);
636 }
637
638 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
639}
640
a5ebca47 641static u32 d40_chan_has_events(struct d40_chan *d40c)
8d318a50
LW		 642{
643 u32 val = 0;
644
645 /* If SSLNK or SDLNK is zero all events are disabled */
646 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
647 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
648 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
649 d40c->phy_chan->num * D40_DREG_PCDELTA +
650 D40_CHAN_REG_SSLNK);
651
652 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM)
653 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
654 d40c->phy_chan->num * D40_DREG_PCDELTA +
655 D40_CHAN_REG_SDLNK);
a5ebca47 656 return val;
8d318a50
LW		 657}
658
659static void d40_config_enable_lidx(struct d40_chan *d40c)
660{
661 /* Set LIDX for lcla */
662 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
663 D40_SREG_ELEM_LOG_LIDX_MASK,
664 d40c->base->virtbase + D40_DREG_PCBASE +
665 d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT);
666
667 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
668 D40_SREG_ELEM_LOG_LIDX_MASK,
669 d40c->base->virtbase + D40_DREG_PCBASE +
670 d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT);
671}
672
673static int d40_config_write(struct d40_chan *d40c)
674{
675 u32 addr_base;
676 u32 var;
677 int res;
678
679 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
680 if (res)
681 return res;
682
683 /* Odd addresses are even addresses + 4 */
684 addr_base = (d40c->phy_chan->num % 2) * 4;
685 /* Setup channel mode to logical or physical */
686 var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
687 D40_CHAN_POS(d40c->phy_chan->num);
688 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
689
690 /* Setup operational mode option register */
691 var = ((d40c->dma_cfg.channel_type >> STEDMA40_INFO_CH_MODE_OPT_POS) &
692 0x3) << D40_CHAN_POS(d40c->phy_chan->num);
693
694 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
695
696 if (d40c->log_num != D40_PHY_CHAN) {
697 /* Set default config for CFG reg */
698 writel(d40c->src_def_cfg,
699 d40c->base->virtbase + D40_DREG_PCBASE +
700 d40c->phy_chan->num * D40_DREG_PCDELTA +
701 D40_CHAN_REG_SSCFG);
702 writel(d40c->dst_def_cfg,
703 d40c->base->virtbase + D40_DREG_PCBASE +
704 d40c->phy_chan->num * D40_DREG_PCDELTA +
705 D40_CHAN_REG_SDCFG);
706
707 d40_config_enable_lidx(d40c);
708 }
709 return res;
710}
711
712static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
713{
8d318a50
LW		 714 if (d40d->lli_phy.dst && d40d->lli_phy.src) {
715 d40_phy_lli_write(d40c->base->virtbase,
716 d40c->phy_chan->num,
717 d40d->lli_phy.dst,
718 d40d->lli_phy.src);
8d318a50 719 } else if (d40d->lli_log.dst && d40d->lli_log.src) {
8d318a50
LW		 720 struct d40_log_lli *src = d40d->lli_log.src;
721 struct d40_log_lli *dst = d40d->lli_log.dst;
508849ad 722 int s;
8d318a50 723
941b77a3
PF		 724 src += d40d->lli_count;
725 dst += d40d->lli_count;
508849ad
LW		 726 s = d40_log_lli_write(d40c->lcpa,
727 d40c->lcla.src, d40c->lcla.dst,
728 dst, src,
729 d40c->base->plat_data->llis_per_log);
730
731 /* If s equals to zero, the job is not linked */
732 if (s > 0) {
733 (void) dma_map_single(d40c->base->dev, d40c->lcla.src,
734 s * sizeof(struct d40_log_lli),
735 DMA_TO_DEVICE);
736 (void) dma_map_single(d40c->base->dev, d40c->lcla.dst,
737 s * sizeof(struct d40_log_lli),
738 DMA_TO_DEVICE);
739 }
8d318a50 740 }
941b77a3 741 d40d->lli_count += d40d->lli_tx_len;
8d318a50
LW		 742}
743
744static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
745{
746 struct d40_chan *d40c = container_of(tx->chan,
747 struct d40_chan,
748 chan);
749 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
750 unsigned long flags;
751
752 spin_lock_irqsave(&d40c->lock, flags);
753
754 tx->cookie = d40_assign_cookie(d40c, d40d);
755
756 d40_desc_queue(d40c, d40d);
757
758 spin_unlock_irqrestore(&d40c->lock, flags);
759
760 return tx->cookie;
761}
762
763static int d40_start(struct d40_chan *d40c)
764{
f4185592
LW		 765 if (d40c->base->rev == 0) {
766 int err;
767
768 if (d40c->log_num != D40_PHY_CHAN) {
769 err = d40_channel_execute_command(d40c,
770 D40_DMA_SUSPEND_REQ);
771 if (err)
772 return err;
773 }
774 }
775
0c32269d 776 if (d40c->log_num != D40_PHY_CHAN)
8d318a50 777 d40_config_set_event(d40c, true);
8d318a50 778
0c32269d 779 return d40_channel_execute_command(d40c, D40_DMA_RUN);
8d318a50
LW		 780}
781
782static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
783{
784 struct d40_desc *d40d;
785 int err;
786
787 /* Start queued jobs, if any */
788 d40d = d40_first_queued(d40c);
789
790 if (d40d != NULL) {
791 d40c->busy = true;
792
793 /* Remove from queue */
794 d40_desc_remove(d40d);
795
796 /* Add to active queue */
797 d40_desc_submit(d40c, d40d);
798
799 /* Initiate DMA job */
800 d40_desc_load(d40c, d40d);
801
802 /* Start dma job */
803 err = d40_start(d40c);
804
805 if (err)
806 return NULL;
807 }
808
809 return d40d;
810}
811
812/* called from interrupt context */
813static void dma_tc_handle(struct d40_chan *d40c)
814{
815 struct d40_desc *d40d;
816
817 if (!d40c->phy_chan)
818 return;
819
820 /* Get first active entry from list */
821 d40d = d40_first_active_get(d40c);
822
823 if (d40d == NULL)
824 return;
825
941b77a3 826 if (d40d->lli_count < d40d->lli_len) {
8d318a50
LW		 827
828 d40_desc_load(d40c, d40d);
829 /* Start dma job */
830 (void) d40_start(d40c);
831 return;
832 }
833
834 if (d40_queue_start(d40c) == NULL)
835 d40c->busy = false;
836
837 d40c->pending_tx++;
838 tasklet_schedule(&d40c->tasklet);
839
840}
841
842static void dma_tasklet(unsigned long data)
843{
844 struct d40_chan *d40c = (struct d40_chan *) data;
845 struct d40_desc *d40d_fin;
846 unsigned long flags;
847 dma_async_tx_callback callback;
848 void *callback_param;
849
850 spin_lock_irqsave(&d40c->lock, flags);
851
852 /* Get first active entry from list */
853 d40d_fin = d40_first_active_get(d40c);
854
855 if (d40d_fin == NULL)
856 goto err;
857
858 d40c->completed = d40d_fin->txd.cookie;
859
860 /*
861 * If terminating a channel pending_tx is set to zero.
862 * This prevents any finished active jobs to return to the client.
863 */
864 if (d40c->pending_tx == 0) {
865 spin_unlock_irqrestore(&d40c->lock, flags);
866 return;
867 }
868
869 /* Callback to client */
870 callback = d40d_fin->txd.callback;
871 callback_param = d40d_fin->txd.callback_param;
872
873 if (async_tx_test_ack(&d40d_fin->txd)) {
874 d40_pool_lli_free(d40d_fin);
875 d40_desc_remove(d40d_fin);
876 /* Return desc to free-list */
877 d40_desc_free(d40c, d40d_fin);
878 } else {
8d318a50
LW		 879 if (!d40d_fin->is_in_client_list) {
880 d40_desc_remove(d40d_fin);
881 list_add_tail(&d40d_fin->node, &d40c->client);
882 d40d_fin->is_in_client_list = true;
883 }
884 }
885
886 d40c->pending_tx--;
887
888 if (d40c->pending_tx)
889 tasklet_schedule(&d40c->tasklet);
890
891 spin_unlock_irqrestore(&d40c->lock, flags);
892
893 if (callback)
894 callback(callback_param);
895
896 return;
897
898 err:
899 /* Rescue manouver if receiving double interrupts */
900 if (d40c->pending_tx > 0)
901 d40c->pending_tx--;
902 spin_unlock_irqrestore(&d40c->lock, flags);
903}
904
905static irqreturn_t d40_handle_interrupt(int irq, void *data)
906{
907 static const struct d40_interrupt_lookup il[] = {
908 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
909 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
910 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
911 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
912 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
913 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
914 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
915 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
916 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
917 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
918 };
919
920 int i;
921 u32 regs[ARRAY_SIZE(il)];
922 u32 tmp;
923 u32 idx;
924 u32 row;
925 long chan = -1;
926 struct d40_chan *d40c;
927 unsigned long flags;
928 struct d40_base *base = data;
929
930 spin_lock_irqsave(&base->interrupt_lock, flags);
931
932 /* Read interrupt status of both logical and physical channels */
933 for (i = 0; i < ARRAY_SIZE(il); i++)
934 regs[i] = readl(base->virtbase + il[i].src);
935
936 for (;;) {
937
938 chan = find_next_bit((unsigned long *)regs,
939 BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
940
941 /* No more set bits found? */
942 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
943 break;
944
945 row = chan / BITS_PER_LONG;
946 idx = chan & (BITS_PER_LONG - 1);
947
948 /* ACK interrupt */
949 tmp = readl(base->virtbase + il[row].clr);
950 tmp |= 1 << idx;
951 writel(tmp, base->virtbase + il[row].clr);
952
953 if (il[row].offset == D40_PHY_CHAN)
954 d40c = base->lookup_phy_chans[idx];
955 else
956 d40c = base->lookup_log_chans[il[row].offset + idx];
957 spin_lock(&d40c->lock);
958
959 if (!il[row].is_error)
960 dma_tc_handle(d40c);
961 else
508849ad
LW		 962 dev_err(base->dev,
963 "[%s] IRQ chan: %ld offset %d idx %d\n",
8d318a50
LW		 964 __func__, chan, il[row].offset, idx);
965
966 spin_unlock(&d40c->lock);
967 }
968
969 spin_unlock_irqrestore(&base->interrupt_lock, flags);
970
971 return IRQ_HANDLED;
972}
973
974
975static int d40_validate_conf(struct d40_chan *d40c,
976 struct stedma40_chan_cfg *conf)
977{
978 int res = 0;
979 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
980 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
981 bool is_log = (conf->channel_type & STEDMA40_CHANNEL_IN_OPER_MODE)
982 == STEDMA40_CHANNEL_IN_LOG_MODE;
983
0747c7ba
LW		 984 if (!conf->dir) {
985 dev_err(&d40c->chan.dev->device, "[%s] Invalid direction.\n",
986 __func__);
987 res = -EINVAL;
988 }
989
990 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
991 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
992 d40c->runtime_addr == 0) {
993
994 dev_err(&d40c->chan.dev->device,
995 "[%s] Invalid TX channel address (%d)\n",
996 __func__, conf->dst_dev_type);
997 res = -EINVAL;
998 }
999
1000 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1001 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1002 d40c->runtime_addr == 0) {
1003 dev_err(&d40c->chan.dev->device,
1004 "[%s] Invalid RX channel address (%d)\n",
1005 __func__, conf->src_dev_type);
1006 res = -EINVAL;
1007 }
1008
1009 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
8d318a50
LW		 1010 dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1011 dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
1012 __func__);
1013 res = -EINVAL;
1014 }
1015
0747c7ba 1016 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
8d318a50
LW		 1017 src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1018 dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
1019 __func__);
1020 res = -EINVAL;
1021 }
1022
1023 if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1024 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1025 dev_err(&d40c->chan.dev->device,
1026 "[%s] No event line\n", __func__);
1027 res = -EINVAL;
1028 }
1029
1030 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1031 (src_event_group != dst_event_group)) {
1032 dev_err(&d40c->chan.dev->device,
1033 "[%s] Invalid event group\n", __func__);
1034 res = -EINVAL;
1035 }
1036
1037 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1038 /*
1039 * DMAC HW supports it. Will be added to this driver,
1040 * in case any dma client requires it.
1041 */
1042 dev_err(&d40c->chan.dev->device,
1043 "[%s] periph to periph not supported\n",
1044 __func__);
1045 res = -EINVAL;
1046 }
1047
1048 return res;
1049}
1050
1051static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
4aed79b2 1052 int log_event_line, bool is_log)
8d318a50
LW		 1053{
1054 unsigned long flags;
1055 spin_lock_irqsave(&phy->lock, flags);
4aed79b2 1056 if (!is_log) {
8d318a50
LW		 1057 /* Physical interrupts are masked per physical full channel */
1058 if (phy->allocated_src == D40_ALLOC_FREE &&
1059 phy->allocated_dst == D40_ALLOC_FREE) {
1060 phy->allocated_dst = D40_ALLOC_PHY;
1061 phy->allocated_src = D40_ALLOC_PHY;
1062 goto found;
1063 } else
1064 goto not_found;
1065 }
1066
1067 /* Logical channel */
1068 if (is_src) {
1069 if (phy->allocated_src == D40_ALLOC_PHY)
1070 goto not_found;
1071
1072 if (phy->allocated_src == D40_ALLOC_FREE)
1073 phy->allocated_src = D40_ALLOC_LOG_FREE;
1074
1075 if (!(phy->allocated_src & (1 << log_event_line))) {
1076 phy->allocated_src |= 1 << log_event_line;
1077 goto found;
1078 } else
1079 goto not_found;
1080 } else {
1081 if (phy->allocated_dst == D40_ALLOC_PHY)
1082 goto not_found;
1083
1084 if (phy->allocated_dst == D40_ALLOC_FREE)
1085 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1086
1087 if (!(phy->allocated_dst & (1 << log_event_line))) {
1088 phy->allocated_dst |= 1 << log_event_line;
1089 goto found;
1090 } else
1091 goto not_found;
1092 }
1093
1094not_found:
1095 spin_unlock_irqrestore(&phy->lock, flags);
1096 return false;
1097found:
1098 spin_unlock_irqrestore(&phy->lock, flags);
1099 return true;
1100}
1101
1102static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1103 int log_event_line)
1104{
1105 unsigned long flags;
1106 bool is_free = false;
1107
1108 spin_lock_irqsave(&phy->lock, flags);
1109 if (!log_event_line) {
1110 /* Physical interrupts are masked per physical full channel */
1111 phy->allocated_dst = D40_ALLOC_FREE;
1112 phy->allocated_src = D40_ALLOC_FREE;
1113 is_free = true;
1114 goto out;
1115 }
1116
1117 /* Logical channel */
1118 if (is_src) {
1119 phy->allocated_src &= ~(1 << log_event_line);
1120 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1121 phy->allocated_src = D40_ALLOC_FREE;
1122 } else {
1123 phy->allocated_dst &= ~(1 << log_event_line);
1124 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1125 phy->allocated_dst = D40_ALLOC_FREE;
1126 }
1127
1128 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1129 D40_ALLOC_FREE);
1130
1131out:
1132 spin_unlock_irqrestore(&phy->lock, flags);
1133
1134 return is_free;
1135}
1136
1137static int d40_allocate_channel(struct d40_chan *d40c)
1138{
1139 int dev_type;
1140 int event_group;
1141 int event_line;
1142 struct d40_phy_res *phys;
1143 int i;
1144 int j;
1145 int log_num;
1146 bool is_src;
508849ad
LW		 1147 bool is_log = (d40c->dma_cfg.channel_type &
1148 STEDMA40_CHANNEL_IN_OPER_MODE)
8d318a50
LW		 1149 == STEDMA40_CHANNEL_IN_LOG_MODE;
1150
1151
1152 phys = d40c->base->phy_res;
1153
1154 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1155 dev_type = d40c->dma_cfg.src_dev_type;
1156 log_num = 2 * dev_type;
1157 is_src = true;
1158 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1159 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1160 /* dst event lines are used for logical memcpy */
1161 dev_type = d40c->dma_cfg.dst_dev_type;
1162 log_num = 2 * dev_type + 1;
1163 is_src = false;
1164 } else
1165 return -EINVAL;
1166
1167 event_group = D40_TYPE_TO_GROUP(dev_type);
1168 event_line = D40_TYPE_TO_EVENT(dev_type);
1169
1170 if (!is_log) {
1171 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1172 /* Find physical half channel */
1173 for (i = 0; i < d40c->base->num_phy_chans; i++) {
1174
4aed79b2
MM		 1175 if (d40_alloc_mask_set(&phys[i], is_src,
1176 0, is_log))
8d318a50
LW		 1177 goto found_phy;
1178 }
1179 } else
1180 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1181 int phy_num = j + event_group * 2;
1182 for (i = phy_num; i < phy_num + 2; i++) {
508849ad
LW		 1183 if (d40_alloc_mask_set(&phys[i],
1184 is_src,
1185 0,
1186 is_log))
8d318a50
LW		 1187 goto found_phy;
1188 }
1189 }
1190 return -EINVAL;
1191found_phy:
1192 d40c->phy_chan = &phys[i];
1193 d40c->log_num = D40_PHY_CHAN;
1194 goto out;
1195 }
1196 if (dev_type == -1)
1197 return -EINVAL;
1198
1199 /* Find logical channel */
1200 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1201 int phy_num = j + event_group * 2;
1202 /*
1203 * Spread logical channels across all available physical rather
1204 * than pack every logical channel at the first available phy
1205 * channels.
1206 */
1207 if (is_src) {
1208 for (i = phy_num; i < phy_num + 2; i++) {
1209 if (d40_alloc_mask_set(&phys[i], is_src,
4aed79b2 1210 event_line, is_log))
8d318a50
LW		 1211 goto found_log;
1212 }
1213 } else {
1214 for (i = phy_num + 1; i >= phy_num; i--) {
1215 if (d40_alloc_mask_set(&phys[i], is_src,
4aed79b2 1216 event_line, is_log))
8d318a50
LW		 1217 goto found_log;
1218 }
1219 }
1220 }
1221 return -EINVAL;
1222
1223found_log:
1224 d40c->phy_chan = &phys[i];
1225 d40c->log_num = log_num;
1226out:
1227
1228 if (is_log)
1229 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1230 else
1231 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1232
1233 return 0;
1234
1235}
1236
8d318a50
LW		 1237static int d40_config_memcpy(struct d40_chan *d40c)
1238{
1239 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1240
1241 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1242 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1243 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1244 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1245 memcpy[d40c->chan.chan_id];
1246
1247 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1248 dma_has_cap(DMA_SLAVE, cap)) {
1249 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1250 } else {
1251 dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
1252 __func__);
1253 return -EINVAL;
1254 }
1255
1256 return 0;
1257}
1258
1259
1260static int d40_free_dma(struct d40_chan *d40c)
1261{
1262
1263 int res = 0;
d181b3a8 1264 u32 event;
8d318a50
LW		 1265 struct d40_phy_res *phy = d40c->phy_chan;
1266 bool is_src;
a8be8627
PF		 1267 struct d40_desc *d;
1268 struct d40_desc *_d;
1269
8d318a50
LW		 1270
1271 /* Terminate all queued and active transfers */
1272 d40_term_all(d40c);
1273
a8be8627
PF		 1274 /* Release client owned descriptors */
1275 if (!list_empty(&d40c->client))
1276 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1277 d40_pool_lli_free(d);
1278 d40_desc_remove(d);
1279 /* Return desc to free-list */
1280 d40_desc_free(d40c, d);
1281 }
1282
8d318a50
LW		 1283 if (phy == NULL) {
1284 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
1285 __func__);
1286 return -EINVAL;
1287 }
1288
1289 if (phy->allocated_src == D40_ALLOC_FREE &&
1290 phy->allocated_dst == D40_ALLOC_FREE) {
1291 dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
1292 __func__);
1293 return -EINVAL;
1294 }
1295
8d318a50
LW		 1296 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1297 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1298 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
8d318a50
LW		 1299 is_src = false;
1300 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1301 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
8d318a50
LW		 1302 is_src = true;
1303 } else {
1304 dev_err(&d40c->chan.dev->device,
1305 "[%s] Unknown direction\n", __func__);
1306 return -EINVAL;
1307 }
1308
d181b3a8
JA		 1309 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1310 if (res) {
1311 dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n",
1312 __func__);
1313 return res;
1314 }
1315
8d318a50 1316 if (d40c->log_num != D40_PHY_CHAN) {
d181b3a8 1317 /* Release logical channel, deactivate the event line */
8d318a50 1318
d181b3a8 1319 d40_config_set_event(d40c, false);
8d318a50
LW		 1320 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1321
1322 /*
1323 * Check if there are more logical allocation
1324 * on this phy channel.
1325 */
1326 if (!d40_alloc_mask_free(phy, is_src, event)) {
1327 /* Resume the other logical channels if any */
1328 if (d40_chan_has_events(d40c)) {
1329 res = d40_channel_execute_command(d40c,
1330 D40_DMA_RUN);
1331 if (res) {
1332 dev_err(&d40c->chan.dev->device,
1333 "[%s] Executing RUN command\n",
1334 __func__);
1335 return res;
1336 }
1337 }
1338 return 0;
1339 }
d181b3a8
JA		 1340 } else {
1341 (void) d40_alloc_mask_free(phy, is_src, 0);
1342 }
8d318a50
LW		 1343
1344 /* Release physical channel */
1345 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1346 if (res) {
1347 dev_err(&d40c->chan.dev->device,
1348 "[%s] Failed to stop channel\n", __func__);
1349 return res;
1350 }
1351 d40c->phy_chan = NULL;
1352 /* Invalidate channel type */
1353 d40c->dma_cfg.channel_type = 0;
1354 d40c->base->lookup_phy_chans[phy->num] = NULL;
1355
1356 return 0;
8d318a50
LW		 1357}
1358
1359static int d40_pause(struct dma_chan *chan)
1360{
1361 struct d40_chan *d40c =
1362 container_of(chan, struct d40_chan, chan);
1363 int res;
8d318a50
LW		 1364 unsigned long flags;
1365
1366 spin_lock_irqsave(&d40c->lock, flags);
1367
1368 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1369 if (res == 0) {
1370 if (d40c->log_num != D40_PHY_CHAN) {
1371 d40_config_set_event(d40c, false);
1372 /* Resume the other logical channels if any */
1373 if (d40_chan_has_events(d40c))
1374 res = d40_channel_execute_command(d40c,
1375 D40_DMA_RUN);
1376 }
1377 }
1378
1379 spin_unlock_irqrestore(&d40c->lock, flags);
1380 return res;
1381}
1382
a5ebca47
JA		 1383static bool d40_is_paused(struct d40_chan *d40c)
1384{
1385 bool is_paused = false;
1386 unsigned long flags;
1387 void __iomem *active_reg;
1388 u32 status;
1389 u32 event;
a5ebca47
JA		 1390
1391 spin_lock_irqsave(&d40c->lock, flags);
1392
1393 if (d40c->log_num == D40_PHY_CHAN) {
1394 if (d40c->phy_chan->num % 2 == 0)
1395 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1396 else
1397 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1398
1399 status = (readl(active_reg) &
1400 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1401 D40_CHAN_POS(d40c->phy_chan->num);
1402 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1403 is_paused = true;
1404
1405 goto _exit;
1406 }
1407
a5ebca47
JA		 1408 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1409 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM)
1410 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1411 else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1412 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1413 else {
1414 dev_err(&d40c->chan.dev->device,
1415 "[%s] Unknown direction\n", __func__);
1416 goto _exit;
1417 }
1418 status = d40_chan_has_events(d40c);
1419 status = (status & D40_EVENTLINE_MASK(event)) >>
1420 D40_EVENTLINE_POS(event);
1421
1422 if (status != D40_DMA_RUN)
1423 is_paused = true;
a5ebca47
JA		 1424_exit:
1425 spin_unlock_irqrestore(&d40c->lock, flags);
1426 return is_paused;
1427
1428}
1429
1430
8d318a50
LW		 1431static bool d40_tx_is_linked(struct d40_chan *d40c)
1432{
1433 bool is_link;
1434
1435 if (d40c->log_num != D40_PHY_CHAN)
1436 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
1437 else
1438 is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1439 d40c->phy_chan->num * D40_DREG_PCDELTA +
1440 D40_CHAN_REG_SDLNK) &
1441 D40_SREG_LNK_PHYS_LNK_MASK;
1442 return is_link;
1443}
1444
1445static u32 d40_residue(struct d40_chan *d40c)
1446{
1447 u32 num_elt;
1448
1449 if (d40c->log_num != D40_PHY_CHAN)
508849ad 1450 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
8d318a50
LW		 1451 >> D40_MEM_LCSP2_ECNT_POS;
1452 else
1453 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
1454 d40c->phy_chan->num * D40_DREG_PCDELTA +
1455 D40_CHAN_REG_SDELT) &
508849ad
LW		 1456 D40_SREG_ELEM_PHY_ECNT_MASK) >>
1457 D40_SREG_ELEM_PHY_ECNT_POS;
8d318a50
LW		 1458 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1459}
1460
1461static int d40_resume(struct dma_chan *chan)
1462{
1463 struct d40_chan *d40c =
1464 container_of(chan, struct d40_chan, chan);
1465 int res = 0;
1466 unsigned long flags;
1467
1468 spin_lock_irqsave(&d40c->lock, flags);
1469
f4185592
LW		 1470 if (d40c->base->rev == 0)
1471 if (d40c->log_num != D40_PHY_CHAN) {
1472 res = d40_channel_execute_command(d40c,
1473 D40_DMA_SUSPEND_REQ);
1474 goto no_suspend;
1475 }
1476
0c32269d
JA		 1477 /* If bytes left to transfer or linked tx resume job */
1478 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
1479 if (d40c->log_num != D40_PHY_CHAN)
8d318a50 1480 d40_config_set_event(d40c, true);
8d318a50 1481 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
0c32269d 1482 }
8d318a50 1483
f4185592 1484no_suspend:
8d318a50
LW		 1485 spin_unlock_irqrestore(&d40c->lock, flags);
1486 return res;
1487}
1488
1489static u32 stedma40_residue(struct dma_chan *chan)
1490{
1491 struct d40_chan *d40c =
1492 container_of(chan, struct d40_chan, chan);
1493 u32 bytes_left;
1494 unsigned long flags;
1495
1496 spin_lock_irqsave(&d40c->lock, flags);
1497 bytes_left = d40_residue(d40c);
1498 spin_unlock_irqrestore(&d40c->lock, flags);
1499
1500 return bytes_left;
1501}
1502
1503/* Public DMA functions in addition to the DMA engine framework */
1504
1505int stedma40_set_psize(struct dma_chan *chan,
1506 int src_psize,
1507 int dst_psize)
1508{
1509 struct d40_chan *d40c =
1510 container_of(chan, struct d40_chan, chan);
1511 unsigned long flags;
1512
1513 spin_lock_irqsave(&d40c->lock, flags);
1514
1515 if (d40c->log_num != D40_PHY_CHAN) {
1516 d40c->log_def.lcsp1 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
1517 d40c->log_def.lcsp3 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
508849ad
LW		 1518 d40c->log_def.lcsp1 |= src_psize <<
1519 D40_MEM_LCSP1_SCFG_PSIZE_POS;
1520 d40c->log_def.lcsp3 |= dst_psize <<
1521 D40_MEM_LCSP1_SCFG_PSIZE_POS;
8d318a50
LW		 1522 goto out;
1523 }
1524
1525 if (src_psize == STEDMA40_PSIZE_PHY_1)
1526 d40c->src_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS);
1527 else {
1528 d40c->src_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS;
1529 d40c->src_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 <<
1530 D40_SREG_CFG_PSIZE_POS);
1531 d40c->src_def_cfg |= src_psize << D40_SREG_CFG_PSIZE_POS;
1532 }
1533
1534 if (dst_psize == STEDMA40_PSIZE_PHY_1)
1535 d40c->dst_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS);
1536 else {
1537 d40c->dst_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS;
1538 d40c->dst_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 <<
1539 D40_SREG_CFG_PSIZE_POS);
1540 d40c->dst_def_cfg |= dst_psize << D40_SREG_CFG_PSIZE_POS;
1541 }
1542out:
1543 spin_unlock_irqrestore(&d40c->lock, flags);
1544 return 0;
1545}
1546EXPORT_SYMBOL(stedma40_set_psize);
1547
1548struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1549 struct scatterlist *sgl_dst,
1550 struct scatterlist *sgl_src,
1551 unsigned int sgl_len,
2a614340 1552 unsigned long dma_flags)
8d318a50
LW		 1553{
1554 int res;
1555 struct d40_desc *d40d;
1556 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1557 chan);
2a614340 1558 unsigned long flags;
8d318a50 1559
0d0f6b8b
JA		 1560 if (d40c->phy_chan == NULL) {
1561 dev_err(&d40c->chan.dev->device,
1562 "[%s] Unallocated channel.\n", __func__);
1563 return ERR_PTR(-EINVAL);
1564 }
1565
2a614340 1566 spin_lock_irqsave(&d40c->lock, flags);
8d318a50
LW		 1567 d40d = d40_desc_get(d40c);
1568
1569 if (d40d == NULL)
1570 goto err;
1571
8d318a50 1572 d40d->lli_len = sgl_len;
941b77a3 1573 d40d->lli_tx_len = d40d->lli_len;
2a614340 1574 d40d->txd.flags = dma_flags;
8d318a50
LW		 1575
1576 if (d40c->log_num != D40_PHY_CHAN) {
941b77a3
PF		 1577 if (d40d->lli_len > d40c->base->plat_data->llis_per_log)
1578 d40d->lli_tx_len = d40c->base->plat_data->llis_per_log;
1579
8d318a50
LW		 1580 if (sgl_len > 1)
1581 /*
1582 * Check if there is space available in lcla. If not,
1583 * split list into 1-length and run only in lcpa
1584 * space.
1585 */
508849ad 1586 if (d40_lcla_id_get(d40c) != 0)
941b77a3 1587 d40d->lli_tx_len = 1;
8d318a50
LW		 1588
1589 if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) {
1590 dev_err(&d40c->chan.dev->device,
1591 "[%s] Out of memory\n", __func__);
1592 goto err;
1593 }
1594
1595 (void) d40_log_sg_to_lli(d40c->lcla.src_id,
1596 sgl_src,
1597 sgl_len,
1598 d40d->lli_log.src,
1599 d40c->log_def.lcsp1,
1600 d40c->dma_cfg.src_info.data_width,
2a614340 1601 dma_flags & DMA_PREP_INTERRUPT,
941b77a3 1602 d40d->lli_tx_len,
8d318a50
LW		 1603 d40c->base->plat_data->llis_per_log);
1604
1605 (void) d40_log_sg_to_lli(d40c->lcla.dst_id,
1606 sgl_dst,
1607 sgl_len,
1608 d40d->lli_log.dst,
1609 d40c->log_def.lcsp3,
1610 d40c->dma_cfg.dst_info.data_width,
2a614340 1611 dma_flags & DMA_PREP_INTERRUPT,
941b77a3 1612 d40d->lli_tx_len,
8d318a50
LW		 1613 d40c->base->plat_data->llis_per_log);
1614
1615
1616 } else {
1617 if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
1618 dev_err(&d40c->chan.dev->device,
1619 "[%s] Out of memory\n", __func__);
1620 goto err;
1621 }
1622
1623 res = d40_phy_sg_to_lli(sgl_src,
1624 sgl_len,
1625 0,
1626 d40d->lli_phy.src,
1627 d40d->lli_phy.src_addr,
1628 d40c->src_def_cfg,
1629 d40c->dma_cfg.src_info.data_width,
1630 d40c->dma_cfg.src_info.psize,
1631 true);
1632
1633 if (res < 0)
1634 goto err;
1635
1636 res = d40_phy_sg_to_lli(sgl_dst,
1637 sgl_len,
1638 0,
1639 d40d->lli_phy.dst,
1640 d40d->lli_phy.dst_addr,
1641 d40c->dst_def_cfg,
1642 d40c->dma_cfg.dst_info.data_width,
1643 d40c->dma_cfg.dst_info.psize,
1644 true);
1645
1646 if (res < 0)
1647 goto err;
1648
1649 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1650 d40d->lli_pool.size, DMA_TO_DEVICE);
1651 }
1652
1653 dma_async_tx_descriptor_init(&d40d->txd, chan);
1654
1655 d40d->txd.tx_submit = d40_tx_submit;
1656
2a614340 1657 spin_unlock_irqrestore(&d40c->lock, flags);
8d318a50
LW		 1658
1659 return &d40d->txd;
1660err:
2a614340 1661 spin_unlock_irqrestore(&d40c->lock, flags);
8d318a50
LW		 1662 return NULL;
1663}
1664EXPORT_SYMBOL(stedma40_memcpy_sg);
1665
1666bool stedma40_filter(struct dma_chan *chan, void *data)
1667{
1668 struct stedma40_chan_cfg *info = data;
1669 struct d40_chan *d40c =
1670 container_of(chan, struct d40_chan, chan);
1671 int err;
1672
1673 if (data) {
1674 err = d40_validate_conf(d40c, info);
1675 if (!err)
1676 d40c->dma_cfg = *info;
1677 } else
1678 err = d40_config_memcpy(d40c);
1679
1680 return err == 0;
1681}
1682EXPORT_SYMBOL(stedma40_filter);
1683
1684/* DMA ENGINE functions */
1685static int d40_alloc_chan_resources(struct dma_chan *chan)
1686{
1687 int err;
1688 unsigned long flags;
1689 struct d40_chan *d40c =
1690 container_of(chan, struct d40_chan, chan);
ef1872ec 1691 bool is_free_phy;
8d318a50
LW		 1692 spin_lock_irqsave(&d40c->lock, flags);
1693
1694 d40c->completed = chan->cookie = 1;
1695
1696 /*
1697 * If no dma configuration is set (channel_type == 0)
ef1872ec 1698 * use default configuration (memcpy)
8d318a50
LW		 1699 */
1700 if (d40c->dma_cfg.channel_type == 0) {
1701 err = d40_config_memcpy(d40c);
ff0b12ba
JA		 1702 if (err) {
1703 dev_err(&d40c->chan.dev->device,
1704 "[%s] Failed to configure memcpy channel\n",
1705 __func__);
1706 goto fail;
1707 }
8d318a50 1708 }
ef1872ec 1709 is_free_phy = (d40c->phy_chan == NULL);
8d318a50
LW		 1710
1711 err = d40_allocate_channel(d40c);
1712 if (err) {
1713 dev_err(&d40c->chan.dev->device,
1714 "[%s] Failed to allocate channel\n", __func__);
ff0b12ba 1715 goto fail;
8d318a50
LW		 1716 }
1717
ef1872ec
LW		 1718 /* Fill in basic CFG register values */
1719 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1720 &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1721
1722 if (d40c->log_num != D40_PHY_CHAN) {
1723 d40_log_cfg(&d40c->dma_cfg,
1724 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1725
1726 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1727 d40c->lcpa = d40c->base->lcpa_base +
1728 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1729 else
1730 d40c->lcpa = d40c->base->lcpa_base +
1731 d40c->dma_cfg.dst_dev_type *
1732 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1733 }
1734
1735 /*
1736 * Only write channel configuration to the DMA if the physical
1737 * resource is free. In case of multiple logical channels
1738 * on the same physical resource, only the first write is necessary.
1739 */
1740 if (is_free_phy) {
1741 err = d40_config_write(d40c);
1742 if (err) {
1743 dev_err(&d40c->chan.dev->device,
1744 "[%s] Failed to configure channel\n",
1745 __func__);
1746 }
8d318a50 1747 }
ff0b12ba 1748fail:
8d318a50 1749 spin_unlock_irqrestore(&d40c->lock, flags);
ff0b12ba 1750 return err;
8d318a50
LW		 1751}
1752
1753static void d40_free_chan_resources(struct dma_chan *chan)
1754{
1755 struct d40_chan *d40c =
1756 container_of(chan, struct d40_chan, chan);
1757 int err;
1758 unsigned long flags;
1759
0d0f6b8b
JA		 1760 if (d40c->phy_chan == NULL) {
1761 dev_err(&d40c->chan.dev->device,
1762 "[%s] Cannot free unallocated channel\n", __func__);
1763 return;
1764 }
1765
1766
8d318a50
LW		 1767 spin_lock_irqsave(&d40c->lock, flags);
1768
1769 err = d40_free_dma(d40c);
1770
1771 if (err)
1772 dev_err(&d40c->chan.dev->device,
1773 "[%s] Failed to free channel\n", __func__);
1774 spin_unlock_irqrestore(&d40c->lock, flags);
1775}
1776
1777static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1778 dma_addr_t dst,
1779 dma_addr_t src,
1780 size_t size,
2a614340 1781 unsigned long dma_flags)
8d318a50
LW		 1782{
1783 struct d40_desc *d40d;
1784 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1785 chan);
2a614340 1786 unsigned long flags;
8d318a50
LW		 1787 int err = 0;
1788
0d0f6b8b
JA		 1789 if (d40c->phy_chan == NULL) {
1790 dev_err(&d40c->chan.dev->device,
1791 "[%s] Channel is not allocated.\n", __func__);
1792 return ERR_PTR(-EINVAL);
1793 }
1794
2a614340 1795 spin_lock_irqsave(&d40c->lock, flags);
8d318a50
LW		 1796 d40d = d40_desc_get(d40c);
1797
1798 if (d40d == NULL) {
1799 dev_err(&d40c->chan.dev->device,
1800 "[%s] Descriptor is NULL\n", __func__);
1801 goto err;
1802 }
1803
2a614340 1804 d40d->txd.flags = dma_flags;
8d318a50
LW		 1805
1806 dma_async_tx_descriptor_init(&d40d->txd, chan);
1807
1808 d40d->txd.tx_submit = d40_tx_submit;
1809
1810 if (d40c->log_num != D40_PHY_CHAN) {
1811
1812 if (d40_pool_lli_alloc(d40d, 1, true) < 0) {
1813 dev_err(&d40c->chan.dev->device,
1814 "[%s] Out of memory\n", __func__);
1815 goto err;
1816 }
1817 d40d->lli_len = 1;
941b77a3 1818 d40d->lli_tx_len = 1;
8d318a50
LW		 1819
1820 d40_log_fill_lli(d40d->lli_log.src,
1821 src,
1822 size,
1823 0,
1824 d40c->log_def.lcsp1,
1825 d40c->dma_cfg.src_info.data_width,
2123a61e 1826 false, true);
8d318a50
LW		 1827
1828 d40_log_fill_lli(d40d->lli_log.dst,
1829 dst,
1830 size,
1831 0,
1832 d40c->log_def.lcsp3,
1833 d40c->dma_cfg.dst_info.data_width,
1834 true, true);
1835
1836 } else {
1837
1838 if (d40_pool_lli_alloc(d40d, 1, false) < 0) {
1839 dev_err(&d40c->chan.dev->device,
1840 "[%s] Out of memory\n", __func__);
1841 goto err;
1842 }
1843
1844 err = d40_phy_fill_lli(d40d->lli_phy.src,
1845 src,
1846 size,
1847 d40c->dma_cfg.src_info.psize,
1848 0,
1849 d40c->src_def_cfg,
1850 true,
1851 d40c->dma_cfg.src_info.data_width,
1852 false);
1853 if (err)
1854 goto err_fill_lli;
1855
1856 err = d40_phy_fill_lli(d40d->lli_phy.dst,
1857 dst,
1858 size,
1859 d40c->dma_cfg.dst_info.psize,
1860 0,
1861 d40c->dst_def_cfg,
1862 true,
1863 d40c->dma_cfg.dst_info.data_width,
1864 false);
1865
1866 if (err)
1867 goto err_fill_lli;
1868
1869 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1870 d40d->lli_pool.size, DMA_TO_DEVICE);
1871 }
1872
2a614340 1873 spin_unlock_irqrestore(&d40c->lock, flags);
8d318a50
LW		 1874 return &d40d->txd;
1875
1876err_fill_lli:
1877 dev_err(&d40c->chan.dev->device,
1878 "[%s] Failed filling in PHY LLI\n", __func__);
1879 d40_pool_lli_free(d40d);
1880err:
2a614340 1881 spin_unlock_irqrestore(&d40c->lock, flags);
8d318a50
LW		 1882 return NULL;
1883}
1884
1885static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1886 struct d40_chan *d40c,
1887 struct scatterlist *sgl,
1888 unsigned int sg_len,
1889 enum dma_data_direction direction,
2a614340 1890 unsigned long dma_flags)
8d318a50
LW		 1891{
1892 dma_addr_t dev_addr = 0;
1893 int total_size;
8d318a50
LW		 1894
1895 if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) {
1896 dev_err(&d40c->chan.dev->device,
1897 "[%s] Out of memory\n", __func__);
1898 return -ENOMEM;
1899 }
1900
1901 d40d->lli_len = sg_len;
941b77a3
PF		 1902 if (d40d->lli_len <= d40c->base->plat_data->llis_per_log)
1903 d40d->lli_tx_len = d40d->lli_len;
1904 else
1905 d40d->lli_tx_len = d40c->base->plat_data->llis_per_log;
8d318a50
LW		 1906
1907 if (sg_len > 1)
1908 /*
1909 * Check if there is space available in lcla.
1910 * If not, split list into 1-length and run only
1911 * in lcpa space.
1912 */
508849ad 1913 if (d40_lcla_id_get(d40c) != 0)
941b77a3 1914 d40d->lli_tx_len = 1;
8d318a50 1915
2a614340 1916 if (direction == DMA_FROM_DEVICE)
95e1400f
LW		 1917 if (d40c->runtime_addr)
1918 dev_addr = d40c->runtime_addr;
1919 else
1920 dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2a614340 1921 else if (direction == DMA_TO_DEVICE)
95e1400f
LW		 1922 if (d40c->runtime_addr)
1923 dev_addr = d40c->runtime_addr;
1924 else
1925 dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1926
2a614340 1927 else
8d318a50 1928 return -EINVAL;
2a614340
JA		 1929
1930 total_size = d40_log_sg_to_dev(&d40c->lcla,
1931 sgl, sg_len,
1932 &d40d->lli_log,
1933 &d40c->log_def,
1934 d40c->dma_cfg.src_info.data_width,
1935 d40c->dma_cfg.dst_info.data_width,
1936 direction,
1937 dma_flags & DMA_PREP_INTERRUPT,
1938 dev_addr, d40d->lli_tx_len,
1939 d40c->base->plat_data->llis_per_log);
1940
8d318a50
LW		 1941 if (total_size < 0)
1942 return -EINVAL;
1943
1944 return 0;
1945}
1946
1947static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1948 struct d40_chan *d40c,
1949 struct scatterlist *sgl,
1950 unsigned int sgl_len,
1951 enum dma_data_direction direction,
2a614340 1952 unsigned long dma_flags)
8d318a50
LW		 1953{
1954 dma_addr_t src_dev_addr;
1955 dma_addr_t dst_dev_addr;
1956 int res;
1957
1958 if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
1959 dev_err(&d40c->chan.dev->device,
1960 "[%s] Out of memory\n", __func__);
1961 return -ENOMEM;
1962 }
1963
1964 d40d->lli_len = sgl_len;
941b77a3 1965 d40d->lli_tx_len = sgl_len;
8d318a50
LW		 1966
1967 if (direction == DMA_FROM_DEVICE) {
1968 dst_dev_addr = 0;
95e1400f
LW		 1969 if (d40c->runtime_addr)
1970 src_dev_addr = d40c->runtime_addr;
1971 else
1972 src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
8d318a50 1973 } else if (direction == DMA_TO_DEVICE) {
95e1400f
LW		 1974 if (d40c->runtime_addr)
1975 dst_dev_addr = d40c->runtime_addr;
1976 else
1977 dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
8d318a50
LW		 1978 src_dev_addr = 0;
1979 } else
1980 return -EINVAL;
1981
1982 res = d40_phy_sg_to_lli(sgl,
1983 sgl_len,
1984 src_dev_addr,
1985 d40d->lli_phy.src,
1986 d40d->lli_phy.src_addr,
1987 d40c->src_def_cfg,
1988 d40c->dma_cfg.src_info.data_width,
1989 d40c->dma_cfg.src_info.psize,
1990 true);
1991 if (res < 0)
1992 return res;
1993
1994 res = d40_phy_sg_to_lli(sgl,
1995 sgl_len,
1996 dst_dev_addr,
1997 d40d->lli_phy.dst,
1998 d40d->lli_phy.dst_addr,
1999 d40c->dst_def_cfg,
2000 d40c->dma_cfg.dst_info.data_width,
2001 d40c->dma_cfg.dst_info.psize,
2002 true);
2003 if (res < 0)
2004 return res;
2005
2006 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2007 d40d->lli_pool.size, DMA_TO_DEVICE);
2008 return 0;
2009}
2010
2011static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2012 struct scatterlist *sgl,
2013 unsigned int sg_len,
2014 enum dma_data_direction direction,
2a614340 2015 unsigned long dma_flags)
8d318a50
LW		 2016{
2017 struct d40_desc *d40d;
2018 struct d40_chan *d40c = container_of(chan, struct d40_chan,
2019 chan);
2a614340 2020 unsigned long flags;
8d318a50
LW		 2021 int err;
2022
0d0f6b8b
JA		 2023 if (d40c->phy_chan == NULL) {
2024 dev_err(&d40c->chan.dev->device,
2025 "[%s] Cannot prepare unallocated channel\n", __func__);
2026 return ERR_PTR(-EINVAL);
2027 }
2028
8d318a50
LW		 2029 if (d40c->dma_cfg.pre_transfer)
2030 d40c->dma_cfg.pre_transfer(chan,
2031 d40c->dma_cfg.pre_transfer_data,
2032 sg_dma_len(sgl));
2033
2a614340 2034 spin_lock_irqsave(&d40c->lock, flags);
8d318a50 2035 d40d = d40_desc_get(d40c);
2a614340 2036 spin_unlock_irqrestore(&d40c->lock, flags);
8d318a50
LW		 2037
2038 if (d40d == NULL)
2039 return NULL;
2040
8d318a50
LW		 2041 if (d40c->log_num != D40_PHY_CHAN)
2042 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2a614340 2043 direction, dma_flags);
8d318a50
LW		 2044 else
2045 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2a614340 2046 direction, dma_flags);
8d318a50
LW		 2047 if (err) {
2048 dev_err(&d40c->chan.dev->device,
2049 "[%s] Failed to prepare %s slave sg job: %d\n",
2050 __func__,
2051 d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
2052 return NULL;
2053 }
2054
2a614340 2055 d40d->txd.flags = dma_flags;
8d318a50
LW		 2056
2057 dma_async_tx_descriptor_init(&d40d->txd, chan);
2058
2059 d40d->txd.tx_submit = d40_tx_submit;
2060
2061 return &d40d->txd;
2062}
2063
2064static enum dma_status d40_tx_status(struct dma_chan *chan,
2065 dma_cookie_t cookie,
2066 struct dma_tx_state *txstate)
2067{
2068 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2069 dma_cookie_t last_used;
2070 dma_cookie_t last_complete;
2071 int ret;
2072
0d0f6b8b
JA		 2073 if (d40c->phy_chan == NULL) {
2074 dev_err(&d40c->chan.dev->device,
2075 "[%s] Cannot read status of unallocated channel\n",
2076 __func__);
2077 return -EINVAL;
2078 }
2079
8d318a50
LW		 2080 last_complete = d40c->completed;
2081 last_used = chan->cookie;
2082
a5ebca47
JA		 2083 if (d40_is_paused(d40c))
2084 ret = DMA_PAUSED;
2085 else
2086 ret = dma_async_is_complete(cookie, last_complete, last_used);
8d318a50 2087
a5ebca47
JA		 2088 dma_set_tx_state(txstate, last_complete, last_used,
2089 stedma40_residue(chan));
8d318a50
LW		 2090
2091 return ret;
2092}
2093
2094static void d40_issue_pending(struct dma_chan *chan)
2095{
2096 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2097 unsigned long flags;
2098
0d0f6b8b
JA		 2099 if (d40c->phy_chan == NULL) {
2100 dev_err(&d40c->chan.dev->device,
2101 "[%s] Channel is not allocated!\n", __func__);
2102 return;
2103 }
2104
8d318a50
LW		 2105 spin_lock_irqsave(&d40c->lock, flags);
2106
2107 /* Busy means that pending jobs are already being processed */
2108 if (!d40c->busy)
2109 (void) d40_queue_start(d40c);
2110
2111 spin_unlock_irqrestore(&d40c->lock, flags);
2112}
2113
95e1400f
LW		 2114/* Runtime reconfiguration extension */
2115static void d40_set_runtime_config(struct dma_chan *chan,
2116 struct dma_slave_config *config)
2117{
2118 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2119 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2120 enum dma_slave_buswidth config_addr_width;
2121 dma_addr_t config_addr;
2122 u32 config_maxburst;
2123 enum stedma40_periph_data_width addr_width;
2124 int psize;
2125
2126 if (config->direction == DMA_FROM_DEVICE) {
2127 dma_addr_t dev_addr_rx =
2128 d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2129
2130 config_addr = config->src_addr;
2131 if (dev_addr_rx)
2132 dev_dbg(d40c->base->dev,
2133 "channel has a pre-wired RX address %08x "
2134 "overriding with %08x\n",
2135 dev_addr_rx, config_addr);
2136 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2137 dev_dbg(d40c->base->dev,
2138 "channel was not configured for peripheral "
2139 "to memory transfer (%d) overriding\n",
2140 cfg->dir);
2141 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2142
2143 config_addr_width = config->src_addr_width;
2144 config_maxburst = config->src_maxburst;
2145
2146 } else if (config->direction == DMA_TO_DEVICE) {
2147 dma_addr_t dev_addr_tx =
2148 d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2149
2150 config_addr = config->dst_addr;
2151 if (dev_addr_tx)
2152 dev_dbg(d40c->base->dev,
2153 "channel has a pre-wired TX address %08x "
2154 "overriding with %08x\n",
2155 dev_addr_tx, config_addr);
2156 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2157 dev_dbg(d40c->base->dev,
2158 "channel was not configured for memory "
2159 "to peripheral transfer (%d) overriding\n",
2160 cfg->dir);
2161 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2162
2163 config_addr_width = config->dst_addr_width;
2164 config_maxburst = config->dst_maxburst;
2165
2166 } else {
2167 dev_err(d40c->base->dev,
2168 "unrecognized channel direction %d\n",
2169 config->direction);
2170 return;
2171 }
2172
2173 switch (config_addr_width) {
2174 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2175 addr_width = STEDMA40_BYTE_WIDTH;
2176 break;
2177 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2178 addr_width = STEDMA40_HALFWORD_WIDTH;
2179 break;
2180 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2181 addr_width = STEDMA40_WORD_WIDTH;
2182 break;
2183 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2184 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2185 break;
2186 default:
2187 dev_err(d40c->base->dev,
2188 "illegal peripheral address width "
2189 "requested (%d)\n",
2190 config->src_addr_width);
2191 return;
2192 }
2193
2194 if (config_maxburst >= 16)
2195 psize = STEDMA40_PSIZE_LOG_16;
2196 else if (config_maxburst >= 8)
2197 psize = STEDMA40_PSIZE_LOG_8;
2198 else if (config_maxburst >= 4)
2199 psize = STEDMA40_PSIZE_LOG_4;
2200 else
2201 psize = STEDMA40_PSIZE_LOG_1;
2202
2203 /* Set up all the endpoint configs */
2204 cfg->src_info.data_width = addr_width;
2205 cfg->src_info.psize = psize;
2206 cfg->src_info.endianess = STEDMA40_LITTLE_ENDIAN;
2207 cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2208 cfg->dst_info.data_width = addr_width;
2209 cfg->dst_info.psize = psize;
2210 cfg->dst_info.endianess = STEDMA40_LITTLE_ENDIAN;
2211 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2212
2213 /* These settings will take precedence later */
2214 d40c->runtime_addr = config_addr;
2215 d40c->runtime_direction = config->direction;
2216 dev_dbg(d40c->base->dev,
2217 "configured channel %s for %s, data width %d, "
2218 "maxburst %d bytes, LE, no flow control\n",
2219 dma_chan_name(chan),
2220 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2221 config_addr_width,
2222 config_maxburst);
2223}
2224
05827630
LW		 2225static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2226 unsigned long arg)
8d318a50
LW		 2227{
2228 unsigned long flags;
2229 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2230
0d0f6b8b
JA		 2231 if (d40c->phy_chan == NULL) {
2232 dev_err(&d40c->chan.dev->device,
2233 "[%s] Channel is not allocated!\n", __func__);
2234 return -EINVAL;
2235 }
2236
8d318a50
LW		 2237 switch (cmd) {
2238 case DMA_TERMINATE_ALL:
2239 spin_lock_irqsave(&d40c->lock, flags);
2240 d40_term_all(d40c);
2241 spin_unlock_irqrestore(&d40c->lock, flags);
2242 return 0;
2243 case DMA_PAUSE:
2244 return d40_pause(chan);
2245 case DMA_RESUME:
2246 return d40_resume(chan);
95e1400f
LW		 2247 case DMA_SLAVE_CONFIG:
2248 d40_set_runtime_config(chan,
2249 (struct dma_slave_config *) arg);
2250 return 0;
2251 default:
2252 break;
8d318a50
LW		 2253 }
2254
2255 /* Other commands are unimplemented */
2256 return -ENXIO;
2257}
2258
2259/* Initialization functions */
2260
2261static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2262 struct d40_chan *chans, int offset,
2263 int num_chans)
2264{
2265 int i = 0;
2266 struct d40_chan *d40c;
2267
2268 INIT_LIST_HEAD(&dma->channels);
2269
2270 for (i = offset; i < offset + num_chans; i++) {
2271 d40c = &chans[i];
2272 d40c->base = base;
2273 d40c->chan.device = dma;
2274
2275 /* Invalidate lcla element */
2276 d40c->lcla.src_id = -1;
2277 d40c->lcla.dst_id = -1;
2278
2279 spin_lock_init(&d40c->lock);
2280
2281 d40c->log_num = D40_PHY_CHAN;
2282
8d318a50
LW		 2283 INIT_LIST_HEAD(&d40c->active);
2284 INIT_LIST_HEAD(&d40c->queue);
2285 INIT_LIST_HEAD(&d40c->client);
2286
8d318a50
LW		 2287 tasklet_init(&d40c->tasklet, dma_tasklet,
2288 (unsigned long) d40c);
2289
2290 list_add_tail(&d40c->chan.device_node,
2291 &dma->channels);
2292 }
2293}
2294
2295static int __init d40_dmaengine_init(struct d40_base *base,
2296 int num_reserved_chans)
2297{
2298 int err ;
2299
2300 d40_chan_init(base, &base->dma_slave, base->log_chans,
2301 0, base->num_log_chans);
2302
2303 dma_cap_zero(base->dma_slave.cap_mask);
2304 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2305
2306 base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2307 base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2308 base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2309 base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2310 base->dma_slave.device_tx_status = d40_tx_status;
2311 base->dma_slave.device_issue_pending = d40_issue_pending;
2312 base->dma_slave.device_control = d40_control;
2313 base->dma_slave.dev = base->dev;
2314
2315 err = dma_async_device_register(&base->dma_slave);
2316
2317 if (err) {
2318 dev_err(base->dev,
2319 "[%s] Failed to register slave channels\n",
2320 __func__);
2321 goto failure1;
2322 }
2323
2324 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2325 base->num_log_chans, base->plat_data->memcpy_len);
2326
2327 dma_cap_zero(base->dma_memcpy.cap_mask);
2328 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2329
2330 base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2331 base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2332 base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2333 base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2334 base->dma_memcpy.device_tx_status = d40_tx_status;
2335 base->dma_memcpy.device_issue_pending = d40_issue_pending;
2336 base->dma_memcpy.device_control = d40_control;
2337 base->dma_memcpy.dev = base->dev;
2338 /*
2339 * This controller can only access address at even
2340 * 32bit boundaries, i.e. 2^2
2341 */
2342 base->dma_memcpy.copy_align = 2;
2343
2344 err = dma_async_device_register(&base->dma_memcpy);
2345
2346 if (err) {
2347 dev_err(base->dev,
2348 "[%s] Failed to regsiter memcpy only channels\n",
2349 __func__);
2350 goto failure2;
2351 }
2352
2353 d40_chan_init(base, &base->dma_both, base->phy_chans,
2354 0, num_reserved_chans);
2355
2356 dma_cap_zero(base->dma_both.cap_mask);
2357 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2358 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2359
2360 base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2361 base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2362 base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2363 base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2364 base->dma_both.device_tx_status = d40_tx_status;
2365 base->dma_both.device_issue_pending = d40_issue_pending;
2366 base->dma_both.device_control = d40_control;
2367 base->dma_both.dev = base->dev;
2368 base->dma_both.copy_align = 2;
2369 err = dma_async_device_register(&base->dma_both);
2370
2371 if (err) {
2372 dev_err(base->dev,
2373 "[%s] Failed to register logical and physical capable channels\n",
2374 __func__);
2375 goto failure3;
2376 }
2377 return 0;
2378failure3:
2379 dma_async_device_unregister(&base->dma_memcpy);
2380failure2:
2381 dma_async_device_unregister(&base->dma_slave);
2382failure1:
2383 return err;
2384}
2385
2386/* Initialization functions. */
2387
2388static int __init d40_phy_res_init(struct d40_base *base)
2389{
2390 int i;
2391 int num_phy_chans_avail = 0;
2392 u32 val[2];
2393 int odd_even_bit = -2;
2394
2395 val[0] = readl(base->virtbase + D40_DREG_PRSME);
2396 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2397
2398 for (i = 0; i < base->num_phy_chans; i++) {
2399 base->phy_res[i].num = i;
2400 odd_even_bit += 2 * ((i % 2) == 0);
2401 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2402 /* Mark security only channels as occupied */
2403 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2404 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2405 } else {
2406 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2407 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2408 num_phy_chans_avail++;
2409 }
2410 spin_lock_init(&base->phy_res[i].lock);
2411 }
6b7acd84
JA		 2412
2413 /* Mark disabled channels as occupied */
2414 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2415 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2416 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2417 num_phy_chans_avail--;
2418 }
2419
8d318a50
LW		 2420 dev_info(base->dev, "%d of %d physical DMA channels available\n",
2421 num_phy_chans_avail, base->num_phy_chans);
2422
2423 /* Verify settings extended vs standard */
2424 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2425
2426 for (i = 0; i < base->num_phy_chans; i++) {
2427
2428 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2429 (val[0] & 0x3) != 1)
2430 dev_info(base->dev,
2431 "[%s] INFO: channel %d is misconfigured (%d)\n",
2432 __func__, i, val[0] & 0x3);
2433
2434 val[0] = val[0] >> 2;
2435 }
2436
2437 return num_phy_chans_avail;
2438}
2439
2440static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2441{
2442 static const struct d40_reg_val dma_id_regs[] = {
2443 /* Peripheral Id */
2444 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2445 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2446 /*
2447 * D40_DREG_PERIPHID2 Depends on HW revision:
2448 * MOP500/HREF ED has 0x0008,
2449 * ? has 0x0018,
2450 * HREF V1 has 0x0028
2451 */
2452 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2453
2454 /* PCell Id */
2455 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2456 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2457 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2458 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2459 };
2460 struct stedma40_platform_data *plat_data;
2461 struct clk *clk = NULL;
2462 void __iomem *virtbase = NULL;
2463 struct resource *res = NULL;
2464 struct d40_base *base = NULL;
2465 int num_log_chans = 0;
2466 int num_phy_chans;
2467 int i;
f4185592 2468 u32 val;
8d318a50
LW		 2469
2470 clk = clk_get(&pdev->dev, NULL);
2471
2472 if (IS_ERR(clk)) {
2473 dev_err(&pdev->dev, "[%s] No matching clock found\n",
2474 __func__);
2475 goto failure;
2476 }
2477
2478 clk_enable(clk);
2479
2480 /* Get IO for DMAC base address */
2481 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2482 if (!res)
2483 goto failure;
2484
2485 if (request_mem_region(res->start, resource_size(res),
2486 D40_NAME " I/O base") == NULL)
2487 goto failure;
2488
2489 virtbase = ioremap(res->start, resource_size(res));
2490 if (!virtbase)
2491 goto failure;
2492
2493 /* HW version check */
2494 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2495 if (dma_id_regs[i].val !=
2496 readl(virtbase + dma_id_regs[i].reg)) {
2497 dev_err(&pdev->dev,
2498 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2499 __func__,
2500 dma_id_regs[i].val,
2501 dma_id_regs[i].reg,
2502 readl(virtbase + dma_id_regs[i].reg));
2503 goto failure;
2504 }
2505 }
2506
f4185592
LW		 2507 /* Get silicon revision */
2508 val = readl(virtbase + D40_DREG_PERIPHID2);
8d318a50 2509
f4185592 2510 if ((val & 0xf) != D40_PERIPHID2_DESIGNER) {
8d318a50
LW		 2511 dev_err(&pdev->dev,
2512 "[%s] Unknown designer! Got %x wanted %x\n",
f4185592 2513 __func__, val & 0xf, D40_PERIPHID2_DESIGNER);
8d318a50
LW		 2514 goto failure;
2515 }
2516
2517 /* The number of physical channels on this HW */
2518 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2519
2520 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
f4185592 2521 (val >> 4) & 0xf, res->start);
8d318a50
LW		 2522
2523 plat_data = pdev->dev.platform_data;
2524
2525 /* Count the number of logical channels in use */
2526 for (i = 0; i < plat_data->dev_len; i++)
2527 if (plat_data->dev_rx[i] != 0)
2528 num_log_chans++;
2529
2530 for (i = 0; i < plat_data->dev_len; i++)
2531 if (plat_data->dev_tx[i] != 0)
2532 num_log_chans++;
2533
2534 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2535 (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2536 sizeof(struct d40_chan), GFP_KERNEL);
2537
2538 if (base == NULL) {
2539 dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
2540 goto failure;
2541 }
2542
f4185592 2543 base->rev = (val >> 4) & 0xf;
8d318a50
LW		 2544 base->clk = clk;
2545 base->num_phy_chans = num_phy_chans;
2546 base->num_log_chans = num_log_chans;
2547 base->phy_start = res->start;
2548 base->phy_size = resource_size(res);
2549 base->virtbase = virtbase;
2550 base->plat_data = plat_data;
2551 base->dev = &pdev->dev;
2552 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2553 base->log_chans = &base->phy_chans[num_phy_chans];
2554
2555 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2556 GFP_KERNEL);
2557 if (!base->phy_res)
2558 goto failure;
2559
2560 base->lookup_phy_chans = kzalloc(num_phy_chans *
2561 sizeof(struct d40_chan *),
2562 GFP_KERNEL);
2563 if (!base->lookup_phy_chans)
2564 goto failure;
2565
2566 if (num_log_chans + plat_data->memcpy_len) {
2567 /*
2568 * The max number of logical channels are event lines for all
2569 * src devices and dst devices
2570 */
2571 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2572 sizeof(struct d40_chan *),
2573 GFP_KERNEL);
2574 if (!base->lookup_log_chans)
2575 goto failure;
2576 }
2577 base->lcla_pool.alloc_map = kzalloc(num_phy_chans * sizeof(u32),
2578 GFP_KERNEL);
2579 if (!base->lcla_pool.alloc_map)
2580 goto failure;
2581
c675b1b4
JA		 2582 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2583 0, SLAB_HWCACHE_ALIGN,
2584 NULL);
2585 if (base->desc_slab == NULL)
2586 goto failure;
2587
8d318a50
LW		 2588 return base;
2589
2590failure:
2591 if (clk) {
2592 clk_disable(clk);
2593 clk_put(clk);
2594 }
2595 if (virtbase)
2596 iounmap(virtbase);
2597 if (res)
2598 release_mem_region(res->start,
2599 resource_size(res));
2600 if (virtbase)
2601 iounmap(virtbase);
2602
2603 if (base) {
2604 kfree(base->lcla_pool.alloc_map);
2605 kfree(base->lookup_log_chans);
2606 kfree(base->lookup_phy_chans);
2607 kfree(base->phy_res);
2608 kfree(base);
2609 }
2610
2611 return NULL;
2612}
2613
2614static void __init d40_hw_init(struct d40_base *base)
2615{
2616
2617 static const struct d40_reg_val dma_init_reg[] = {
2618 /* Clock every part of the DMA block from start */
2619 { .reg = D40_DREG_GCC, .val = 0x0000ff01},
2620
2621 /* Interrupts on all logical channels */
2622 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2623 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2624 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2625 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2626 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2627 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2628 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2629 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2630 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2631 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2632 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2633 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2634 };
2635 int i;
2636 u32 prmseo[2] = {0, 0};
2637 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2638 u32 pcmis = 0;
2639 u32 pcicr = 0;
2640
2641 for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2642 writel(dma_init_reg[i].val,
2643 base->virtbase + dma_init_reg[i].reg);
2644
2645 /* Configure all our dma channels to default settings */
2646 for (i = 0; i < base->num_phy_chans; i++) {
2647
2648 activeo[i % 2] = activeo[i % 2] << 2;
2649
2650 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2651 == D40_ALLOC_PHY) {
2652 activeo[i % 2] |= 3;
2653 continue;
2654 }
2655
2656 /* Enable interrupt # */
2657 pcmis = (pcmis << 1) | 1;
2658
2659 /* Clear interrupt # */
2660 pcicr = (pcicr << 1) | 1;
2661
2662 /* Set channel to physical mode */
2663 prmseo[i % 2] = prmseo[i % 2] << 2;
2664 prmseo[i % 2] |= 1;
2665
2666 }
2667
2668 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2669 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2670 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2671 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2672
2673 /* Write which interrupt to enable */
2674 writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2675
2676 /* Write which interrupt to clear */
2677 writel(pcicr, base->virtbase + D40_DREG_PCICR);
2678
2679}
2680
508849ad
LW		 2681static int __init d40_lcla_allocate(struct d40_base *base)
2682{
2683 unsigned long *page_list;
2684 int i, j;
2685 int ret = 0;
2686
2687 /*
2688 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2689 * To full fill this hardware requirement without wasting 256 kb
2690 * we allocate pages until we get an aligned one.
2691 */
2692 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2693 GFP_KERNEL);
2694
2695 if (!page_list) {
2696 ret = -ENOMEM;
2697 goto failure;
2698 }
2699
2700 /* Calculating how many pages that are required */
2701 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2702
2703 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2704 page_list[i] = __get_free_pages(GFP_KERNEL,
2705 base->lcla_pool.pages);
2706 if (!page_list[i]) {
2707
2708 dev_err(base->dev,
2709 "[%s] Failed to allocate %d pages.\n",
2710 __func__, base->lcla_pool.pages);
2711
2712 for (j = 0; j < i; j++)
2713 free_pages(page_list[j], base->lcla_pool.pages);
2714 goto failure;
2715 }
2716
2717 if ((virt_to_phys((void *)page_list[i]) &
2718 (LCLA_ALIGNMENT - 1)) == 0)
2719 break;
2720 }
2721
2722 for (j = 0; j < i; j++)
2723 free_pages(page_list[j], base->lcla_pool.pages);
2724
2725 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2726 base->lcla_pool.base = (void *)page_list[i];
2727 } else {
2728 /* After many attempts, no succees with finding the correct
2729 * alignment try with allocating a big buffer */
2730 dev_warn(base->dev,
2731 "[%s] Failed to get %d pages @ 18 bit align.\n",
2732 __func__, base->lcla_pool.pages);
2733 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2734 base->num_phy_chans +
2735 LCLA_ALIGNMENT,
2736 GFP_KERNEL);
2737 if (!base->lcla_pool.base_unaligned) {
2738 ret = -ENOMEM;
2739 goto failure;
2740 }
2741
2742 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2743 LCLA_ALIGNMENT);
2744 }
2745
2746 writel(virt_to_phys(base->lcla_pool.base),
2747 base->virtbase + D40_DREG_LCLA);
2748failure:
2749 kfree(page_list);
2750 return ret;
2751}
2752
8d318a50
LW		 2753static int __init d40_probe(struct platform_device *pdev)
2754{
2755 int err;
2756 int ret = -ENOENT;
2757 struct d40_base *base;
2758 struct resource *res = NULL;
2759 int num_reserved_chans;
2760 u32 val;
2761
2762 base = d40_hw_detect_init(pdev);
2763
2764 if (!base)
2765 goto failure;
2766
2767 num_reserved_chans = d40_phy_res_init(base);
2768
2769 platform_set_drvdata(pdev, base);
2770
2771 spin_lock_init(&base->interrupt_lock);
2772 spin_lock_init(&base->execmd_lock);
2773
2774 /* Get IO for logical channel parameter address */
2775 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2776 if (!res) {
2777 ret = -ENOENT;
2778 dev_err(&pdev->dev,
2779 "[%s] No \"lcpa\" memory resource\n",
2780 __func__);
2781 goto failure;
2782 }
2783 base->lcpa_size = resource_size(res);
2784 base->phy_lcpa = res->start;
2785
2786 if (request_mem_region(res->start, resource_size(res),
2787 D40_NAME " I/O lcpa") == NULL) {
2788 ret = -EBUSY;
2789 dev_err(&pdev->dev,
2790 "[%s] Failed to request LCPA region 0x%x-0x%x\n",
2791 __func__, res->start, res->end);
2792 goto failure;
2793 }
2794
2795 /* We make use of ESRAM memory for this. */
2796 val = readl(base->virtbase + D40_DREG_LCPA);
2797 if (res->start != val && val != 0) {
2798 dev_warn(&pdev->dev,
2799 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2800 __func__, val, res->start);
2801 } else
2802 writel(res->start, base->virtbase + D40_DREG_LCPA);
2803
2804 base->lcpa_base = ioremap(res->start, resource_size(res));
2805 if (!base->lcpa_base) {
2806 ret = -ENOMEM;
2807 dev_err(&pdev->dev,
2808 "[%s] Failed to ioremap LCPA region\n",
2809 __func__);
2810 goto failure;
2811 }
8d318a50 2812
508849ad
LW		 2813 ret = d40_lcla_allocate(base);
2814 if (ret) {
2815 dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n",
2816 __func__);
8d318a50
LW		 2817 goto failure;
2818 }
2819
2820 spin_lock_init(&base->lcla_pool.lock);
2821
2822 base->lcla_pool.num_blocks = base->num_phy_chans;
2823
2824 base->irq = platform_get_irq(pdev, 0);
2825
2826 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2827
2828 if (ret) {
2829 dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
2830 goto failure;
2831 }
2832
2833 err = d40_dmaengine_init(base, num_reserved_chans);
2834 if (err)
2835 goto failure;
2836
2837 d40_hw_init(base);
2838
2839 dev_info(base->dev, "initialized\n");
2840 return 0;
2841
2842failure:
2843 if (base) {
c675b1b4
JA		 2844 if (base->desc_slab)
2845 kmem_cache_destroy(base->desc_slab);
8d318a50
LW		 2846 if (base->virtbase)
2847 iounmap(base->virtbase);
508849ad
LW		 2848 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2849 free_pages((unsigned long)base->lcla_pool.base,
2850 base->lcla_pool.pages);
2851 if (base->lcla_pool.base_unaligned)
2852 kfree(base->lcla_pool.base_unaligned);
8d318a50
LW		 2853 if (base->phy_lcpa)
2854 release_mem_region(base->phy_lcpa,
2855 base->lcpa_size);
2856 if (base->phy_start)
2857 release_mem_region(base->phy_start,
2858 base->phy_size);
2859 if (base->clk) {
2860 clk_disable(base->clk);
2861 clk_put(base->clk);
2862 }
2863
2864 kfree(base->lcla_pool.alloc_map);
2865 kfree(base->lookup_log_chans);
2866 kfree(base->lookup_phy_chans);
2867 kfree(base->phy_res);
2868 kfree(base);
2869 }
2870
2871 dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
2872 return ret;
2873}
2874
2875static struct platform_driver d40_driver = {
2876 .driver = {
2877 .owner = THIS_MODULE,
2878 .name = D40_NAME,
2879 },
2880};
2881
2882int __init stedma40_init(void)
2883{
2884 return platform_driver_probe(&d40_driver, d40_probe);
2885}
2886arch_initcall(stedma40_init);
Clone of davem's net-next-2.6 tree