]>
Commit | Line | Data |
---|---|---|
58f07778 DD |
1 | /***********************license start*************** |
2 | * Author: Cavium Networks | |
3 | * | |
4 | * Contact: support@caviumnetworks.com | |
5 | * This file is part of the OCTEON SDK | |
6 | * | |
b8db85b5 | 7 | * Copyright (c) 2003-2010 Cavium Networks |
58f07778 DD |
8 | * |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more | |
17 | * details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License | |
20 | * along with this file; if not, write to the Free Software | |
21 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
22 | * or visit http://www.gnu.org/licenses/. | |
23 | * | |
24 | * This file may also be available under a different license from Cavium. | |
25 | * Contact Cavium Networks for more information | |
26 | ***********************license end**************************************/ | |
27 | ||
28 | /* | |
b8db85b5 DD |
29 | * Implementation of the Level 2 Cache (L2C) control, |
30 | * measurement, and debugging facilities. | |
58f07778 DD |
31 | */ |
32 | ||
33 | #include <asm/octeon/cvmx.h> | |
34 | #include <asm/octeon/cvmx-l2c.h> | |
35 | #include <asm/octeon/cvmx-spinlock.h> | |
36 | ||
37 | /* | |
38 | * This spinlock is used internally to ensure that only one core is | |
39 | * performing certain L2 operations at a time. | |
40 | * | |
41 | * NOTE: This only protects calls from within a single application - | |
42 | * if multiple applications or operating systems are running, then it | |
43 | * is up to the user program to coordinate between them. | |
44 | */ | |
b8db85b5 | 45 | cvmx_spinlock_t cvmx_l2c_spinlock; |
58f07778 DD |
46 | |
47 | int cvmx_l2c_get_core_way_partition(uint32_t core) | |
48 | { | |
49 | uint32_t field; | |
50 | ||
51 | /* Validate the core number */ | |
52 | if (core >= cvmx_octeon_num_cores()) | |
53 | return -1; | |
54 | ||
b8db85b5 DD |
55 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
56 | return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff; | |
57 | ||
58f07778 DD |
58 | /* |
59 | * Use the lower two bits of the coreNumber to determine the | |
60 | * bit offset of the UMSK[] field in the L2C_SPAR register. | |
61 | */ | |
62 | field = (core & 0x3) * 8; | |
63 | ||
64 | /* | |
65 | * Return the UMSK[] field from the appropriate L2C_SPAR | |
66 | * register based on the coreNumber. | |
67 | */ | |
68 | ||
69 | switch (core & 0xC) { | |
70 | case 0x0: | |
b8db85b5 | 71 | return (cvmx_read_csr(CVMX_L2C_SPAR0) & (0xFF << field)) >> field; |
58f07778 | 72 | case 0x4: |
b8db85b5 | 73 | return (cvmx_read_csr(CVMX_L2C_SPAR1) & (0xFF << field)) >> field; |
58f07778 | 74 | case 0x8: |
b8db85b5 | 75 | return (cvmx_read_csr(CVMX_L2C_SPAR2) & (0xFF << field)) >> field; |
58f07778 | 76 | case 0xC: |
b8db85b5 | 77 | return (cvmx_read_csr(CVMX_L2C_SPAR3) & (0xFF << field)) >> field; |
58f07778 DD |
78 | } |
79 | return 0; | |
80 | } | |
81 | ||
82 | int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask) | |
83 | { | |
84 | uint32_t field; | |
85 | uint32_t valid_mask; | |
86 | ||
87 | valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1; | |
88 | ||
89 | mask &= valid_mask; | |
90 | ||
b8db85b5 DD |
91 | /* A UMSK setting which blocks all L2C Ways is an error on some chips */ |
92 | if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX)) | |
58f07778 DD |
93 | return -1; |
94 | ||
95 | /* Validate the core number */ | |
96 | if (core >= cvmx_octeon_num_cores()) | |
97 | return -1; | |
98 | ||
b8db85b5 DD |
99 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
100 | cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask); | |
101 | return 0; | |
102 | } | |
58f07778 | 103 | |
b8db85b5 DD |
104 | /* |
105 | * Use the lower two bits of core to determine the bit offset of the | |
58f07778 DD |
106 | * UMSK[] field in the L2C_SPAR register. |
107 | */ | |
108 | field = (core & 0x3) * 8; | |
109 | ||
b8db85b5 DD |
110 | /* |
111 | * Assign the new mask setting to the UMSK[] field in the appropriate | |
58f07778 DD |
112 | * L2C_SPAR register based on the core_num. |
113 | * | |
114 | */ | |
115 | switch (core & 0xC) { | |
116 | case 0x0: | |
117 | cvmx_write_csr(CVMX_L2C_SPAR0, | |
b8db85b5 DD |
118 | (cvmx_read_csr(CVMX_L2C_SPAR0) & ~(0xFF << field)) | |
119 | mask << field); | |
58f07778 DD |
120 | break; |
121 | case 0x4: | |
122 | cvmx_write_csr(CVMX_L2C_SPAR1, | |
b8db85b5 DD |
123 | (cvmx_read_csr(CVMX_L2C_SPAR1) & ~(0xFF << field)) | |
124 | mask << field); | |
58f07778 DD |
125 | break; |
126 | case 0x8: | |
127 | cvmx_write_csr(CVMX_L2C_SPAR2, | |
b8db85b5 DD |
128 | (cvmx_read_csr(CVMX_L2C_SPAR2) & ~(0xFF << field)) | |
129 | mask << field); | |
58f07778 DD |
130 | break; |
131 | case 0xC: | |
132 | cvmx_write_csr(CVMX_L2C_SPAR3, | |
b8db85b5 DD |
133 | (cvmx_read_csr(CVMX_L2C_SPAR3) & ~(0xFF << field)) | |
134 | mask << field); | |
58f07778 DD |
135 | break; |
136 | } | |
137 | return 0; | |
138 | } | |
139 | ||
140 | int cvmx_l2c_set_hw_way_partition(uint32_t mask) | |
141 | { | |
142 | uint32_t valid_mask; | |
143 | ||
b8db85b5 | 144 | valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1; |
58f07778 DD |
145 | mask &= valid_mask; |
146 | ||
b8db85b5 DD |
147 | /* A UMSK setting which blocks all L2C Ways is an error on some chips */ |
148 | if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX)) | |
58f07778 DD |
149 | return -1; |
150 | ||
b8db85b5 DD |
151 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
152 | cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask); | |
153 | else | |
154 | cvmx_write_csr(CVMX_L2C_SPAR4, | |
155 | (cvmx_read_csr(CVMX_L2C_SPAR4) & ~0xFF) | mask); | |
58f07778 DD |
156 | return 0; |
157 | } | |
158 | ||
159 | int cvmx_l2c_get_hw_way_partition(void) | |
160 | { | |
b8db85b5 DD |
161 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
162 | return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff; | |
163 | else | |
164 | return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF); | |
58f07778 DD |
165 | } |
166 | ||
167 | void cvmx_l2c_config_perf(uint32_t counter, enum cvmx_l2c_event event, | |
168 | uint32_t clear_on_read) | |
169 | { | |
b8db85b5 DD |
170 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) { |
171 | union cvmx_l2c_pfctl pfctl; | |
58f07778 | 172 | |
b8db85b5 | 173 | pfctl.u64 = cvmx_read_csr(CVMX_L2C_PFCTL); |
58f07778 | 174 | |
b8db85b5 DD |
175 | switch (counter) { |
176 | case 0: | |
177 | pfctl.s.cnt0sel = event; | |
178 | pfctl.s.cnt0ena = 1; | |
58f07778 | 179 | pfctl.s.cnt0rdclr = clear_on_read; |
b8db85b5 DD |
180 | break; |
181 | case 1: | |
182 | pfctl.s.cnt1sel = event; | |
183 | pfctl.s.cnt1ena = 1; | |
58f07778 | 184 | pfctl.s.cnt1rdclr = clear_on_read; |
b8db85b5 DD |
185 | break; |
186 | case 2: | |
187 | pfctl.s.cnt2sel = event; | |
188 | pfctl.s.cnt2ena = 1; | |
58f07778 | 189 | pfctl.s.cnt2rdclr = clear_on_read; |
b8db85b5 DD |
190 | break; |
191 | case 3: | |
192 | default: | |
193 | pfctl.s.cnt3sel = event; | |
194 | pfctl.s.cnt3ena = 1; | |
58f07778 | 195 | pfctl.s.cnt3rdclr = clear_on_read; |
b8db85b5 DD |
196 | break; |
197 | } | |
58f07778 | 198 | |
b8db85b5 DD |
199 | cvmx_write_csr(CVMX_L2C_PFCTL, pfctl.u64); |
200 | } else { | |
201 | union cvmx_l2c_tadx_prf l2c_tadx_prf; | |
202 | int tad; | |
203 | ||
204 | cvmx_dprintf("L2C performance counter events are different for this chip, mapping 'event' to cvmx_l2c_tad_event_t\n"); | |
205 | if (clear_on_read) | |
206 | cvmx_dprintf("L2C counters don't support clear on read for this chip\n"); | |
207 | ||
208 | l2c_tadx_prf.u64 = cvmx_read_csr(CVMX_L2C_TADX_PRF(0)); | |
209 | ||
210 | switch (counter) { | |
211 | case 0: | |
212 | l2c_tadx_prf.s.cnt0sel = event; | |
213 | break; | |
214 | case 1: | |
215 | l2c_tadx_prf.s.cnt1sel = event; | |
216 | break; | |
217 | case 2: | |
218 | l2c_tadx_prf.s.cnt2sel = event; | |
219 | break; | |
220 | default: | |
221 | case 3: | |
222 | l2c_tadx_prf.s.cnt3sel = event; | |
223 | break; | |
224 | } | |
225 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
226 | cvmx_write_csr(CVMX_L2C_TADX_PRF(tad), | |
227 | l2c_tadx_prf.u64); | |
228 | } | |
58f07778 DD |
229 | } |
230 | ||
231 | uint64_t cvmx_l2c_read_perf(uint32_t counter) | |
232 | { | |
233 | switch (counter) { | |
234 | case 0: | |
b8db85b5 DD |
235 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
236 | return cvmx_read_csr(CVMX_L2C_PFC0); | |
237 | else { | |
238 | uint64_t counter = 0; | |
239 | int tad; | |
240 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
241 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC0(tad)); | |
242 | return counter; | |
243 | } | |
58f07778 | 244 | case 1: |
b8db85b5 DD |
245 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
246 | return cvmx_read_csr(CVMX_L2C_PFC1); | |
247 | else { | |
248 | uint64_t counter = 0; | |
249 | int tad; | |
250 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
251 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC1(tad)); | |
252 | return counter; | |
253 | } | |
58f07778 | 254 | case 2: |
b8db85b5 DD |
255 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
256 | return cvmx_read_csr(CVMX_L2C_PFC2); | |
257 | else { | |
258 | uint64_t counter = 0; | |
259 | int tad; | |
260 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
261 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC2(tad)); | |
262 | return counter; | |
263 | } | |
58f07778 DD |
264 | case 3: |
265 | default: | |
b8db85b5 DD |
266 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
267 | return cvmx_read_csr(CVMX_L2C_PFC3); | |
268 | else { | |
269 | uint64_t counter = 0; | |
270 | int tad; | |
271 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
272 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC3(tad)); | |
273 | return counter; | |
274 | } | |
58f07778 DD |
275 | } |
276 | } | |
277 | ||
278 | /** | |
279 | * @INTERNAL | |
280 | * Helper function use to fault in cache lines for L2 cache locking | |
281 | * | |
282 | * @addr: Address of base of memory region to read into L2 cache | |
283 | * @len: Length (in bytes) of region to fault in | |
284 | */ | |
285 | static void fault_in(uint64_t addr, int len) | |
286 | { | |
287 | volatile char *ptr; | |
288 | volatile char dummy; | |
289 | /* | |
290 | * Adjust addr and length so we get all cache lines even for | |
b8db85b5 | 291 | * small ranges spanning two cache lines. |
58f07778 DD |
292 | */ |
293 | len += addr & CVMX_CACHE_LINE_MASK; | |
294 | addr &= ~CVMX_CACHE_LINE_MASK; | |
295 | ptr = (volatile char *)cvmx_phys_to_ptr(addr); | |
296 | /* | |
297 | * Invalidate L1 cache to make sure all loads result in data | |
298 | * being in L2. | |
299 | */ | |
300 | CVMX_DCACHE_INVALIDATE; | |
301 | while (len > 0) { | |
302 | dummy += *ptr; | |
303 | len -= CVMX_CACHE_LINE_SIZE; | |
304 | ptr += CVMX_CACHE_LINE_SIZE; | |
305 | } | |
306 | } | |
307 | ||
308 | int cvmx_l2c_lock_line(uint64_t addr) | |
309 | { | |
b8db85b5 DD |
310 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
311 | int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT; | |
312 | uint64_t assoc = cvmx_l2c_get_num_assoc(); | |
313 | uint64_t tag = addr >> shift; | |
314 | uint64_t index = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, cvmx_l2c_address_to_index(addr) << CVMX_L2C_IDX_ADDR_SHIFT); | |
315 | uint64_t way; | |
316 | union cvmx_l2c_tadx_tag l2c_tadx_tag; | |
317 | ||
318 | CVMX_CACHE_LCKL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, addr), 0); | |
319 | ||
320 | /* Make sure we were able to lock the line */ | |
321 | for (way = 0; way < assoc; way++) { | |
322 | CVMX_CACHE_LTGL2I(index | (way << shift), 0); | |
323 | /* make sure CVMX_L2C_TADX_TAG is updated */ | |
324 | CVMX_SYNC; | |
325 | l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0)); | |
326 | if (l2c_tadx_tag.s.valid && l2c_tadx_tag.s.tag == tag) | |
327 | break; | |
328 | } | |
58f07778 | 329 | |
b8db85b5 DD |
330 | /* Check if a valid line is found */ |
331 | if (way >= assoc) { | |
332 | /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: line not found for locking at 0x%llx address\n", (unsigned long long)addr); */ | |
333 | return -1; | |
334 | } | |
58f07778 | 335 | |
b8db85b5 DD |
336 | /* Check if lock bit is not set */ |
337 | if (!l2c_tadx_tag.s.lock) { | |
338 | /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: Not able to lock at 0x%llx address\n", (unsigned long long)addr); */ | |
339 | return -1; | |
340 | } | |
341 | return way; | |
58f07778 | 342 | } else { |
b8db85b5 DD |
343 | int retval = 0; |
344 | union cvmx_l2c_dbg l2cdbg; | |
345 | union cvmx_l2c_lckbase lckbase; | |
346 | union cvmx_l2c_lckoff lckoff; | |
347 | union cvmx_l2t_err l2t_err; | |
58f07778 | 348 | |
b8db85b5 | 349 | cvmx_spinlock_lock(&cvmx_l2c_spinlock); |
58f07778 | 350 | |
b8db85b5 DD |
351 | l2cdbg.u64 = 0; |
352 | lckbase.u64 = 0; | |
353 | lckoff.u64 = 0; | |
58f07778 | 354 | |
b8db85b5 DD |
355 | /* Clear l2t error bits if set */ |
356 | l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR); | |
357 | l2t_err.s.lckerr = 1; | |
358 | l2t_err.s.lckerr2 = 1; | |
359 | cvmx_write_csr(CVMX_L2T_ERR, l2t_err.u64); | |
58f07778 | 360 | |
b8db85b5 | 361 | addr &= ~CVMX_CACHE_LINE_MASK; |
58f07778 | 362 | |
b8db85b5 DD |
363 | /* Set this core as debug core */ |
364 | l2cdbg.s.ppnum = cvmx_get_core_num(); | |
365 | CVMX_SYNC; | |
366 | cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64); | |
367 | cvmx_read_csr(CVMX_L2C_DBG); | |
368 | ||
369 | lckoff.s.lck_offset = 0; /* Only lock 1 line at a time */ | |
370 | cvmx_write_csr(CVMX_L2C_LCKOFF, lckoff.u64); | |
371 | cvmx_read_csr(CVMX_L2C_LCKOFF); | |
372 | ||
373 | if (((union cvmx_l2c_cfg)(cvmx_read_csr(CVMX_L2C_CFG))).s.idxalias) { | |
374 | int alias_shift = CVMX_L2C_IDX_ADDR_SHIFT + 2 * CVMX_L2_SET_BITS - 1; | |
375 | uint64_t addr_tmp = addr ^ (addr & ((1 << alias_shift) - 1)) >> CVMX_L2_SET_BITS; | |
376 | lckbase.s.lck_base = addr_tmp >> 7; | |
377 | } else { | |
378 | lckbase.s.lck_base = addr >> 7; | |
379 | } | |
58f07778 | 380 | |
b8db85b5 DD |
381 | lckbase.s.lck_ena = 1; |
382 | cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64); | |
383 | /* Make sure it gets there */ | |
384 | cvmx_read_csr(CVMX_L2C_LCKBASE); | |
58f07778 | 385 | |
b8db85b5 DD |
386 | fault_in(addr, CVMX_CACHE_LINE_SIZE); |
387 | ||
388 | lckbase.s.lck_ena = 0; | |
389 | cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64); | |
390 | /* Make sure it gets there */ | |
391 | cvmx_read_csr(CVMX_L2C_LCKBASE); | |
392 | ||
393 | /* Stop being debug core */ | |
394 | cvmx_write_csr(CVMX_L2C_DBG, 0); | |
395 | cvmx_read_csr(CVMX_L2C_DBG); | |
396 | ||
397 | l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR); | |
398 | if (l2t_err.s.lckerr || l2t_err.s.lckerr2) | |
399 | retval = 1; /* We were unable to lock the line */ | |
400 | ||
401 | cvmx_spinlock_unlock(&cvmx_l2c_spinlock); | |
402 | return retval; | |
403 | } | |
58f07778 DD |
404 | } |
405 | ||
406 | int cvmx_l2c_lock_mem_region(uint64_t start, uint64_t len) | |
407 | { | |
408 | int retval = 0; | |
409 | ||
410 | /* Round start/end to cache line boundaries */ | |
411 | len += start & CVMX_CACHE_LINE_MASK; | |
412 | start &= ~CVMX_CACHE_LINE_MASK; | |
413 | len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK; | |
414 | ||
415 | while (len) { | |
416 | retval += cvmx_l2c_lock_line(start); | |
417 | start += CVMX_CACHE_LINE_SIZE; | |
418 | len -= CVMX_CACHE_LINE_SIZE; | |
419 | } | |
58f07778 DD |
420 | return retval; |
421 | } | |
422 | ||
423 | void cvmx_l2c_flush(void) | |
424 | { | |
425 | uint64_t assoc, set; | |
426 | uint64_t n_assoc, n_set; | |
58f07778 | 427 | |
b8db85b5 DD |
428 | n_set = cvmx_l2c_get_num_sets(); |
429 | n_assoc = cvmx_l2c_get_num_assoc(); | |
430 | ||
431 | if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) { | |
432 | uint64_t address; | |
433 | /* These may look like constants, but they aren't... */ | |
434 | int assoc_shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT; | |
435 | int set_shift = CVMX_L2C_IDX_ADDR_SHIFT; | |
436 | for (set = 0; set < n_set; set++) { | |
437 | for (assoc = 0; assoc < n_assoc; assoc++) { | |
438 | address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
439 | (assoc << assoc_shift) | (set << set_shift)); | |
440 | CVMX_CACHE_WBIL2I(address, 0); | |
441 | } | |
58f07778 | 442 | } |
b8db85b5 DD |
443 | } else { |
444 | for (set = 0; set < n_set; set++) | |
445 | for (assoc = 0; assoc < n_assoc; assoc++) | |
446 | cvmx_l2c_flush_line(assoc, set); | |
58f07778 | 447 | } |
58f07778 DD |
448 | } |
449 | ||
b8db85b5 | 450 | |
58f07778 DD |
451 | int cvmx_l2c_unlock_line(uint64_t address) |
452 | { | |
58f07778 | 453 | |
b8db85b5 DD |
454 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
455 | int assoc; | |
456 | union cvmx_l2c_tag tag; | |
457 | uint32_t tag_addr; | |
458 | uint32_t index = cvmx_l2c_address_to_index(address); | |
459 | ||
460 | tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1)); | |
461 | ||
462 | /* | |
463 | * For 63XX, we can flush a line by using the physical | |
464 | * address directly, so finding the cache line used by | |
465 | * the address is only required to provide the proper | |
466 | * return value for the function. | |
467 | */ | |
468 | for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) { | |
469 | tag = cvmx_l2c_get_tag(assoc, index); | |
470 | ||
471 | if (tag.s.V && (tag.s.addr == tag_addr)) { | |
472 | CVMX_CACHE_WBIL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, address), 0); | |
473 | return tag.s.L; | |
474 | } | |
475 | } | |
476 | } else { | |
477 | int assoc; | |
478 | union cvmx_l2c_tag tag; | |
479 | uint32_t tag_addr; | |
58f07778 | 480 | |
b8db85b5 | 481 | uint32_t index = cvmx_l2c_address_to_index(address); |
58f07778 | 482 | |
b8db85b5 DD |
483 | /* Compute portion of address that is stored in tag */ |
484 | tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1)); | |
485 | for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) { | |
486 | tag = cvmx_l2c_get_tag(assoc, index); | |
58f07778 | 487 | |
b8db85b5 DD |
488 | if (tag.s.V && (tag.s.addr == tag_addr)) { |
489 | cvmx_l2c_flush_line(assoc, index); | |
490 | return tag.s.L; | |
491 | } | |
58f07778 DD |
492 | } |
493 | } | |
58f07778 DD |
494 | return 0; |
495 | } | |
496 | ||
497 | int cvmx_l2c_unlock_mem_region(uint64_t start, uint64_t len) | |
498 | { | |
499 | int num_unlocked = 0; | |
500 | /* Round start/end to cache line boundaries */ | |
501 | len += start & CVMX_CACHE_LINE_MASK; | |
502 | start &= ~CVMX_CACHE_LINE_MASK; | |
503 | len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK; | |
504 | while (len > 0) { | |
505 | num_unlocked += cvmx_l2c_unlock_line(start); | |
506 | start += CVMX_CACHE_LINE_SIZE; | |
507 | len -= CVMX_CACHE_LINE_SIZE; | |
508 | } | |
509 | ||
510 | return num_unlocked; | |
511 | } | |
512 | ||
513 | /* | |
514 | * Internal l2c tag types. These are converted to a generic structure | |
515 | * that can be used on all chips. | |
516 | */ | |
517 | union __cvmx_l2c_tag { | |
518 | uint64_t u64; | |
519 | struct cvmx_l2c_tag_cn50xx { | |
520 | uint64_t reserved:40; | |
b8db85b5 DD |
521 | uint64_t V:1; /* Line valid */ |
522 | uint64_t D:1; /* Line dirty */ | |
523 | uint64_t L:1; /* Line locked */ | |
524 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
525 | uint64_t addr:20; /* Phys mem addr (33..14) */ |
526 | } cn50xx; | |
527 | struct cvmx_l2c_tag_cn30xx { | |
528 | uint64_t reserved:41; | |
b8db85b5 DD |
529 | uint64_t V:1; /* Line valid */ |
530 | uint64_t D:1; /* Line dirty */ | |
531 | uint64_t L:1; /* Line locked */ | |
532 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
533 | uint64_t addr:19; /* Phys mem addr (33..15) */ |
534 | } cn30xx; | |
535 | struct cvmx_l2c_tag_cn31xx { | |
536 | uint64_t reserved:42; | |
b8db85b5 DD |
537 | uint64_t V:1; /* Line valid */ |
538 | uint64_t D:1; /* Line dirty */ | |
539 | uint64_t L:1; /* Line locked */ | |
540 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
541 | uint64_t addr:18; /* Phys mem addr (33..16) */ |
542 | } cn31xx; | |
543 | struct cvmx_l2c_tag_cn38xx { | |
544 | uint64_t reserved:43; | |
b8db85b5 DD |
545 | uint64_t V:1; /* Line valid */ |
546 | uint64_t D:1; /* Line dirty */ | |
547 | uint64_t L:1; /* Line locked */ | |
548 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
549 | uint64_t addr:17; /* Phys mem addr (33..17) */ |
550 | } cn38xx; | |
551 | struct cvmx_l2c_tag_cn58xx { | |
552 | uint64_t reserved:44; | |
b8db85b5 DD |
553 | uint64_t V:1; /* Line valid */ |
554 | uint64_t D:1; /* Line dirty */ | |
555 | uint64_t L:1; /* Line locked */ | |
556 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
557 | uint64_t addr:16; /* Phys mem addr (33..18) */ |
558 | } cn58xx; | |
559 | struct cvmx_l2c_tag_cn58xx cn56xx; /* 2048 sets */ | |
560 | struct cvmx_l2c_tag_cn31xx cn52xx; /* 512 sets */ | |
561 | }; | |
562 | ||
b8db85b5 | 563 | |
58f07778 DD |
564 | /** |
565 | * @INTERNAL | |
566 | * Function to read a L2C tag. This code make the current core | |
567 | * the 'debug core' for the L2. This code must only be executed by | |
568 | * 1 core at a time. | |
569 | * | |
570 | * @assoc: Association (way) of the tag to dump | |
571 | * @index: Index of the cacheline | |
572 | * | |
573 | * Returns The Octeon model specific tag structure. This is | |
574 | * translated by a wrapper function to a generic form that is | |
575 | * easier for applications to use. | |
576 | */ | |
577 | static union __cvmx_l2c_tag __read_l2_tag(uint64_t assoc, uint64_t index) | |
578 | { | |
579 | ||
b8db85b5 | 580 | uint64_t debug_tag_addr = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, (index << 7) + 96); |
58f07778 DD |
581 | uint64_t core = cvmx_get_core_num(); |
582 | union __cvmx_l2c_tag tag_val; | |
583 | uint64_t dbg_addr = CVMX_L2C_DBG; | |
584 | unsigned long flags; | |
585 | ||
586 | union cvmx_l2c_dbg debug_val; | |
587 | debug_val.u64 = 0; | |
588 | /* | |
b8db85b5 DD |
589 | * For low core count parts, the core number is always small |
590 | * enough to stay in the correct field and not set any | |
591 | * reserved bits. | |
58f07778 DD |
592 | */ |
593 | debug_val.s.ppnum = core; | |
594 | debug_val.s.l2t = 1; | |
595 | debug_val.s.set = assoc; | |
b8db85b5 DD |
596 | |
597 | local_irq_save(flags); | |
58f07778 DD |
598 | /* |
599 | * Make sure core is quiet (no prefetches, etc.) before | |
600 | * entering debug mode. | |
601 | */ | |
602 | CVMX_SYNC; | |
603 | /* Flush L1 to make sure debug load misses L1 */ | |
604 | CVMX_DCACHE_INVALIDATE; | |
605 | ||
58f07778 DD |
606 | /* |
607 | * The following must be done in assembly as when in debug | |
608 | * mode all data loads from L2 return special debug data, not | |
b8db85b5 DD |
609 | * normal memory contents. Also, interrupts must be disabled, |
610 | * since if an interrupt occurs while in debug mode the ISR | |
611 | * will get debug data from all its memory * reads instead of | |
612 | * the contents of memory. | |
58f07778 DD |
613 | */ |
614 | ||
b8db85b5 DD |
615 | asm volatile ( |
616 | ".set push\n\t" | |
617 | ".set mips64\n\t" | |
618 | ".set noreorder\n\t" | |
619 | "sd %[dbg_val], 0(%[dbg_addr])\n\t" /* Enter debug mode, wait for store */ | |
620 | "ld $0, 0(%[dbg_addr])\n\t" | |
621 | "ld %[tag_val], 0(%[tag_addr])\n\t" /* Read L2C tag data */ | |
622 | "sd $0, 0(%[dbg_addr])\n\t" /* Exit debug mode, wait for store */ | |
623 | "ld $0, 0(%[dbg_addr])\n\t" | |
624 | "cache 9, 0($0)\n\t" /* Invalidate dcache to discard debug data */ | |
625 | ".set pop" | |
626 | : [tag_val] "=r" (tag_val) | |
627 | : [dbg_addr] "r" (dbg_addr), [dbg_val] "r" (debug_val), [tag_addr] "r" (debug_tag_addr) | |
628 | : "memory"); | |
58f07778 DD |
629 | |
630 | local_irq_restore(flags); | |
58f07778 | 631 | |
b8db85b5 | 632 | return tag_val; |
58f07778 DD |
633 | } |
634 | ||
b8db85b5 | 635 | |
58f07778 DD |
636 | union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index) |
637 | { | |
58f07778 DD |
638 | union cvmx_l2c_tag tag; |
639 | tag.u64 = 0; | |
640 | ||
641 | if ((int)association >= cvmx_l2c_get_num_assoc()) { | |
b8db85b5 | 642 | cvmx_dprintf("ERROR: cvmx_l2c_get_tag association out of range\n"); |
58f07778 DD |
643 | return tag; |
644 | } | |
645 | if ((int)index >= cvmx_l2c_get_num_sets()) { | |
b8db85b5 DD |
646 | cvmx_dprintf("ERROR: cvmx_l2c_get_tag index out of range (arg: %d, max: %d)\n", |
647 | (int)index, cvmx_l2c_get_num_sets()); | |
58f07778 DD |
648 | return tag; |
649 | } | |
b8db85b5 DD |
650 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
651 | union cvmx_l2c_tadx_tag l2c_tadx_tag; | |
652 | uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
653 | (association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) | | |
654 | (index << CVMX_L2C_IDX_ADDR_SHIFT)); | |
655 | /* | |
656 | * Use L2 cache Index load tag cache instruction, as | |
657 | * hardware loads the virtual tag for the L2 cache | |
658 | * block with the contents of L2C_TAD0_TAG | |
659 | * register. | |
660 | */ | |
661 | CVMX_CACHE_LTGL2I(address, 0); | |
662 | CVMX_SYNC; /* make sure CVMX_L2C_TADX_TAG is updated */ | |
663 | l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0)); | |
664 | ||
665 | tag.s.V = l2c_tadx_tag.s.valid; | |
666 | tag.s.D = l2c_tadx_tag.s.dirty; | |
667 | tag.s.L = l2c_tadx_tag.s.lock; | |
668 | tag.s.U = l2c_tadx_tag.s.use; | |
669 | tag.s.addr = l2c_tadx_tag.s.tag; | |
58f07778 | 670 | } else { |
b8db85b5 DD |
671 | union __cvmx_l2c_tag tmp_tag; |
672 | /* __read_l2_tag is intended for internal use only */ | |
673 | tmp_tag = __read_l2_tag(association, index); | |
674 | ||
675 | /* | |
676 | * Convert all tag structure types to generic version, | |
677 | * as it can represent all models. | |
678 | */ | |
679 | if (OCTEON_IS_MODEL(OCTEON_CN58XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) { | |
680 | tag.s.V = tmp_tag.cn58xx.V; | |
681 | tag.s.D = tmp_tag.cn58xx.D; | |
682 | tag.s.L = tmp_tag.cn58xx.L; | |
683 | tag.s.U = tmp_tag.cn58xx.U; | |
684 | tag.s.addr = tmp_tag.cn58xx.addr; | |
685 | } else if (OCTEON_IS_MODEL(OCTEON_CN38XX)) { | |
686 | tag.s.V = tmp_tag.cn38xx.V; | |
687 | tag.s.D = tmp_tag.cn38xx.D; | |
688 | tag.s.L = tmp_tag.cn38xx.L; | |
689 | tag.s.U = tmp_tag.cn38xx.U; | |
690 | tag.s.addr = tmp_tag.cn38xx.addr; | |
691 | } else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) { | |
692 | tag.s.V = tmp_tag.cn31xx.V; | |
693 | tag.s.D = tmp_tag.cn31xx.D; | |
694 | tag.s.L = tmp_tag.cn31xx.L; | |
695 | tag.s.U = tmp_tag.cn31xx.U; | |
696 | tag.s.addr = tmp_tag.cn31xx.addr; | |
697 | } else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) { | |
698 | tag.s.V = tmp_tag.cn30xx.V; | |
699 | tag.s.D = tmp_tag.cn30xx.D; | |
700 | tag.s.L = tmp_tag.cn30xx.L; | |
701 | tag.s.U = tmp_tag.cn30xx.U; | |
702 | tag.s.addr = tmp_tag.cn30xx.addr; | |
703 | } else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) { | |
704 | tag.s.V = tmp_tag.cn50xx.V; | |
705 | tag.s.D = tmp_tag.cn50xx.D; | |
706 | tag.s.L = tmp_tag.cn50xx.L; | |
707 | tag.s.U = tmp_tag.cn50xx.U; | |
708 | tag.s.addr = tmp_tag.cn50xx.addr; | |
709 | } else { | |
710 | cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__); | |
711 | } | |
58f07778 | 712 | } |
58f07778 DD |
713 | return tag; |
714 | } | |
715 | ||
716 | uint32_t cvmx_l2c_address_to_index(uint64_t addr) | |
717 | { | |
718 | uint64_t idx = addr >> CVMX_L2C_IDX_ADDR_SHIFT; | |
b8db85b5 | 719 | int indxalias = 0; |
58f07778 | 720 | |
b8db85b5 DD |
721 | if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) { |
722 | union cvmx_l2c_ctl l2c_ctl; | |
723 | l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL); | |
724 | indxalias = !l2c_ctl.s.disidxalias; | |
725 | } else { | |
726 | union cvmx_l2c_cfg l2c_cfg; | |
727 | l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG); | |
728 | indxalias = l2c_cfg.s.idxalias; | |
729 | } | |
730 | ||
731 | if (indxalias) { | |
732 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { | |
733 | uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7; | |
734 | idx ^= idx / cvmx_l2c_get_num_sets(); | |
735 | idx ^= a_14_12; | |
736 | } else { | |
737 | idx ^= ((addr & CVMX_L2C_ALIAS_MASK) >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT); | |
738 | } | |
58f07778 DD |
739 | } |
740 | idx &= CVMX_L2C_IDX_MASK; | |
741 | return idx; | |
742 | } | |
743 | ||
744 | int cvmx_l2c_get_cache_size_bytes(void) | |
745 | { | |
746 | return cvmx_l2c_get_num_sets() * cvmx_l2c_get_num_assoc() * | |
747 | CVMX_CACHE_LINE_SIZE; | |
748 | } | |
749 | ||
750 | /** | |
751 | * Return log base 2 of the number of sets in the L2 cache | |
752 | * Returns | |
753 | */ | |
754 | int cvmx_l2c_get_set_bits(void) | |
755 | { | |
756 | int l2_set_bits; | |
757 | if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX)) | |
758 | l2_set_bits = 11; /* 2048 sets */ | |
b8db85b5 | 759 | else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX)) |
58f07778 | 760 | l2_set_bits = 10; /* 1024 sets */ |
b8db85b5 | 761 | else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) |
58f07778 DD |
762 | l2_set_bits = 9; /* 512 sets */ |
763 | else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) | |
764 | l2_set_bits = 8; /* 256 sets */ | |
765 | else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) | |
766 | l2_set_bits = 7; /* 128 sets */ | |
767 | else { | |
768 | cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__); | |
769 | l2_set_bits = 11; /* 2048 sets */ | |
770 | } | |
771 | return l2_set_bits; | |
58f07778 DD |
772 | } |
773 | ||
774 | /* Return the number of sets in the L2 Cache */ | |
775 | int cvmx_l2c_get_num_sets(void) | |
776 | { | |
777 | return 1 << cvmx_l2c_get_set_bits(); | |
778 | } | |
779 | ||
780 | /* Return the number of associations in the L2 Cache */ | |
781 | int cvmx_l2c_get_num_assoc(void) | |
782 | { | |
783 | int l2_assoc; | |
784 | if (OCTEON_IS_MODEL(OCTEON_CN56XX) || | |
785 | OCTEON_IS_MODEL(OCTEON_CN52XX) || | |
786 | OCTEON_IS_MODEL(OCTEON_CN58XX) || | |
b8db85b5 DD |
787 | OCTEON_IS_MODEL(OCTEON_CN50XX) || |
788 | OCTEON_IS_MODEL(OCTEON_CN38XX)) | |
58f07778 | 789 | l2_assoc = 8; |
b8db85b5 DD |
790 | else if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
791 | l2_assoc = 16; | |
58f07778 DD |
792 | else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || |
793 | OCTEON_IS_MODEL(OCTEON_CN30XX)) | |
794 | l2_assoc = 4; | |
795 | else { | |
796 | cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__); | |
797 | l2_assoc = 8; | |
798 | } | |
799 | ||
800 | /* Check to see if part of the cache is disabled */ | |
b8db85b5 DD |
801 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
802 | union cvmx_mio_fus_dat3 mio_fus_dat3; | |
803 | ||
804 | mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3); | |
805 | /* | |
806 | * cvmx_mio_fus_dat3.s.l2c_crip fuses map as follows | |
807 | * <2> will be not used for 63xx | |
808 | * <1> disables 1/2 ways | |
809 | * <0> disables 1/4 ways | |
810 | * They are cumulative, so for 63xx: | |
811 | * <1> <0> | |
812 | * 0 0 16-way 2MB cache | |
813 | * 0 1 12-way 1.5MB cache | |
814 | * 1 0 8-way 1MB cache | |
815 | * 1 1 4-way 512KB cache | |
816 | */ | |
817 | ||
818 | if (mio_fus_dat3.s.l2c_crip == 3) | |
819 | l2_assoc = 4; | |
820 | else if (mio_fus_dat3.s.l2c_crip == 2) | |
821 | l2_assoc = 8; | |
822 | else if (mio_fus_dat3.s.l2c_crip == 1) | |
823 | l2_assoc = 12; | |
824 | } else { | |
825 | union cvmx_l2d_fus3 val; | |
826 | val.u64 = cvmx_read_csr(CVMX_L2D_FUS3); | |
827 | /* | |
828 | * Using shifts here, as bit position names are | |
829 | * different for each model but they all mean the | |
830 | * same. | |
831 | */ | |
832 | if ((val.u64 >> 35) & 0x1) | |
833 | l2_assoc = l2_assoc >> 2; | |
834 | else if ((val.u64 >> 34) & 0x1) | |
835 | l2_assoc = l2_assoc >> 1; | |
836 | } | |
58f07778 DD |
837 | return l2_assoc; |
838 | } | |
839 | ||
840 | /** | |
841 | * Flush a line from the L2 cache | |
842 | * This should only be called from one core at a time, as this routine | |
843 | * sets the core to the 'debug' core in order to flush the line. | |
844 | * | |
845 | * @assoc: Association (or way) to flush | |
846 | * @index: Index to flush | |
847 | */ | |
848 | void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index) | |
849 | { | |
b8db85b5 DD |
850 | /* Check the range of the index. */ |
851 | if (index > (uint32_t)cvmx_l2c_get_num_sets()) { | |
852 | cvmx_dprintf("ERROR: cvmx_l2c_flush_line index out of range.\n"); | |
853 | return; | |
854 | } | |
58f07778 | 855 | |
b8db85b5 DD |
856 | /* Check the range of association. */ |
857 | if (assoc > (uint32_t)cvmx_l2c_get_num_assoc()) { | |
858 | cvmx_dprintf("ERROR: cvmx_l2c_flush_line association out of range.\n"); | |
859 | return; | |
860 | } | |
58f07778 | 861 | |
b8db85b5 DD |
862 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
863 | uint64_t address; | |
864 | /* Create the address based on index and association. | |
865 | * Bits<20:17> select the way of the cache block involved in | |
866 | * the operation | |
867 | * Bits<16:7> of the effect address select the index | |
868 | */ | |
869 | address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
870 | (assoc << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) | | |
871 | (index << CVMX_L2C_IDX_ADDR_SHIFT)); | |
872 | CVMX_CACHE_WBIL2I(address, 0); | |
873 | } else { | |
874 | union cvmx_l2c_dbg l2cdbg; | |
875 | ||
876 | l2cdbg.u64 = 0; | |
877 | if (!OCTEON_IS_MODEL(OCTEON_CN30XX)) | |
878 | l2cdbg.s.ppnum = cvmx_get_core_num(); | |
879 | l2cdbg.s.finv = 1; | |
880 | ||
881 | l2cdbg.s.set = assoc; | |
882 | cvmx_spinlock_lock(&cvmx_l2c_spinlock); | |
883 | /* | |
884 | * Enter debug mode, and make sure all other writes | |
885 | * complete before we enter debug mode | |
886 | */ | |
887 | CVMX_SYNC; | |
888 | cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64); | |
889 | cvmx_read_csr(CVMX_L2C_DBG); | |
890 | ||
891 | CVMX_PREPARE_FOR_STORE(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
892 | index * CVMX_CACHE_LINE_SIZE), | |
893 | 0); | |
894 | /* Exit debug mode */ | |
895 | CVMX_SYNC; | |
896 | cvmx_write_csr(CVMX_L2C_DBG, 0); | |
897 | cvmx_read_csr(CVMX_L2C_DBG); | |
898 | cvmx_spinlock_unlock(&cvmx_l2c_spinlock); | |
899 | } | |
58f07778 | 900 | } |