]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * ppc64 MMU hashtable management routines | |
3 | * | |
3c726f8d | 4 | * (c) Copyright IBM Corp. 2003, 2005 |
1da177e4 LT |
5 | * |
6 | * Maintained by: Benjamin Herrenschmidt | |
7 | * <benh@kernel.crashing.org> | |
8 | * | |
9 | * This file is covered by the GNU Public Licence v2 as | |
10 | * described in the kernel's COPYING file. | |
11 | */ | |
12 | ||
ab1f9dac | 13 | #include <asm/reg.h> |
1da177e4 LT |
14 | #include <asm/pgtable.h> |
15 | #include <asm/mmu.h> | |
16 | #include <asm/page.h> | |
17 | #include <asm/types.h> | |
18 | #include <asm/ppc_asm.h> | |
0013a854 | 19 | #include <asm/asm-offsets.h> |
1da177e4 LT |
20 | #include <asm/cputable.h> |
21 | ||
22 | .text | |
23 | ||
24 | /* | |
25 | * Stackframe: | |
26 | * | |
27 | * +-> Back chain (SP + 256) | |
28 | * | General register save area (SP + 112) | |
29 | * | Parameter save area (SP + 48) | |
30 | * | TOC save area (SP + 40) | |
31 | * | link editor doubleword (SP + 32) | |
32 | * | compiler doubleword (SP + 24) | |
33 | * | LR save area (SP + 16) | |
34 | * | CR save area (SP + 8) | |
35 | * SP ---> +-- Back chain (SP + 0) | |
36 | */ | |
37 | #define STACKFRAMESIZE 256 | |
38 | ||
39 | /* Save parameters offsets */ | |
40 | #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) | |
41 | ||
42 | /* Save non-volatile offsets */ | |
43 | #define STK_REG(i) (112 + ((i)-14)*8) | |
44 | ||
3c726f8d BH |
45 | |
46 | #ifndef CONFIG_PPC_64K_PAGES | |
47 | ||
48 | /***************************************************************************** | |
49 | * * | |
50 | * 4K SW & 4K HW pages implementation * | |
51 | * * | |
52 | *****************************************************************************/ | |
53 | ||
54 | ||
1da177e4 | 55 | /* |
3c726f8d | 56 | * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, |
1189be65 | 57 | * pte_t *ptep, unsigned long trap, int local, int ssize) |
1da177e4 | 58 | * |
3c726f8d | 59 | * Adds a 4K page to the hash table in a segment of 4K pages only |
1da177e4 LT |
60 | */ |
61 | ||
3c726f8d | 62 | _GLOBAL(__hash_page_4K) |
1da177e4 LT |
63 | mflr r0 |
64 | std r0,16(r1) | |
65 | stdu r1,-STACKFRAMESIZE(r1) | |
66 | /* Save all params that we need after a function call */ | |
67 | std r6,STK_PARM(r6)(r1) | |
68 | std r8,STK_PARM(r8)(r1) | |
1189be65 | 69 | std r9,STK_PARM(r9)(r1) |
1da177e4 | 70 | |
1da177e4 LT |
71 | /* Save non-volatile registers. |
72 | * r31 will hold "old PTE" | |
73 | * r30 is "new PTE" | |
74 | * r29 is "va" | |
75 | * r28 is a hash value | |
76 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
77 | */ | |
78 | std r27,STK_REG(r27)(r1) | |
79 | std r28,STK_REG(r28)(r1) | |
80 | std r29,STK_REG(r29)(r1) | |
81 | std r30,STK_REG(r30)(r1) | |
82 | std r31,STK_REG(r31)(r1) | |
83 | ||
84 | /* Step 1: | |
85 | * | |
86 | * Check permissions, atomically mark the linux PTE busy | |
87 | * and hashed. | |
88 | */ | |
89 | 1: | |
90 | ldarx r31,0,r6 | |
91 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
92 | andc. r0,r4,r31 | |
93 | bne- htab_wrong_access | |
94 | /* Check if PTE is busy */ | |
95 | andi. r0,r31,_PAGE_BUSY | |
d03853d5 OJ |
96 | /* If so, just bail out and refault if needed. Someone else |
97 | * is changing this PTE anyway and might hash it. | |
98 | */ | |
3c726f8d BH |
99 | bne- htab_bail_ok |
100 | ||
1da177e4 LT |
101 | /* Prepare new PTE value (turn access RW into DIRTY, then |
102 | * add BUSY,HASHPTE and ACCESSED) | |
103 | */ | |
104 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
105 | or r30,r30,r31 | |
106 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
107 | /* Write the linux PTE atomically (setting busy) */ | |
108 | stdcx. r30,0,r6 | |
109 | bne- 1b | |
110 | isync | |
111 | ||
112 | /* Step 2: | |
113 | * | |
114 | * Insert/Update the HPTE in the hash table. At this point, | |
115 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
116 | */ | |
117 | ||
1189be65 PM |
118 | BEGIN_FTR_SECTION |
119 | cmpdi r9,0 /* check segment size */ | |
120 | bne 3f | |
121 | END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) | |
1da177e4 LT |
122 | /* Calc va and put it in r29 */ |
123 | rldicr r29,r5,28,63-28 | |
124 | rldicl r3,r3,0,36 | |
125 | or r29,r3,r29 | |
126 | ||
127 | /* Calculate hash value for primary slot and store it in r28 */ | |
128 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
129 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
130 | xor r28,r5,r0 | |
1189be65 PM |
131 | b 4f |
132 | ||
133 | 3: /* Calc VA and hash in r29 and r28 for 1T segment */ | |
134 | sldi r29,r5,40 /* vsid << 40 */ | |
135 | clrldi r3,r3,24 /* ea & 0xffffffffff */ | |
136 | rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ | |
137 | clrldi r5,r5,40 /* vsid & 0xffffff */ | |
138 | rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ | |
139 | xor r28,r28,r5 | |
140 | or r29,r3,r29 /* VA */ | |
141 | xor r28,r28,r0 /* hash */ | |
1da177e4 LT |
142 | |
143 | /* Convert linux PTE bits into HW equivalents */ | |
1189be65 | 144 | 4: andi. r3,r30,0x1fe /* Get basic set of flags */ |
3c726f8d | 145 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
1da177e4 LT |
146 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
147 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
3c726f8d | 148 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ |
1da177e4 LT |
149 | andc r0,r30,r0 /* r0 = pte & ~r0 */ |
150 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 151 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
1da177e4 LT |
152 | |
153 | /* We eventually do the icache sync here (maybe inline that | |
154 | * code rather than call a C function...) | |
155 | */ | |
1da177e4 LT |
156 | BEGIN_FTR_SECTION |
157 | mr r4,r30 | |
158 | mr r5,r7 | |
159 | bl .hash_page_do_lazy_icache | |
8913ca1c | 160 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) |
1da177e4 LT |
161 | |
162 | /* At this point, r3 contains new PP bits, save them in | |
163 | * place of "access" in the param area (sic) | |
164 | */ | |
165 | std r3,STK_PARM(r4)(r1) | |
166 | ||
167 | /* Get htab_hash_mask */ | |
168 | ld r4,htab_hash_mask@got(2) | |
169 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
170 | ||
171 | /* Check if we may already be in the hashtable, in this case, we | |
172 | * go to out-of-line code to try to modify the HPTE | |
173 | */ | |
174 | andi. r0,r31,_PAGE_HASHPTE | |
175 | bne htab_modify_pte | |
176 | ||
177 | htab_insert_pte: | |
178 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
179 | * add _PAGE_HASHPTE | |
180 | */ | |
181 | lis r0,_PAGE_HPTEFLAGS@h | |
182 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
183 | andc r30,r30,r0 | |
184 | ori r30,r30,_PAGE_HASHPTE | |
185 | ||
3c726f8d BH |
186 | /* physical address r5 */ |
187 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
188 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
189 | |
190 | /* Calculate primary group hash */ | |
191 | and r0,r28,r27 | |
3c726f8d | 192 | rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ |
1da177e4 LT |
193 | |
194 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 195 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 196 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
197 | li r7,0 /* !bolted, !secondary */ |
198 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 199 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
1da177e4 | 200 | _GLOBAL(htab_call_hpte_insert1) |
3c726f8d | 201 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
202 | cmpdi 0,r3,0 |
203 | bge htab_pte_insert_ok /* Insertion successful */ | |
204 | cmpdi 0,r3,-2 /* Critical failure */ | |
205 | beq- htab_pte_insert_failure | |
206 | ||
207 | /* Now try secondary slot */ | |
208 | ||
3c726f8d BH |
209 | /* physical address r5 */ |
210 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
211 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
212 | |
213 | /* Calculate secondary group hash */ | |
214 | andc r0,r27,r28 | |
215 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
216 | ||
217 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 218 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 219 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
220 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ |
221 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 222 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
1da177e4 | 223 | _GLOBAL(htab_call_hpte_insert2) |
3c726f8d | 224 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
225 | cmpdi 0,r3,0 |
226 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
227 | cmpdi 0,r3,-2 /* Critical failure */ | |
228 | beq- htab_pte_insert_failure | |
229 | ||
230 | /* Both are full, we need to evict something */ | |
231 | mftb r0 | |
232 | /* Pick a random group based on TB */ | |
233 | andi. r0,r0,1 | |
234 | mr r5,r28 | |
235 | bne 2f | |
236 | not r5,r5 | |
237 | 2: and r0,r5,r27 | |
238 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
239 | /* Call ppc_md.hpte_remove */ | |
240 | _GLOBAL(htab_call_hpte_remove) | |
3c726f8d | 241 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
242 | |
243 | /* Try all again */ | |
244 | b htab_insert_pte | |
245 | ||
3c726f8d | 246 | htab_bail_ok: |
d03853d5 | 247 | li r3,0 |
3c726f8d | 248 | b htab_bail |
d03853d5 | 249 | |
1da177e4 LT |
250 | htab_pte_insert_ok: |
251 | /* Insert slot number & secondary bit in PTE */ | |
252 | rldimi r30,r3,12,63-15 | |
253 | ||
254 | /* Write out the PTE with a normal write | |
255 | * (maybe add eieio may be good still ?) | |
256 | */ | |
257 | htab_write_out_pte: | |
258 | ld r6,STK_PARM(r6)(r1) | |
259 | std r30,0(r6) | |
260 | li r3, 0 | |
3c726f8d | 261 | htab_bail: |
1da177e4 LT |
262 | ld r27,STK_REG(r27)(r1) |
263 | ld r28,STK_REG(r28)(r1) | |
264 | ld r29,STK_REG(r29)(r1) | |
265 | ld r30,STK_REG(r30)(r1) | |
266 | ld r31,STK_REG(r31)(r1) | |
267 | addi r1,r1,STACKFRAMESIZE | |
268 | ld r0,16(r1) | |
269 | mtlr r0 | |
270 | blr | |
271 | ||
272 | htab_modify_pte: | |
273 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
274 | mr r4,r3 | |
275 | rlwinm r3,r31,32-12,29,31 | |
276 | ||
277 | /* Secondary group ? if yes, get a inverted hash value */ | |
278 | mr r5,r28 | |
279 | andi. r0,r31,_PAGE_SECONDARY | |
280 | beq 1f | |
281 | not r5,r5 | |
282 | 1: | |
283 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
284 | and r0,r5,r27 | |
285 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
286 | add r3,r0,r3 /* add slot idx */ | |
287 | ||
288 | /* Call ppc_md.hpte_updatepp */ | |
289 | mr r5,r29 /* va */ | |
3c726f8d | 290 | li r6,MMU_PAGE_4K /* page size */ |
1189be65 PM |
291 | ld r7,STK_PARM(r9)(r1) /* segment size */ |
292 | ld r8,STK_PARM(r8)(r1) /* get "local" param */ | |
1da177e4 | 293 | _GLOBAL(htab_call_hpte_updatepp) |
3c726f8d | 294 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
295 | |
296 | /* if we failed because typically the HPTE wasn't really here | |
297 | * we try an insertion. | |
298 | */ | |
299 | cmpdi 0,r3,-1 | |
300 | beq- htab_insert_pte | |
301 | ||
302 | /* Clear the BUSY bit and Write out the PTE */ | |
303 | li r0,_PAGE_BUSY | |
304 | andc r30,r30,r0 | |
305 | b htab_write_out_pte | |
306 | ||
307 | htab_wrong_access: | |
308 | /* Bail out clearing reservation */ | |
309 | stdcx. r31,0,r6 | |
310 | li r3,1 | |
3c726f8d BH |
311 | b htab_bail |
312 | ||
313 | htab_pte_insert_failure: | |
314 | /* Bail out restoring old PTE */ | |
315 | ld r6,STK_PARM(r6)(r1) | |
316 | std r31,0(r6) | |
317 | li r3,-1 | |
318 | b htab_bail | |
319 | ||
320 | ||
321 | #else /* CONFIG_PPC_64K_PAGES */ | |
322 | ||
323 | ||
324 | /***************************************************************************** | |
325 | * * | |
326 | * 64K SW & 4K or 64K HW in a 4K segment pages implementation * | |
327 | * * | |
328 | *****************************************************************************/ | |
329 | ||
330 | /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, | |
fa28237c PM |
331 | * pte_t *ptep, unsigned long trap, int local, int ssize, |
332 | * int subpg_prot) | |
3c726f8d BH |
333 | */ |
334 | ||
335 | /* | |
336 | * For now, we do NOT implement Admixed pages | |
337 | */ | |
338 | _GLOBAL(__hash_page_4K) | |
339 | mflr r0 | |
340 | std r0,16(r1) | |
341 | stdu r1,-STACKFRAMESIZE(r1) | |
342 | /* Save all params that we need after a function call */ | |
343 | std r6,STK_PARM(r6)(r1) | |
344 | std r8,STK_PARM(r8)(r1) | |
1189be65 | 345 | std r9,STK_PARM(r9)(r1) |
3c726f8d | 346 | |
3c726f8d BH |
347 | /* Save non-volatile registers. |
348 | * r31 will hold "old PTE" | |
349 | * r30 is "new PTE" | |
350 | * r29 is "va" | |
351 | * r28 is a hash value | |
352 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
353 | * r26 is the hidx mask | |
354 | * r25 is the index in combo page | |
355 | */ | |
356 | std r25,STK_REG(r25)(r1) | |
357 | std r26,STK_REG(r26)(r1) | |
358 | std r27,STK_REG(r27)(r1) | |
359 | std r28,STK_REG(r28)(r1) | |
360 | std r29,STK_REG(r29)(r1) | |
361 | std r30,STK_REG(r30)(r1) | |
362 | std r31,STK_REG(r31)(r1) | |
363 | ||
364 | /* Step 1: | |
365 | * | |
366 | * Check permissions, atomically mark the linux PTE busy | |
367 | * and hashed. | |
368 | */ | |
369 | 1: | |
370 | ldarx r31,0,r6 | |
371 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
372 | andc. r0,r4,r31 | |
373 | bne- htab_wrong_access | |
374 | /* Check if PTE is busy */ | |
375 | andi. r0,r31,_PAGE_BUSY | |
376 | /* If so, just bail out and refault if needed. Someone else | |
377 | * is changing this PTE anyway and might hash it. | |
378 | */ | |
379 | bne- htab_bail_ok | |
380 | /* Prepare new PTE value (turn access RW into DIRTY, then | |
381 | * add BUSY and ACCESSED) | |
382 | */ | |
383 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
384 | or r30,r30,r31 | |
41743a4e | 385 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED |
bf72aeba | 386 | oris r30,r30,_PAGE_COMBO@h |
3c726f8d BH |
387 | /* Write the linux PTE atomically (setting busy) */ |
388 | stdcx. r30,0,r6 | |
389 | bne- 1b | |
390 | isync | |
391 | ||
392 | /* Step 2: | |
393 | * | |
394 | * Insert/Update the HPTE in the hash table. At this point, | |
395 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
396 | */ | |
397 | ||
398 | /* Load the hidx index */ | |
399 | rldicl r25,r3,64-12,60 | |
400 | ||
1189be65 PM |
401 | BEGIN_FTR_SECTION |
402 | cmpdi r9,0 /* check segment size */ | |
403 | bne 3f | |
404 | END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) | |
3c726f8d BH |
405 | /* Calc va and put it in r29 */ |
406 | rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ | |
407 | rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ | |
1189be65 | 408 | or r29,r3,r29 /* r29 = va */ |
3c726f8d BH |
409 | |
410 | /* Calculate hash value for primary slot and store it in r28 */ | |
411 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
412 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
413 | xor r28,r5,r0 | |
1189be65 PM |
414 | b 4f |
415 | ||
416 | 3: /* Calc VA and hash in r29 and r28 for 1T segment */ | |
417 | sldi r29,r5,40 /* vsid << 40 */ | |
418 | clrldi r3,r3,24 /* ea & 0xffffffffff */ | |
419 | rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ | |
420 | clrldi r5,r5,40 /* vsid & 0xffffff */ | |
421 | rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ | |
422 | xor r28,r28,r5 | |
423 | or r29,r3,r29 /* VA */ | |
424 | xor r28,r28,r0 /* hash */ | |
3c726f8d BH |
425 | |
426 | /* Convert linux PTE bits into HW equivalents */ | |
fa28237c PM |
427 | 4: |
428 | #ifdef CONFIG_PPC_SUBPAGE_PROT | |
429 | andc r10,r30,r10 | |
430 | andi. r3,r10,0x1fe /* Get basic set of flags */ | |
431 | rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
432 | #else | |
433 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
3c726f8d | 434 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
fa28237c PM |
435 | #endif |
436 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | |
3c726f8d BH |
437 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ |
438 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
fa28237c | 439 | andc r0,r3,r0 /* r0 = pte & ~r0 */ |
3c726f8d | 440 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ |
c5cf0e30 | 441 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
3c726f8d BH |
442 | |
443 | /* We eventually do the icache sync here (maybe inline that | |
444 | * code rather than call a C function...) | |
445 | */ | |
446 | BEGIN_FTR_SECTION | |
447 | mr r4,r30 | |
448 | mr r5,r7 | |
449 | bl .hash_page_do_lazy_icache | |
450 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
451 | ||
452 | /* At this point, r3 contains new PP bits, save them in | |
453 | * place of "access" in the param area (sic) | |
454 | */ | |
455 | std r3,STK_PARM(r4)(r1) | |
456 | ||
457 | /* Get htab_hash_mask */ | |
458 | ld r4,htab_hash_mask@got(2) | |
459 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
460 | ||
461 | /* Check if we may already be in the hashtable, in this case, we | |
462 | * go to out-of-line code to try to modify the HPTE. We look for | |
463 | * the bit at (1 >> (index + 32)) | |
464 | */ | |
41743a4e | 465 | rldicl. r0,r31,64-12,48 |
3c726f8d BH |
466 | li r26,0 /* Default hidx */ |
467 | beq htab_insert_pte | |
bf72aeba PM |
468 | |
469 | /* | |
470 | * Check if the pte was already inserted into the hash table | |
471 | * as a 64k HW page, and invalidate the 64k HPTE if so. | |
472 | */ | |
473 | andis. r0,r31,_PAGE_COMBO@h | |
474 | beq htab_inval_old_hpte | |
475 | ||
3c726f8d BH |
476 | ld r6,STK_PARM(r6)(r1) |
477 | ori r26,r6,0x8000 /* Load the hidx mask */ | |
478 | ld r26,0(r26) | |
479 | addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ | |
480 | rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ | |
481 | bne htab_modify_pte | |
482 | ||
483 | htab_insert_pte: | |
484 | /* real page number in r5, PTE RPN value + index */ | |
721151d0 PM |
485 | andis. r0,r31,_PAGE_4K_PFN@h |
486 | srdi r5,r31,PTE_RPN_SHIFT | |
487 | bne- htab_special_pfn | |
3c726f8d BH |
488 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT |
489 | add r5,r5,r25 | |
721151d0 | 490 | htab_special_pfn: |
3c726f8d BH |
491 | sldi r5,r5,HW_PAGE_SHIFT |
492 | ||
493 | /* Calculate primary group hash */ | |
494 | and r0,r28,r27 | |
495 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
496 | ||
497 | /* Call ppc_md.hpte_insert */ | |
498 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
499 | mr r4,r29 /* Retreive va */ | |
500 | li r7,0 /* !bolted, !secondary */ | |
501 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 502 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
503 | _GLOBAL(htab_call_hpte_insert1) |
504 | bl . /* patched by htab_finish_init() */ | |
505 | cmpdi 0,r3,0 | |
506 | bge htab_pte_insert_ok /* Insertion successful */ | |
507 | cmpdi 0,r3,-2 /* Critical failure */ | |
508 | beq- htab_pte_insert_failure | |
509 | ||
510 | /* Now try secondary slot */ | |
511 | ||
512 | /* real page number in r5, PTE RPN value + index */ | |
430404ed PM |
513 | andis. r0,r31,_PAGE_4K_PFN@h |
514 | srdi r5,r31,PTE_RPN_SHIFT | |
515 | bne- 3f | |
3c726f8d BH |
516 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT |
517 | add r5,r5,r25 | |
430404ed | 518 | 3: sldi r5,r5,HW_PAGE_SHIFT |
3c726f8d BH |
519 | |
520 | /* Calculate secondary group hash */ | |
521 | andc r0,r27,r28 | |
522 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
523 | ||
524 | /* Call ppc_md.hpte_insert */ | |
525 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
526 | mr r4,r29 /* Retreive va */ | |
527 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
528 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 529 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
530 | _GLOBAL(htab_call_hpte_insert2) |
531 | bl . /* patched by htab_finish_init() */ | |
532 | cmpdi 0,r3,0 | |
533 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
534 | cmpdi 0,r3,-2 /* Critical failure */ | |
535 | beq- htab_pte_insert_failure | |
536 | ||
537 | /* Both are full, we need to evict something */ | |
538 | mftb r0 | |
539 | /* Pick a random group based on TB */ | |
540 | andi. r0,r0,1 | |
541 | mr r5,r28 | |
542 | bne 2f | |
543 | not r5,r5 | |
544 | 2: and r0,r5,r27 | |
545 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
546 | /* Call ppc_md.hpte_remove */ | |
547 | _GLOBAL(htab_call_hpte_remove) | |
548 | bl . /* patched by htab_finish_init() */ | |
549 | ||
550 | /* Try all again */ | |
551 | b htab_insert_pte | |
552 | ||
bf72aeba PM |
553 | /* |
554 | * Call out to C code to invalidate an 64k HW HPTE that is | |
555 | * useless now that the segment has been switched to 4k pages. | |
556 | */ | |
557 | htab_inval_old_hpte: | |
558 | mr r3,r29 /* virtual addr */ | |
559 | mr r4,r31 /* PTE.pte */ | |
560 | li r5,0 /* PTE.hidx */ | |
561 | li r6,MMU_PAGE_64K /* psize */ | |
f6ab0b92 BH |
562 | ld r7,STK_PARM(r9)(r1) /* ssize */ |
563 | ld r8,STK_PARM(r8)(r1) /* local */ | |
bf72aeba | 564 | bl .flush_hash_page |
65ba6cdc PM |
565 | /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */ |
566 | lis r0,_PAGE_HPTE_SUB@h | |
567 | ori r0,r0,_PAGE_HPTE_SUB@l | |
568 | andc r30,r30,r0 | |
bf72aeba PM |
569 | b htab_insert_pte |
570 | ||
3c726f8d BH |
571 | htab_bail_ok: |
572 | li r3,0 | |
573 | b htab_bail | |
574 | ||
575 | htab_pte_insert_ok: | |
576 | /* Insert slot number & secondary bit in PTE second half, | |
577 | * clear _PAGE_BUSY and set approriate HPTE slot bit | |
578 | */ | |
579 | ld r6,STK_PARM(r6)(r1) | |
580 | li r0,_PAGE_BUSY | |
581 | andc r30,r30,r0 | |
582 | /* HPTE SUB bit */ | |
583 | li r0,1 | |
584 | subfic r5,r25,27 /* Must match bit position in */ | |
585 | sld r0,r0,r5 /* pgtable.h */ | |
586 | or r30,r30,r0 | |
587 | /* hindx */ | |
588 | sldi r5,r25,2 | |
589 | sld r3,r3,r5 | |
590 | li r4,0xf | |
591 | sld r4,r4,r5 | |
592 | andc r26,r26,r4 | |
593 | or r26,r26,r3 | |
594 | ori r5,r6,0x8000 | |
595 | std r26,0(r5) | |
596 | lwsync | |
597 | std r30,0(r6) | |
598 | li r3, 0 | |
599 | htab_bail: | |
600 | ld r25,STK_REG(r25)(r1) | |
601 | ld r26,STK_REG(r26)(r1) | |
602 | ld r27,STK_REG(r27)(r1) | |
603 | ld r28,STK_REG(r28)(r1) | |
604 | ld r29,STK_REG(r29)(r1) | |
605 | ld r30,STK_REG(r30)(r1) | |
606 | ld r31,STK_REG(r31)(r1) | |
607 | addi r1,r1,STACKFRAMESIZE | |
608 | ld r0,16(r1) | |
609 | mtlr r0 | |
610 | blr | |
611 | ||
612 | htab_modify_pte: | |
613 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
614 | mr r4,r3 | |
615 | sldi r5,r25,2 | |
616 | srd r3,r26,r5 | |
617 | ||
618 | /* Secondary group ? if yes, get a inverted hash value */ | |
619 | mr r5,r28 | |
620 | andi. r0,r3,0x8 /* page secondary ? */ | |
621 | beq 1f | |
622 | not r5,r5 | |
623 | 1: andi. r3,r3,0x7 /* extract idx alone */ | |
624 | ||
625 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
626 | and r0,r5,r27 | |
627 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
628 | add r3,r0,r3 /* add slot idx */ | |
629 | ||
630 | /* Call ppc_md.hpte_updatepp */ | |
631 | mr r5,r29 /* va */ | |
632 | li r6,MMU_PAGE_4K /* page size */ | |
1189be65 PM |
633 | ld r7,STK_PARM(r9)(r1) /* segment size */ |
634 | ld r8,STK_PARM(r8)(r1) /* get "local" param */ | |
3c726f8d BH |
635 | _GLOBAL(htab_call_hpte_updatepp) |
636 | bl . /* patched by htab_finish_init() */ | |
637 | ||
638 | /* if we failed because typically the HPTE wasn't really here | |
639 | * we try an insertion. | |
640 | */ | |
641 | cmpdi 0,r3,-1 | |
642 | beq- htab_insert_pte | |
643 | ||
644 | /* Clear the BUSY bit and Write out the PTE */ | |
645 | li r0,_PAGE_BUSY | |
646 | andc r30,r30,r0 | |
647 | ld r6,STK_PARM(r6)(r1) | |
648 | std r30,0(r6) | |
649 | li r3,0 | |
650 | b htab_bail | |
651 | ||
652 | htab_wrong_access: | |
653 | /* Bail out clearing reservation */ | |
654 | stdcx. r31,0,r6 | |
655 | li r3,1 | |
656 | b htab_bail | |
1da177e4 LT |
657 | |
658 | htab_pte_insert_failure: | |
659 | /* Bail out restoring old PTE */ | |
660 | ld r6,STK_PARM(r6)(r1) | |
661 | std r31,0(r6) | |
662 | li r3,-1 | |
3c726f8d BH |
663 | b htab_bail |
664 | ||
16c2d476 BH |
665 | #endif /* CONFIG_PPC_64K_PAGES */ |
666 | ||
667 | #ifdef CONFIG_PPC_HAS_HASH_64K | |
3c726f8d BH |
668 | |
669 | /***************************************************************************** | |
670 | * * | |
671 | * 64K SW & 64K HW in a 64K segment pages implementation * | |
672 | * * | |
673 | *****************************************************************************/ | |
674 | ||
675 | _GLOBAL(__hash_page_64K) | |
676 | mflr r0 | |
677 | std r0,16(r1) | |
678 | stdu r1,-STACKFRAMESIZE(r1) | |
679 | /* Save all params that we need after a function call */ | |
680 | std r6,STK_PARM(r6)(r1) | |
681 | std r8,STK_PARM(r8)(r1) | |
1189be65 | 682 | std r9,STK_PARM(r9)(r1) |
3c726f8d | 683 | |
3c726f8d BH |
684 | /* Save non-volatile registers. |
685 | * r31 will hold "old PTE" | |
686 | * r30 is "new PTE" | |
687 | * r29 is "va" | |
688 | * r28 is a hash value | |
689 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
690 | */ | |
691 | std r27,STK_REG(r27)(r1) | |
692 | std r28,STK_REG(r28)(r1) | |
693 | std r29,STK_REG(r29)(r1) | |
694 | std r30,STK_REG(r30)(r1) | |
695 | std r31,STK_REG(r31)(r1) | |
696 | ||
697 | /* Step 1: | |
698 | * | |
699 | * Check permissions, atomically mark the linux PTE busy | |
700 | * and hashed. | |
701 | */ | |
702 | 1: | |
703 | ldarx r31,0,r6 | |
704 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
705 | andc. r0,r4,r31 | |
706 | bne- ht64_wrong_access | |
707 | /* Check if PTE is busy */ | |
708 | andi. r0,r31,_PAGE_BUSY | |
709 | /* If so, just bail out and refault if needed. Someone else | |
710 | * is changing this PTE anyway and might hash it. | |
711 | */ | |
712 | bne- ht64_bail_ok | |
bf72aeba PM |
713 | BEGIN_FTR_SECTION |
714 | /* Check if PTE has the cache-inhibit bit set */ | |
715 | andi. r0,r31,_PAGE_NO_CACHE | |
716 | /* If so, bail out and refault as a 4k page */ | |
717 | bne- ht64_bail_ok | |
718 | END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) | |
3c726f8d | 719 | /* Prepare new PTE value (turn access RW into DIRTY, then |
41743a4e | 720 | * add BUSY and ACCESSED) |
3c726f8d BH |
721 | */ |
722 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
723 | or r30,r30,r31 | |
41743a4e | 724 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED |
3c726f8d BH |
725 | /* Write the linux PTE atomically (setting busy) */ |
726 | stdcx. r30,0,r6 | |
727 | bne- 1b | |
728 | isync | |
729 | ||
730 | /* Step 2: | |
731 | * | |
732 | * Insert/Update the HPTE in the hash table. At this point, | |
733 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
734 | */ | |
735 | ||
1189be65 PM |
736 | BEGIN_FTR_SECTION |
737 | cmpdi r9,0 /* check segment size */ | |
738 | bne 3f | |
739 | END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) | |
3c726f8d BH |
740 | /* Calc va and put it in r29 */ |
741 | rldicr r29,r5,28,63-28 | |
742 | rldicl r3,r3,0,36 | |
743 | or r29,r3,r29 | |
744 | ||
745 | /* Calculate hash value for primary slot and store it in r28 */ | |
746 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
747 | rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ | |
748 | xor r28,r5,r0 | |
1189be65 PM |
749 | b 4f |
750 | ||
751 | 3: /* Calc VA and hash in r29 and r28 for 1T segment */ | |
752 | sldi r29,r5,40 /* vsid << 40 */ | |
753 | clrldi r3,r3,24 /* ea & 0xffffffffff */ | |
754 | rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ | |
755 | clrldi r5,r5,40 /* vsid & 0xffffff */ | |
756 | rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */ | |
757 | xor r28,r28,r5 | |
758 | or r29,r3,r29 /* VA */ | |
759 | xor r28,r28,r0 /* hash */ | |
3c726f8d BH |
760 | |
761 | /* Convert linux PTE bits into HW equivalents */ | |
1189be65 | 762 | 4: andi. r3,r30,0x1fe /* Get basic set of flags */ |
3c726f8d BH |
763 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
764 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
765 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
766 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
767 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | |
768 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 769 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
3c726f8d BH |
770 | |
771 | /* We eventually do the icache sync here (maybe inline that | |
772 | * code rather than call a C function...) | |
773 | */ | |
774 | BEGIN_FTR_SECTION | |
775 | mr r4,r30 | |
776 | mr r5,r7 | |
777 | bl .hash_page_do_lazy_icache | |
778 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
779 | ||
780 | /* At this point, r3 contains new PP bits, save them in | |
781 | * place of "access" in the param area (sic) | |
782 | */ | |
783 | std r3,STK_PARM(r4)(r1) | |
784 | ||
785 | /* Get htab_hash_mask */ | |
786 | ld r4,htab_hash_mask@got(2) | |
787 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
788 | ||
789 | /* Check if we may already be in the hashtable, in this case, we | |
790 | * go to out-of-line code to try to modify the HPTE | |
791 | */ | |
41743a4e | 792 | rldicl. r0,r31,64-12,48 |
3c726f8d BH |
793 | bne ht64_modify_pte |
794 | ||
795 | ht64_insert_pte: | |
796 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
41743a4e | 797 | * add _PAGE_HPTE_SUB0 |
3c726f8d BH |
798 | */ |
799 | lis r0,_PAGE_HPTEFLAGS@h | |
800 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
801 | andc r30,r30,r0 | |
41743a4e BH |
802 | #ifdef CONFIG_PPC_64K_PAGES |
803 | oris r30,r30,_PAGE_HPTE_SUB0@h | |
804 | #else | |
3c726f8d | 805 | ori r30,r30,_PAGE_HASHPTE |
41743a4e | 806 | #endif |
3c726f8d BH |
807 | /* Phyical address in r5 */ |
808 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
809 | sldi r5,r5,PAGE_SHIFT | |
810 | ||
811 | /* Calculate primary group hash */ | |
812 | and r0,r28,r27 | |
813 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
814 | ||
815 | /* Call ppc_md.hpte_insert */ | |
816 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
817 | mr r4,r29 /* Retreive va */ | |
818 | li r7,0 /* !bolted, !secondary */ | |
819 | li r8,MMU_PAGE_64K | |
1189be65 | 820 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
821 | _GLOBAL(ht64_call_hpte_insert1) |
822 | bl . /* patched by htab_finish_init() */ | |
823 | cmpdi 0,r3,0 | |
824 | bge ht64_pte_insert_ok /* Insertion successful */ | |
825 | cmpdi 0,r3,-2 /* Critical failure */ | |
826 | beq- ht64_pte_insert_failure | |
827 | ||
828 | /* Now try secondary slot */ | |
829 | ||
830 | /* Phyical address in r5 */ | |
831 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
832 | sldi r5,r5,PAGE_SHIFT | |
833 | ||
834 | /* Calculate secondary group hash */ | |
835 | andc r0,r27,r28 | |
836 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
837 | ||
838 | /* Call ppc_md.hpte_insert */ | |
839 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
840 | mr r4,r29 /* Retreive va */ | |
841 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
842 | li r8,MMU_PAGE_64K | |
1189be65 | 843 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
844 | _GLOBAL(ht64_call_hpte_insert2) |
845 | bl . /* patched by htab_finish_init() */ | |
846 | cmpdi 0,r3,0 | |
847 | bge+ ht64_pte_insert_ok /* Insertion successful */ | |
848 | cmpdi 0,r3,-2 /* Critical failure */ | |
849 | beq- ht64_pte_insert_failure | |
850 | ||
851 | /* Both are full, we need to evict something */ | |
852 | mftb r0 | |
853 | /* Pick a random group based on TB */ | |
854 | andi. r0,r0,1 | |
855 | mr r5,r28 | |
856 | bne 2f | |
857 | not r5,r5 | |
858 | 2: and r0,r5,r27 | |
859 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
860 | /* Call ppc_md.hpte_remove */ | |
861 | _GLOBAL(ht64_call_hpte_remove) | |
862 | bl . /* patched by htab_finish_init() */ | |
863 | ||
864 | /* Try all again */ | |
865 | b ht64_insert_pte | |
866 | ||
867 | ht64_bail_ok: | |
868 | li r3,0 | |
869 | b ht64_bail | |
870 | ||
871 | ht64_pte_insert_ok: | |
872 | /* Insert slot number & secondary bit in PTE */ | |
873 | rldimi r30,r3,12,63-15 | |
874 | ||
875 | /* Write out the PTE with a normal write | |
876 | * (maybe add eieio may be good still ?) | |
877 | */ | |
878 | ht64_write_out_pte: | |
879 | ld r6,STK_PARM(r6)(r1) | |
880 | std r30,0(r6) | |
881 | li r3, 0 | |
882 | ht64_bail: | |
883 | ld r27,STK_REG(r27)(r1) | |
884 | ld r28,STK_REG(r28)(r1) | |
885 | ld r29,STK_REG(r29)(r1) | |
886 | ld r30,STK_REG(r30)(r1) | |
887 | ld r31,STK_REG(r31)(r1) | |
888 | addi r1,r1,STACKFRAMESIZE | |
889 | ld r0,16(r1) | |
890 | mtlr r0 | |
891 | blr | |
892 | ||
893 | ht64_modify_pte: | |
894 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
895 | mr r4,r3 | |
896 | rlwinm r3,r31,32-12,29,31 | |
897 | ||
898 | /* Secondary group ? if yes, get a inverted hash value */ | |
899 | mr r5,r28 | |
900 | andi. r0,r31,_PAGE_F_SECOND | |
901 | beq 1f | |
902 | not r5,r5 | |
903 | 1: | |
904 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
905 | and r0,r5,r27 | |
906 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
907 | add r3,r0,r3 /* add slot idx */ | |
908 | ||
909 | /* Call ppc_md.hpte_updatepp */ | |
910 | mr r5,r29 /* va */ | |
911 | li r6,MMU_PAGE_64K | |
1189be65 PM |
912 | ld r7,STK_PARM(r9)(r1) /* segment size */ |
913 | ld r8,STK_PARM(r8)(r1) /* get "local" param */ | |
3c726f8d BH |
914 | _GLOBAL(ht64_call_hpte_updatepp) |
915 | bl . /* patched by htab_finish_init() */ | |
916 | ||
917 | /* if we failed because typically the HPTE wasn't really here | |
918 | * we try an insertion. | |
919 | */ | |
920 | cmpdi 0,r3,-1 | |
921 | beq- ht64_insert_pte | |
922 | ||
923 | /* Clear the BUSY bit and Write out the PTE */ | |
924 | li r0,_PAGE_BUSY | |
925 | andc r30,r30,r0 | |
926 | b ht64_write_out_pte | |
927 | ||
928 | ht64_wrong_access: | |
929 | /* Bail out clearing reservation */ | |
930 | stdcx. r31,0,r6 | |
931 | li r3,1 | |
932 | b ht64_bail | |
933 | ||
934 | ht64_pte_insert_failure: | |
935 | /* Bail out restoring old PTE */ | |
936 | ld r6,STK_PARM(r6)(r1) | |
937 | std r31,0(r6) | |
938 | li r3,-1 | |
939 | b ht64_bail | |
940 | ||
941 | ||
16c2d476 | 942 | #endif /* CONFIG_PPC_HAS_HASH_64K */ |
1da177e4 LT |
943 | |
944 | ||
3c726f8d BH |
945 | /***************************************************************************** |
946 | * * | |
947 | * Huge pages implementation is in hugetlbpage.c * | |
948 | * * | |
949 | *****************************************************************************/ |