]>
Commit | Line | Data |
---|---|---|
53492b1d GS |
1 | /* |
2 | * IBM System z Huge TLB Page Support for Kernel. | |
3 | * | |
4 | * Copyright IBM Corp. 2008 | |
5 | * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> | |
6 | */ | |
7 | ||
8 | #ifndef _ASM_S390_HUGETLB_H | |
9 | #define _ASM_S390_HUGETLB_H | |
10 | ||
11 | #include <asm/page.h> | |
12 | #include <asm/pgtable.h> | |
13 | ||
14 | ||
15 | #define is_hugepage_only_range(mm, addr, len) 0 | |
16 | #define hugetlb_free_pgd_range free_pgd_range | |
17 | ||
18 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | |
19 | pte_t *ptep, pte_t pte); | |
20 | ||
21 | /* | |
22 | * If the arch doesn't supply something else, assume that hugepage | |
23 | * size aligned regions are ok without further preparation. | |
24 | */ | |
a5516438 AK |
25 | static inline int prepare_hugepage_range(struct file *file, |
26 | unsigned long addr, unsigned long len) | |
53492b1d GS |
27 | { |
28 | if (len & ~HPAGE_MASK) | |
29 | return -EINVAL; | |
30 | if (addr & ~HPAGE_MASK) | |
31 | return -EINVAL; | |
32 | return 0; | |
33 | } | |
34 | ||
35 | #define hugetlb_prefault_arch_hook(mm) do { } while (0) | |
36 | ||
37 | int arch_prepare_hugepage(struct page *page); | |
38 | void arch_release_hugepage(struct page *page); | |
39 | ||
40 | static inline pte_t pte_mkhuge(pte_t pte) | |
41 | { | |
42 | /* | |
43 | * PROT_NONE needs to be remapped from the pte type to the ste type. | |
44 | * The HW invalid bit is also different for pte and ste. The pte | |
45 | * invalid bit happens to be the same as the ste _SEGMENT_ENTRY_LARGE | |
46 | * bit, so we don't have to clear it. | |
47 | */ | |
48 | if (pte_val(pte) & _PAGE_INVALID) { | |
49 | if (pte_val(pte) & _PAGE_SWT) | |
50 | pte_val(pte) |= _HPAGE_TYPE_NONE; | |
51 | pte_val(pte) |= _SEGMENT_ENTRY_INV; | |
52 | } | |
53 | /* | |
54 | * Clear SW pte bits SWT and SWX, there are no SW bits in a segment | |
55 | * table entry. | |
56 | */ | |
57 | pte_val(pte) &= ~(_PAGE_SWT | _PAGE_SWX); | |
58 | /* | |
59 | * Also set the change-override bit because we don't need dirty bit | |
60 | * tracking for hugetlbfs pages. | |
61 | */ | |
62 | pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO); | |
63 | return pte; | |
64 | } | |
65 | ||
66 | static inline pte_t huge_pte_wrprotect(pte_t pte) | |
67 | { | |
68 | pte_val(pte) |= _PAGE_RO; | |
69 | return pte; | |
70 | } | |
71 | ||
72 | static inline int huge_pte_none(pte_t pte) | |
73 | { | |
74 | return (pte_val(pte) & _SEGMENT_ENTRY_INV) && | |
75 | !(pte_val(pte) & _SEGMENT_ENTRY_RO); | |
76 | } | |
77 | ||
78 | static inline pte_t huge_ptep_get(pte_t *ptep) | |
79 | { | |
80 | pte_t pte = *ptep; | |
81 | unsigned long mask; | |
82 | ||
83 | if (!MACHINE_HAS_HPAGE) { | |
84 | ptep = (pte_t *) (pte_val(pte) & _SEGMENT_ENTRY_ORIGIN); | |
85 | if (ptep) { | |
86 | mask = pte_val(pte) & | |
87 | (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO); | |
88 | pte = pte_mkhuge(*ptep); | |
89 | pte_val(pte) |= mask; | |
90 | } | |
91 | } | |
92 | return pte; | |
93 | } | |
94 | ||
95 | static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | |
96 | unsigned long addr, pte_t *ptep) | |
97 | { | |
98 | pte_t pte = huge_ptep_get(ptep); | |
99 | ||
050eef36 | 100 | mm->context.flush_mm = 1; |
53492b1d GS |
101 | pmd_clear((pmd_t *) ptep); |
102 | return pte; | |
103 | } | |
104 | ||
105 | static inline void __pmd_csp(pmd_t *pmdp) | |
106 | { | |
107 | register unsigned long reg2 asm("2") = pmd_val(*pmdp); | |
108 | register unsigned long reg3 asm("3") = pmd_val(*pmdp) | | |
109 | _SEGMENT_ENTRY_INV; | |
110 | register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5; | |
111 | ||
112 | asm volatile( | |
113 | " csp %1,%3" | |
114 | : "=m" (*pmdp) | |
115 | : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc"); | |
116 | pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY; | |
117 | } | |
118 | ||
119 | static inline void __pmd_idte(unsigned long address, pmd_t *pmdp) | |
120 | { | |
121 | unsigned long sto = (unsigned long) pmdp - | |
122 | pmd_index(address) * sizeof(pmd_t); | |
123 | ||
124 | if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) { | |
125 | asm volatile( | |
126 | " .insn rrf,0xb98e0000,%2,%3,0,0" | |
127 | : "=m" (*pmdp) | |
128 | : "m" (*pmdp), "a" (sto), | |
129 | "a" ((address & HPAGE_MASK)) | |
130 | ); | |
131 | } | |
132 | pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY; | |
133 | } | |
134 | ||
135 | static inline void huge_ptep_invalidate(struct mm_struct *mm, | |
136 | unsigned long address, pte_t *ptep) | |
137 | { | |
138 | pmd_t *pmdp = (pmd_t *) ptep; | |
139 | ||
140 | if (!MACHINE_HAS_IDTE) { | |
141 | __pmd_csp(pmdp); | |
142 | if (mm->context.noexec) { | |
143 | pmdp = get_shadow_table(pmdp); | |
144 | __pmd_csp(pmdp); | |
145 | } | |
146 | return; | |
147 | } | |
148 | ||
149 | __pmd_idte(address, pmdp); | |
150 | if (mm->context.noexec) { | |
151 | pmdp = get_shadow_table(pmdp); | |
152 | __pmd_idte(address, pmdp); | |
153 | } | |
154 | return; | |
155 | } | |
156 | ||
157 | #define huge_ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \ | |
158 | ({ \ | |
159 | int __changed = !pte_same(huge_ptep_get(__ptep), __entry); \ | |
160 | if (__changed) { \ | |
161 | huge_ptep_invalidate((__vma)->vm_mm, __addr, __ptep); \ | |
162 | set_huge_pte_at((__vma)->vm_mm, __addr, __ptep, __entry); \ | |
163 | } \ | |
164 | __changed; \ | |
165 | }) | |
166 | ||
167 | #define huge_ptep_set_wrprotect(__mm, __addr, __ptep) \ | |
168 | ({ \ | |
169 | pte_t __pte = huge_ptep_get(__ptep); \ | |
170 | if (pte_write(__pte)) { \ | |
050eef36 MS |
171 | (__mm)->context.flush_mm = 1; \ |
172 | if (atomic_read(&(__mm)->context.attach_count) > 1 || \ | |
53492b1d GS |
173 | (__mm) != current->active_mm) \ |
174 | huge_ptep_invalidate(__mm, __addr, __ptep); \ | |
175 | set_huge_pte_at(__mm, __addr, __ptep, \ | |
176 | huge_pte_wrprotect(__pte)); \ | |
177 | } \ | |
178 | }) | |
179 | ||
180 | static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, | |
181 | unsigned long address, pte_t *ptep) | |
182 | { | |
183 | huge_ptep_invalidate(vma->vm_mm, address, ptep); | |
184 | } | |
185 | ||
186 | #endif /* _ASM_S390_HUGETLB_H */ |