]>
Commit | Line | Data |
---|---|---|
4c921d4d JS |
1 | /* |
2 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU Lesser General Public License as published by | |
6 | * the Free Software Foundation; either version 2.1 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU Lesser General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU Lesser General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
17 | */ | |
18 | ||
19 | #ifndef __GRU_INSTRUCTIONS_H__ | |
20 | #define __GRU_INSTRUCTIONS_H__ | |
21 | ||
fe5bb6b0 JS |
22 | extern int gru_check_status_proc(void *cb); |
23 | extern int gru_wait_proc(void *cb); | |
24 | extern void gru_wait_abort_proc(void *cb); | |
25 | ||
26 | ||
4c921d4d JS |
27 | |
28 | /* | |
29 | * Architecture dependent functions | |
30 | */ | |
31 | ||
923f7f69 | 32 | #if defined(CONFIG_IA64) |
4c921d4d JS |
33 | #include <linux/compiler.h> |
34 | #include <asm/intrinsics.h> | |
fe5bb6b0 | 35 | #define __flush_cache(p) ia64_fc((unsigned long)p) |
4c921d4d | 36 | /* Use volatile on IA64 to ensure ordering via st4.rel */ |
fe5bb6b0 | 37 | #define gru_ordered_store_int(p, v) \ |
4c921d4d JS |
38 | do { \ |
39 | barrier(); \ | |
40 | *((volatile int *)(p)) = v; /* force st.rel */ \ | |
41 | } while (0) | |
923f7f69 | 42 | #elif defined(CONFIG_X86_64) |
4c921d4d | 43 | #define __flush_cache(p) clflush(p) |
fe5bb6b0 | 44 | #define gru_ordered_store_int(p, v) \ |
4c921d4d JS |
45 | do { \ |
46 | barrier(); \ | |
47 | *(int *)p = v; \ | |
48 | } while (0) | |
49 | #else | |
50 | #error "Unsupported architecture" | |
51 | #endif | |
52 | ||
53 | /* | |
54 | * Control block status and exception codes | |
55 | */ | |
56 | #define CBS_IDLE 0 | |
57 | #define CBS_EXCEPTION 1 | |
58 | #define CBS_ACTIVE 2 | |
59 | #define CBS_CALL_OS 3 | |
60 | ||
61 | /* CB substatus bitmasks */ | |
62 | #define CBSS_MSG_QUEUE_MASK 7 | |
63 | #define CBSS_IMPLICIT_ABORT_ACTIVE_MASK 8 | |
64 | ||
65 | /* CB substatus message queue values (low 3 bits of substatus) */ | |
66 | #define CBSS_NO_ERROR 0 | |
67 | #define CBSS_LB_OVERFLOWED 1 | |
68 | #define CBSS_QLIMIT_REACHED 2 | |
69 | #define CBSS_PAGE_OVERFLOW 3 | |
70 | #define CBSS_AMO_NACKED 4 | |
71 | #define CBSS_PUT_NACKED 5 | |
72 | ||
73 | /* | |
74 | * Structure used to fetch exception detail for CBs that terminate with | |
75 | * CBS_EXCEPTION | |
76 | */ | |
77 | struct control_block_extended_exc_detail { | |
78 | unsigned long cb; | |
79 | int opc; | |
80 | int ecause; | |
81 | int exopc; | |
82 | long exceptdet0; | |
83 | int exceptdet1; | |
cd1334f0 JS |
84 | int cbrstate; |
85 | int cbrexecstatus; | |
4c921d4d JS |
86 | }; |
87 | ||
88 | /* | |
89 | * Instruction formats | |
90 | */ | |
91 | ||
92 | /* | |
93 | * Generic instruction format. | |
94 | * This definition has precise bit field definitions. | |
95 | */ | |
96 | struct gru_instruction_bits { | |
97 | /* DW 0 - low */ | |
98 | unsigned int icmd: 1; | |
99 | unsigned char ima: 3; /* CB_DelRep, unmapped mode */ | |
100 | unsigned char reserved0: 4; | |
101 | unsigned int xtype: 3; | |
102 | unsigned int iaa0: 2; | |
103 | unsigned int iaa1: 2; | |
104 | unsigned char reserved1: 1; | |
105 | unsigned char opc: 8; /* opcode */ | |
106 | unsigned char exopc: 8; /* extended opcode */ | |
107 | /* DW 0 - high */ | |
108 | unsigned int idef2: 22; /* TRi0 */ | |
109 | unsigned char reserved2: 2; | |
110 | unsigned char istatus: 2; | |
111 | unsigned char isubstatus:4; | |
cd1334f0 JS |
112 | unsigned char reserved3: 1; |
113 | unsigned char tlb_fault_color: 1; | |
4c921d4d JS |
114 | /* DW 1 */ |
115 | unsigned long idef4; /* 42 bits: TRi1, BufSize */ | |
116 | /* DW 2-6 */ | |
117 | unsigned long idef1; /* BAddr0 */ | |
118 | unsigned long idef5; /* Nelem */ | |
119 | unsigned long idef6; /* Stride, Operand1 */ | |
120 | unsigned long idef3; /* BAddr1, Value, Operand2 */ | |
121 | unsigned long reserved4; | |
122 | /* DW 7 */ | |
123 | unsigned long avalue; /* AValue */ | |
124 | }; | |
125 | ||
126 | /* | |
127 | * Generic instruction with friendlier names. This format is used | |
128 | * for inline instructions. | |
129 | */ | |
130 | struct gru_instruction { | |
131 | /* DW 0 */ | |
132 | unsigned int op32; /* icmd,xtype,iaa0,ima,opc */ | |
133 | unsigned int tri0; | |
134 | unsigned long tri1_bufsize; /* DW 1 */ | |
135 | unsigned long baddr0; /* DW 2 */ | |
136 | unsigned long nelem; /* DW 3 */ | |
137 | unsigned long op1_stride; /* DW 4 */ | |
138 | unsigned long op2_value_baddr1; /* DW 5 */ | |
139 | unsigned long reserved0; /* DW 6 */ | |
140 | unsigned long avalue; /* DW 7 */ | |
141 | }; | |
142 | ||
143 | /* Some shifts and masks for the low 32 bits of a GRU command */ | |
144 | #define GRU_CB_ICMD_SHFT 0 | |
145 | #define GRU_CB_ICMD_MASK 0x1 | |
146 | #define GRU_CB_XTYPE_SHFT 8 | |
147 | #define GRU_CB_XTYPE_MASK 0x7 | |
148 | #define GRU_CB_IAA0_SHFT 11 | |
149 | #define GRU_CB_IAA0_MASK 0x3 | |
150 | #define GRU_CB_IAA1_SHFT 13 | |
151 | #define GRU_CB_IAA1_MASK 0x3 | |
152 | #define GRU_CB_IMA_SHFT 1 | |
153 | #define GRU_CB_IMA_MASK 0x3 | |
154 | #define GRU_CB_OPC_SHFT 16 | |
155 | #define GRU_CB_OPC_MASK 0xff | |
156 | #define GRU_CB_EXOPC_SHFT 24 | |
157 | #define GRU_CB_EXOPC_MASK 0xff | |
158 | ||
159 | /* GRU instruction opcodes (opc field) */ | |
160 | #define OP_NOP 0x00 | |
161 | #define OP_BCOPY 0x01 | |
162 | #define OP_VLOAD 0x02 | |
163 | #define OP_IVLOAD 0x03 | |
164 | #define OP_VSTORE 0x04 | |
165 | #define OP_IVSTORE 0x05 | |
166 | #define OP_VSET 0x06 | |
167 | #define OP_IVSET 0x07 | |
168 | #define OP_MESQ 0x08 | |
169 | #define OP_GAMXR 0x09 | |
170 | #define OP_GAMIR 0x0a | |
171 | #define OP_GAMIRR 0x0b | |
172 | #define OP_GAMER 0x0c | |
173 | #define OP_GAMERR 0x0d | |
174 | #define OP_BSTORE 0x0e | |
175 | #define OP_VFLUSH 0x0f | |
176 | ||
177 | ||
178 | /* Extended opcodes values (exopc field) */ | |
179 | ||
180 | /* GAMIR - AMOs with implicit operands */ | |
181 | #define EOP_IR_FETCH 0x01 /* Plain fetch of memory */ | |
182 | #define EOP_IR_CLR 0x02 /* Fetch and clear */ | |
183 | #define EOP_IR_INC 0x05 /* Fetch and increment */ | |
184 | #define EOP_IR_DEC 0x07 /* Fetch and decrement */ | |
185 | #define EOP_IR_QCHK1 0x0d /* Queue check, 64 byte msg */ | |
186 | #define EOP_IR_QCHK2 0x0e /* Queue check, 128 byte msg */ | |
187 | ||
188 | /* GAMIRR - Registered AMOs with implicit operands */ | |
189 | #define EOP_IRR_FETCH 0x01 /* Registered fetch of memory */ | |
190 | #define EOP_IRR_CLR 0x02 /* Registered fetch and clear */ | |
191 | #define EOP_IRR_INC 0x05 /* Registered fetch and increment */ | |
192 | #define EOP_IRR_DEC 0x07 /* Registered fetch and decrement */ | |
193 | #define EOP_IRR_DECZ 0x0f /* Registered fetch and decrement, update on zero*/ | |
194 | ||
195 | /* GAMER - AMOs with explicit operands */ | |
196 | #define EOP_ER_SWAP 0x00 /* Exchange argument and memory */ | |
197 | #define EOP_ER_OR 0x01 /* Logical OR with memory */ | |
198 | #define EOP_ER_AND 0x02 /* Logical AND with memory */ | |
199 | #define EOP_ER_XOR 0x03 /* Logical XOR with memory */ | |
200 | #define EOP_ER_ADD 0x04 /* Add value to memory */ | |
201 | #define EOP_ER_CSWAP 0x08 /* Compare with operand2, write operand1 if match*/ | |
202 | #define EOP_ER_CADD 0x0c /* Queue check, operand1*64 byte msg */ | |
203 | ||
204 | /* GAMERR - Registered AMOs with explicit operands */ | |
205 | #define EOP_ERR_SWAP 0x00 /* Exchange argument and memory */ | |
206 | #define EOP_ERR_OR 0x01 /* Logical OR with memory */ | |
207 | #define EOP_ERR_AND 0x02 /* Logical AND with memory */ | |
208 | #define EOP_ERR_XOR 0x03 /* Logical XOR with memory */ | |
209 | #define EOP_ERR_ADD 0x04 /* Add value to memory */ | |
210 | #define EOP_ERR_CSWAP 0x08 /* Compare with operand2, write operand1 if match*/ | |
211 | #define EOP_ERR_EPOLL 0x09 /* Poll for equality */ | |
212 | #define EOP_ERR_NPOLL 0x0a /* Poll for inequality */ | |
213 | ||
214 | /* GAMXR - SGI Arithmetic unit */ | |
215 | #define EOP_XR_CSWAP 0x0b /* Masked compare exchange */ | |
216 | ||
217 | ||
218 | /* Transfer types (xtype field) */ | |
219 | #define XTYPE_B 0x0 /* byte */ | |
220 | #define XTYPE_S 0x1 /* short (2-byte) */ | |
221 | #define XTYPE_W 0x2 /* word (4-byte) */ | |
222 | #define XTYPE_DW 0x3 /* doubleword (8-byte) */ | |
223 | #define XTYPE_CL 0x6 /* cacheline (64-byte) */ | |
224 | ||
225 | ||
226 | /* Instruction access attributes (iaa0, iaa1 fields) */ | |
227 | #define IAA_RAM 0x0 /* normal cached RAM access */ | |
228 | #define IAA_NCRAM 0x2 /* noncoherent RAM access */ | |
229 | #define IAA_MMIO 0x1 /* noncoherent memory-mapped I/O space */ | |
230 | #define IAA_REGISTER 0x3 /* memory-mapped registers, etc. */ | |
231 | ||
232 | ||
233 | /* Instruction mode attributes (ima field) */ | |
234 | #define IMA_MAPPED 0x0 /* Virtual mode */ | |
235 | #define IMA_CB_DELAY 0x1 /* hold read responses until status changes */ | |
236 | #define IMA_UNMAPPED 0x2 /* bypass the TLBs (OS only) */ | |
237 | #define IMA_INTERRUPT 0x4 /* Interrupt when instruction completes */ | |
238 | ||
239 | /* CBE ecause bits */ | |
240 | #define CBE_CAUSE_RI (1 << 0) | |
241 | #define CBE_CAUSE_INVALID_INSTRUCTION (1 << 1) | |
242 | #define CBE_CAUSE_UNMAPPED_MODE_FORBIDDEN (1 << 2) | |
243 | #define CBE_CAUSE_PE_CHECK_DATA_ERROR (1 << 3) | |
244 | #define CBE_CAUSE_IAA_GAA_MISMATCH (1 << 4) | |
245 | #define CBE_CAUSE_DATA_SEGMENT_LIMIT_EXCEPTION (1 << 5) | |
246 | #define CBE_CAUSE_OS_FATAL_TLB_FAULT (1 << 6) | |
247 | #define CBE_CAUSE_EXECUTION_HW_ERROR (1 << 7) | |
248 | #define CBE_CAUSE_TLBHW_ERROR (1 << 8) | |
249 | #define CBE_CAUSE_RA_REQUEST_TIMEOUT (1 << 9) | |
250 | #define CBE_CAUSE_HA_REQUEST_TIMEOUT (1 << 10) | |
251 | #define CBE_CAUSE_RA_RESPONSE_FATAL (1 << 11) | |
252 | #define CBE_CAUSE_RA_RESPONSE_NON_FATAL (1 << 12) | |
253 | #define CBE_CAUSE_HA_RESPONSE_FATAL (1 << 13) | |
254 | #define CBE_CAUSE_HA_RESPONSE_NON_FATAL (1 << 14) | |
255 | #define CBE_CAUSE_ADDRESS_SPACE_DECODE_ERROR (1 << 15) | |
270952a9 JS |
256 | #define CBE_CAUSE_PROTOCOL_STATE_DATA_ERROR (1 << 16) |
257 | #define CBE_CAUSE_RA_RESPONSE_DATA_ERROR (1 << 17) | |
258 | #define CBE_CAUSE_HA_RESPONSE_DATA_ERROR (1 << 18) | |
4c921d4d | 259 | |
cd1334f0 JS |
260 | /* CBE cbrexecstatus bits */ |
261 | #define CBR_EXS_ABORT_OCC_BIT 0 | |
262 | #define CBR_EXS_INT_OCC_BIT 1 | |
263 | #define CBR_EXS_PENDING_BIT 2 | |
264 | #define CBR_EXS_QUEUED_BIT 3 | |
270952a9 | 265 | #define CBR_EXS_TLB_INVAL_BIT 4 |
cd1334f0 JS |
266 | #define CBR_EXS_EXCEPTION_BIT 5 |
267 | ||
268 | #define CBR_EXS_ABORT_OCC (1 << CBR_EXS_ABORT_OCC_BIT) | |
269 | #define CBR_EXS_INT_OCC (1 << CBR_EXS_INT_OCC_BIT) | |
270 | #define CBR_EXS_PENDING (1 << CBR_EXS_PENDING_BIT) | |
271 | #define CBR_EXS_QUEUED (1 << CBR_EXS_QUEUED_BIT) | |
270952a9 | 272 | #define CBR_TLB_INVAL (1 << CBR_EXS_TLB_INVAL_BIT) |
cd1334f0 JS |
273 | #define CBR_EXS_EXCEPTION (1 << CBR_EXS_EXCEPTION_BIT) |
274 | ||
4c921d4d JS |
275 | /* |
276 | * Exceptions are retried for the following cases. If any OTHER bits are set | |
277 | * in ecause, the exception is not retryable. | |
278 | */ | |
270952a9 | 279 | #define EXCEPTION_RETRY_BITS (CBE_CAUSE_EXECUTION_HW_ERROR | \ |
4c921d4d | 280 | CBE_CAUSE_TLBHW_ERROR | \ |
270952a9 JS |
281 | CBE_CAUSE_RA_REQUEST_TIMEOUT | \ |
282 | CBE_CAUSE_RA_RESPONSE_NON_FATAL | \ | |
283 | CBE_CAUSE_HA_RESPONSE_NON_FATAL | \ | |
284 | CBE_CAUSE_RA_RESPONSE_DATA_ERROR | \ | |
285 | CBE_CAUSE_HA_RESPONSE_DATA_ERROR \ | |
286 | ) | |
4c921d4d JS |
287 | |
288 | /* Message queue head structure */ | |
289 | union gru_mesqhead { | |
290 | unsigned long val; | |
291 | struct { | |
292 | unsigned int head; | |
293 | unsigned int limit; | |
294 | }; | |
295 | }; | |
296 | ||
297 | ||
298 | /* Generate the low word of a GRU instruction */ | |
299 | static inline unsigned int | |
300 | __opword(unsigned char opcode, unsigned char exopc, unsigned char xtype, | |
301 | unsigned char iaa0, unsigned char iaa1, | |
302 | unsigned char ima) | |
303 | { | |
304 | return (1 << GRU_CB_ICMD_SHFT) | | |
305 | (iaa0 << GRU_CB_IAA0_SHFT) | | |
306 | (iaa1 << GRU_CB_IAA1_SHFT) | | |
307 | (ima << GRU_CB_IMA_SHFT) | | |
308 | (xtype << GRU_CB_XTYPE_SHFT) | | |
309 | (opcode << GRU_CB_OPC_SHFT) | | |
310 | (exopc << GRU_CB_EXOPC_SHFT); | |
311 | } | |
312 | ||
4c921d4d JS |
313 | /* |
314 | * Architecture specific intrinsics | |
315 | */ | |
316 | static inline void gru_flush_cache(void *p) | |
317 | { | |
318 | __flush_cache(p); | |
319 | } | |
320 | ||
321 | /* | |
322 | * Store the lower 32 bits of the command including the "start" bit. Then | |
323 | * start the instruction executing. | |
324 | */ | |
325 | static inline void gru_start_instruction(struct gru_instruction *ins, int op32) | |
326 | { | |
327 | gru_ordered_store_int(ins, op32); | |
923f7f69 | 328 | gru_flush_cache(ins); |
4c921d4d JS |
329 | } |
330 | ||
331 | ||
332 | /* Convert "hints" to IMA */ | |
333 | #define CB_IMA(h) ((h) | IMA_UNMAPPED) | |
334 | ||
335 | /* Convert data segment cache line index into TRI0 / TRI1 value */ | |
336 | #define GRU_DINDEX(i) ((i) * GRU_CACHE_LINE_BYTES) | |
337 | ||
338 | /* Inline functions for GRU instructions. | |
339 | * Note: | |
340 | * - nelem and stride are in elements | |
341 | * - tri0/tri1 is in bytes for the beginning of the data segment. | |
342 | */ | |
289750d1 RH |
343 | static inline void gru_vload_phys(void *cb, unsigned long gpa, |
344 | unsigned int tri0, int iaa, unsigned long hints) | |
345 | { | |
346 | struct gru_instruction *ins = (struct gru_instruction *)cb; | |
347 | ||
348 | ins->baddr0 = (long)gpa | ((unsigned long)iaa << 62); | |
349 | ins->nelem = 1; | |
350 | ins->tri0 = tri0; | |
351 | ins->op1_stride = 1; | |
352 | gru_start_instruction(ins, __opword(OP_VLOAD, 0, XTYPE_DW, iaa, 0, | |
353 | CB_IMA(hints))); | |
354 | } | |
355 | ||
4c921d4d JS |
356 | static inline void gru_vload(void *cb, unsigned long mem_addr, |
357 | unsigned int tri0, unsigned char xtype, unsigned long nelem, | |
358 | unsigned long stride, unsigned long hints) | |
359 | { | |
360 | struct gru_instruction *ins = (struct gru_instruction *)cb; | |
361 | ||
362 | ins->baddr0 = (long)mem_addr; | |
363 | ins->nelem = nelem; | |
364 | ins->tri0 = tri0; | |
365 | ins->op1_stride = stride; | |
366 | gru_start_instruction(ins, __opword(OP_VLOAD, 0, xtype, IAA_RAM, 0, | |
367 | CB_IMA(hints))); | |
368 | } | |
369 | ||
370 | static inline void gru_vstore(void *cb, unsigned long mem_addr, | |
371 | unsigned int tri0, unsigned char xtype, unsigned long nelem, | |
372 | unsigned long stride, unsigned long hints) | |
373 | { | |
374 | struct gru_instruction *ins = (void *)cb; | |
375 | ||
376 | ins->baddr0 = (long)mem_addr; | |
377 | ins->nelem = nelem; | |
378 | ins->tri0 = tri0; | |
379 | ins->op1_stride = stride; | |
380 | gru_start_instruction(ins, __opword(OP_VSTORE, 0, xtype, IAA_RAM, 0, | |
381 | CB_IMA(hints))); | |
382 | } | |
383 | ||
384 | static inline void gru_ivload(void *cb, unsigned long mem_addr, | |
385 | unsigned int tri0, unsigned int tri1, unsigned char xtype, | |
386 | unsigned long nelem, unsigned long hints) | |
387 | { | |
388 | struct gru_instruction *ins = (void *)cb; | |
389 | ||
390 | ins->baddr0 = (long)mem_addr; | |
391 | ins->nelem = nelem; | |
392 | ins->tri0 = tri0; | |
393 | ins->tri1_bufsize = tri1; | |
394 | gru_start_instruction(ins, __opword(OP_IVLOAD, 0, xtype, IAA_RAM, 0, | |
395 | CB_IMA(hints))); | |
396 | } | |
397 | ||
398 | static inline void gru_ivstore(void *cb, unsigned long mem_addr, | |
399 | unsigned int tri0, unsigned int tri1, | |
400 | unsigned char xtype, unsigned long nelem, unsigned long hints) | |
401 | { | |
402 | struct gru_instruction *ins = (void *)cb; | |
403 | ||
404 | ins->baddr0 = (long)mem_addr; | |
405 | ins->nelem = nelem; | |
406 | ins->tri0 = tri0; | |
407 | ins->tri1_bufsize = tri1; | |
408 | gru_start_instruction(ins, __opword(OP_IVSTORE, 0, xtype, IAA_RAM, 0, | |
409 | CB_IMA(hints))); | |
410 | } | |
411 | ||
412 | static inline void gru_vset(void *cb, unsigned long mem_addr, | |
413 | unsigned long value, unsigned char xtype, unsigned long nelem, | |
414 | unsigned long stride, unsigned long hints) | |
415 | { | |
416 | struct gru_instruction *ins = (void *)cb; | |
417 | ||
418 | ins->baddr0 = (long)mem_addr; | |
419 | ins->op2_value_baddr1 = value; | |
420 | ins->nelem = nelem; | |
421 | ins->op1_stride = stride; | |
422 | gru_start_instruction(ins, __opword(OP_VSET, 0, xtype, IAA_RAM, 0, | |
423 | CB_IMA(hints))); | |
424 | } | |
425 | ||
426 | static inline void gru_ivset(void *cb, unsigned long mem_addr, | |
427 | unsigned int tri1, unsigned long value, unsigned char xtype, | |
428 | unsigned long nelem, unsigned long hints) | |
429 | { | |
430 | struct gru_instruction *ins = (void *)cb; | |
431 | ||
432 | ins->baddr0 = (long)mem_addr; | |
433 | ins->op2_value_baddr1 = value; | |
434 | ins->nelem = nelem; | |
435 | ins->tri1_bufsize = tri1; | |
436 | gru_start_instruction(ins, __opword(OP_IVSET, 0, xtype, IAA_RAM, 0, | |
437 | CB_IMA(hints))); | |
438 | } | |
439 | ||
440 | static inline void gru_vflush(void *cb, unsigned long mem_addr, | |
441 | unsigned long nelem, unsigned char xtype, unsigned long stride, | |
442 | unsigned long hints) | |
443 | { | |
444 | struct gru_instruction *ins = (void *)cb; | |
445 | ||
446 | ins->baddr0 = (long)mem_addr; | |
447 | ins->op1_stride = stride; | |
448 | ins->nelem = nelem; | |
449 | gru_start_instruction(ins, __opword(OP_VFLUSH, 0, xtype, IAA_RAM, 0, | |
450 | CB_IMA(hints))); | |
451 | } | |
452 | ||
453 | static inline void gru_nop(void *cb, int hints) | |
454 | { | |
455 | struct gru_instruction *ins = (void *)cb; | |
456 | ||
457 | gru_start_instruction(ins, __opword(OP_NOP, 0, 0, 0, 0, CB_IMA(hints))); | |
458 | } | |
459 | ||
460 | ||
461 | static inline void gru_bcopy(void *cb, const unsigned long src, | |
462 | unsigned long dest, | |
463 | unsigned int tri0, unsigned int xtype, unsigned long nelem, | |
464 | unsigned int bufsize, unsigned long hints) | |
465 | { | |
466 | struct gru_instruction *ins = (void *)cb; | |
467 | ||
468 | ins->baddr0 = (long)src; | |
469 | ins->op2_value_baddr1 = (long)dest; | |
470 | ins->nelem = nelem; | |
471 | ins->tri0 = tri0; | |
472 | ins->tri1_bufsize = bufsize; | |
473 | gru_start_instruction(ins, __opword(OP_BCOPY, 0, xtype, IAA_RAM, | |
474 | IAA_RAM, CB_IMA(hints))); | |
475 | } | |
476 | ||
477 | static inline void gru_bstore(void *cb, const unsigned long src, | |
478 | unsigned long dest, unsigned int tri0, unsigned int xtype, | |
479 | unsigned long nelem, unsigned long hints) | |
480 | { | |
481 | struct gru_instruction *ins = (void *)cb; | |
482 | ||
483 | ins->baddr0 = (long)src; | |
484 | ins->op2_value_baddr1 = (long)dest; | |
485 | ins->nelem = nelem; | |
486 | ins->tri0 = tri0; | |
487 | gru_start_instruction(ins, __opword(OP_BSTORE, 0, xtype, 0, IAA_RAM, | |
488 | CB_IMA(hints))); | |
489 | } | |
490 | ||
491 | static inline void gru_gamir(void *cb, int exopc, unsigned long src, | |
492 | unsigned int xtype, unsigned long hints) | |
493 | { | |
494 | struct gru_instruction *ins = (void *)cb; | |
495 | ||
496 | ins->baddr0 = (long)src; | |
497 | gru_start_instruction(ins, __opword(OP_GAMIR, exopc, xtype, IAA_RAM, 0, | |
498 | CB_IMA(hints))); | |
499 | } | |
500 | ||
501 | static inline void gru_gamirr(void *cb, int exopc, unsigned long src, | |
502 | unsigned int xtype, unsigned long hints) | |
503 | { | |
504 | struct gru_instruction *ins = (void *)cb; | |
505 | ||
506 | ins->baddr0 = (long)src; | |
507 | gru_start_instruction(ins, __opword(OP_GAMIRR, exopc, xtype, IAA_RAM, 0, | |
508 | CB_IMA(hints))); | |
509 | } | |
510 | ||
511 | static inline void gru_gamer(void *cb, int exopc, unsigned long src, | |
512 | unsigned int xtype, | |
513 | unsigned long operand1, unsigned long operand2, | |
514 | unsigned long hints) | |
515 | { | |
516 | struct gru_instruction *ins = (void *)cb; | |
517 | ||
518 | ins->baddr0 = (long)src; | |
519 | ins->op1_stride = operand1; | |
520 | ins->op2_value_baddr1 = operand2; | |
521 | gru_start_instruction(ins, __opword(OP_GAMER, exopc, xtype, IAA_RAM, 0, | |
522 | CB_IMA(hints))); | |
523 | } | |
524 | ||
525 | static inline void gru_gamerr(void *cb, int exopc, unsigned long src, | |
526 | unsigned int xtype, unsigned long operand1, | |
527 | unsigned long operand2, unsigned long hints) | |
528 | { | |
529 | struct gru_instruction *ins = (void *)cb; | |
530 | ||
531 | ins->baddr0 = (long)src; | |
532 | ins->op1_stride = operand1; | |
533 | ins->op2_value_baddr1 = operand2; | |
534 | gru_start_instruction(ins, __opword(OP_GAMERR, exopc, xtype, IAA_RAM, 0, | |
535 | CB_IMA(hints))); | |
536 | } | |
537 | ||
538 | static inline void gru_gamxr(void *cb, unsigned long src, | |
539 | unsigned int tri0, unsigned long hints) | |
540 | { | |
541 | struct gru_instruction *ins = (void *)cb; | |
542 | ||
543 | ins->baddr0 = (long)src; | |
544 | ins->nelem = 4; | |
545 | gru_start_instruction(ins, __opword(OP_GAMXR, EOP_XR_CSWAP, XTYPE_DW, | |
546 | IAA_RAM, 0, CB_IMA(hints))); | |
547 | } | |
548 | ||
549 | static inline void gru_mesq(void *cb, unsigned long queue, | |
550 | unsigned long tri0, unsigned long nelem, | |
551 | unsigned long hints) | |
552 | { | |
553 | struct gru_instruction *ins = (void *)cb; | |
554 | ||
555 | ins->baddr0 = (long)queue; | |
556 | ins->nelem = nelem; | |
557 | ins->tri0 = tri0; | |
558 | gru_start_instruction(ins, __opword(OP_MESQ, 0, XTYPE_CL, IAA_RAM, 0, | |
559 | CB_IMA(hints))); | |
560 | } | |
561 | ||
562 | static inline unsigned long gru_get_amo_value(void *cb) | |
563 | { | |
564 | struct gru_instruction *ins = (void *)cb; | |
565 | ||
566 | return ins->avalue; | |
567 | } | |
568 | ||
569 | static inline int gru_get_amo_value_head(void *cb) | |
570 | { | |
571 | struct gru_instruction *ins = (void *)cb; | |
572 | ||
573 | return ins->avalue & 0xffffffff; | |
574 | } | |
575 | ||
576 | static inline int gru_get_amo_value_limit(void *cb) | |
577 | { | |
578 | struct gru_instruction *ins = (void *)cb; | |
579 | ||
580 | return ins->avalue >> 32; | |
581 | } | |
582 | ||
583 | static inline union gru_mesqhead gru_mesq_head(int head, int limit) | |
584 | { | |
585 | union gru_mesqhead mqh; | |
586 | ||
587 | mqh.head = head; | |
588 | mqh.limit = limit; | |
589 | return mqh; | |
590 | } | |
591 | ||
592 | /* | |
593 | * Get struct control_block_extended_exc_detail for CB. | |
594 | */ | |
595 | extern int gru_get_cb_exception_detail(void *cb, | |
596 | struct control_block_extended_exc_detail *excdet); | |
597 | ||
598 | #define GRU_EXC_STR_SIZE 256 | |
599 | ||
4c921d4d JS |
600 | |
601 | /* | |
602 | * Control block definition for checking status | |
603 | */ | |
604 | struct gru_control_block_status { | |
605 | unsigned int icmd :1; | |
fe5bb6b0 JS |
606 | unsigned int ima :3; |
607 | unsigned int reserved0 :4; | |
608 | unsigned int unused1 :24; | |
4c921d4d JS |
609 | unsigned int unused2 :24; |
610 | unsigned int istatus :2; | |
611 | unsigned int isubstatus :4; | |
fe5bb6b0 | 612 | unsigned int unused3 :2; |
4c921d4d JS |
613 | }; |
614 | ||
615 | /* Get CB status */ | |
616 | static inline int gru_get_cb_status(void *cb) | |
617 | { | |
618 | struct gru_control_block_status *cbs = (void *)cb; | |
619 | ||
620 | return cbs->istatus; | |
621 | } | |
622 | ||
623 | /* Get CB message queue substatus */ | |
624 | static inline int gru_get_cb_message_queue_substatus(void *cb) | |
625 | { | |
626 | struct gru_control_block_status *cbs = (void *)cb; | |
627 | ||
628 | return cbs->isubstatus & CBSS_MSG_QUEUE_MASK; | |
629 | } | |
630 | ||
631 | /* Get CB substatus */ | |
632 | static inline int gru_get_cb_substatus(void *cb) | |
633 | { | |
634 | struct gru_control_block_status *cbs = (void *)cb; | |
635 | ||
636 | return cbs->isubstatus; | |
637 | } | |
638 | ||
9f250114 JS |
639 | /* |
640 | * User interface to check an instruction status. UPM and exceptions | |
641 | * are handled automatically. However, this function does NOT wait | |
642 | * for an active instruction to complete. | |
643 | * | |
4c921d4d JS |
644 | */ |
645 | static inline int gru_check_status(void *cb) | |
646 | { | |
647 | struct gru_control_block_status *cbs = (void *)cb; | |
923f7f69 | 648 | int ret; |
4c921d4d | 649 | |
923f7f69 | 650 | ret = cbs->istatus; |
9f250114 | 651 | if (ret != CBS_ACTIVE) |
4c921d4d JS |
652 | ret = gru_check_status_proc(cb); |
653 | return ret; | |
654 | } | |
655 | ||
9f250114 JS |
656 | /* |
657 | * User interface (via inline function) to wait for an instruction | |
658 | * to complete. Completion status (IDLE or EXCEPTION is returned | |
659 | * to the user. Exception due to hardware errors are automatically | |
660 | * retried before returning an exception. | |
661 | * | |
4c921d4d JS |
662 | */ |
663 | static inline int gru_wait(void *cb) | |
664 | { | |
9f250114 | 665 | return gru_wait_proc(cb); |
4c921d4d JS |
666 | } |
667 | ||
9f250114 JS |
668 | /* |
669 | * Wait for CB to complete. Aborts program if error. (Note: error does NOT | |
4c921d4d JS |
670 | * mean TLB mis - only fatal errors such as memory parity error or user |
671 | * bugs will cause termination. | |
672 | */ | |
673 | static inline void gru_wait_abort(void *cb) | |
674 | { | |
9f250114 | 675 | gru_wait_abort_proc(cb); |
4c921d4d JS |
676 | } |
677 | ||
678 | ||
679 | /* | |
680 | * Get a pointer to a control block | |
681 | * gseg - GSeg address returned from gru_get_thread_gru_segment() | |
682 | * index - index of desired CB | |
683 | */ | |
684 | static inline void *gru_get_cb_pointer(void *gseg, | |
685 | int index) | |
686 | { | |
687 | return gseg + GRU_CB_BASE + index * GRU_HANDLE_STRIDE; | |
688 | } | |
689 | ||
690 | /* | |
691 | * Get a pointer to a cacheline in the data segment portion of a GSeg | |
692 | * gseg - GSeg address returned from gru_get_thread_gru_segment() | |
693 | * index - index of desired cache line | |
694 | */ | |
695 | static inline void *gru_get_data_pointer(void *gseg, int index) | |
696 | { | |
697 | return gseg + GRU_DS_BASE + index * GRU_CACHE_LINE_BYTES; | |
698 | } | |
699 | ||
700 | /* | |
701 | * Convert a vaddr into the tri index within the GSEG | |
702 | * vaddr - virtual address of within gseg | |
703 | */ | |
704 | static inline int gru_get_tri(void *vaddr) | |
705 | { | |
706 | return ((unsigned long)vaddr & (GRU_GSEG_PAGESIZE - 1)) - GRU_DS_BASE; | |
707 | } | |
708 | #endif /* __GRU_INSTRUCTIONS_H__ */ |