aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/kvm/booke_interrupts.S
blob: 8d6929b7fdb68907f9f72a1e33a62a80fd89ff0e (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 * Copyright IBM Corp. 2007
 *
 * Authors: Hollis Blanchard <hollisb@us.ibm.com>
 */

#include <asm/ppc_asm.h>
#include <asm/kvm_asm.h>
#include <asm/reg.h>
#include <asm/mmu-44x.h>
#include <asm/page.h>
#include <asm/asm-offsets.h>

#define KVMPPC_MSR_MASK (MSR_CE|MSR_EE|MSR_PR|MSR_DE|MSR_ME|MSR_IS|MSR_DS)

#define VCPU_GPR(n)     (VCPU_GPRS + (n * 4))

/* The host stack layout: */
#define HOST_R1         0 /* Implied by stwu. */
#define HOST_CALLEE_LR  4
#define HOST_RUN        8
/* r2 is special: it holds 'current', and it made nonvolatile in the
 * kernel with the -ffixed-r2 gcc option. */
#define HOST_R2         12
#define HOST_NV_GPRS    16
#define HOST_NV_GPR(n)  (HOST_NV_GPRS + ((n - 14) * 4))
#define HOST_MIN_STACK_SIZE (HOST_NV_GPR(31) + 4)
#define HOST_STACK_SIZE (((HOST_MIN_STACK_SIZE + 15) / 16) * 16) /* Align. */
#define HOST_STACK_LR   (HOST_STACK_SIZE + 4) /* In caller stack frame. */

#define NEED_INST_MASK ((1<<BOOKE_INTERRUPT_PROGRAM) | \
                        (1<<BOOKE_INTERRUPT_DTLB_MISS) | \
                        (1<<BOOKE_INTERRUPT_DEBUG))

#define NEED_DEAR_MASK ((1<<BOOKE_INTERRUPT_DATA_STORAGE) | \
                        (1<<BOOKE_INTERRUPT_DTLB_MISS))

#define NEED_ESR_MASK ((1<<BOOKE_INTERRUPT_DATA_STORAGE) | \
                       (1<<BOOKE_INTERRUPT_INST_STORAGE) | \
                       (1<<BOOKE_INTERRUPT_PROGRAM) | \
                       (1<<BOOKE_INTERRUPT_DTLB_MISS))

.macro KVM_HANDLER ivor_nr
_GLOBAL(kvmppc_handler_\ivor_nr)
	/* Get pointer to vcpu and record exit number. */
	mtspr	SPRN_SPRG0, r4
	mfspr	r4, SPRN_SPRG1
	stw	r5, VCPU_GPR(r5)(r4)
	stw	r6, VCPU_GPR(r6)(r4)
	mfctr	r5
	lis	r6, kvmppc_resume_host@h
	stw	r5, VCPU_CTR(r4)
	li	r5, \ivor_nr
	ori	r6, r6, kvmppc_resume_host@l
	mtctr	r6
	bctr
.endm

_GLOBAL(kvmppc_handlers_start)
KVM_HANDLER BOOKE_INTERRUPT_CRITICAL
KVM_HANDLER BOOKE_INTERRUPT_MACHINE_CHECK
KVM_HANDLER BOOKE_INTERRUPT_DATA_STORAGE
KVM_HANDLER BOOKE_INTERRUPT_INST_STORAGE
KVM_HANDLER BOOKE_INTERRUPT_EXTERNAL
KVM_HANDLER BOOKE_INTERRUPT_ALIGNMENT
KVM_HANDLER BOOKE_INTERRUPT_PROGRAM
KVM_HANDLER BOOKE_INTERRUPT_FP_UNAVAIL
KVM_HANDLER BOOKE_INTERRUPT_SYSCALL
KVM_HANDLER BOOKE_INTERRUPT_AP_UNAVAIL
KVM_HANDLER BOOKE_INTERRUPT_DECREMENTER
KVM_HANDLER BOOKE_INTERRUPT_FIT
KVM_HANDLER BOOKE_INTERRUPT_WATCHDOG
KVM_HANDLER BOOKE_INTERRUPT_DTLB_MISS
KVM_HANDLER BOOKE_INTERRUPT_ITLB_MISS
KVM_HANDLER BOOKE_INTERRUPT_DEBUG

_GLOBAL(kvmppc_handler_len)
	.long kvmppc_handler_1 - kvmppc_handler_0


/* Registers:
 *  SPRG0: guest r4
 *  r4: vcpu pointer
 *  r5: KVM exit number
 */
_GLOBAL(kvmppc_resume_host)
	stw	r3, VCPU_GPR(r3)(r4)
	mfcr	r3
	stw	r3, VCPU_CR(r4)
	stw	r7, VCPU_GPR(r7)(r4)
	stw	r8, VCPU_GPR(r8)(r4)
	stw	r9, VCPU_GPR(r9)(r4)

	li	r6, 1
	slw	r6, r6, r5

	/* Save the faulting instruction and all GPRs for emulation. */
	andi.	r7, r6, NEED_INST_MASK
	beq	..skip_inst_copy
	mfspr	r9, SPRN_SRR0
	mfmsr	r8
	ori	r7, r8, MSR_DS
	mtmsr	r7
	isync
	lwz	r9, 0(r9)
	mtmsr	r8
	isync
	stw	r9, VCPU_LAST_INST(r4)

	stw	r15, VCPU_GPR(r15)(r4)
	stw	r16, VCPU_GPR(r16)(r4)
	stw	r17, VCPU_GPR(r17)(r4)
	stw	r18, VCPU_GPR(r18)(r4)
	stw	r19, VCPU_GPR(r19)(r4)
	stw	r20, VCPU_GPR(r20)(r4)
	stw	r21, VCPU_GPR(r21)(r4)
	stw	r22, VCPU_GPR(r22)(r4)
	stw	r23, VCPU_GPR(r23)(r4)
	stw	r24, VCPU_GPR(r24)(r4)
	stw	r25, VCPU_GPR(r25)(r4)
	stw	r26, VCPU_GPR(r26)(r4)
	stw	r27, VCPU_GPR(r27)(r4)
	stw	r28, VCPU_GPR(r28)(r4)
	stw	r29, VCPU_GPR(r29)(r4)
	stw	r30, VCPU_GPR(r30)(r4)
	stw	r31, VCPU_GPR(r31)(r4)
..skip_inst_copy:

	/* Also grab DEAR and ESR before the host can clobber them. */

	andi.	r7, r6, NEED_DEAR_MASK
	beq	..skip_dear
	mfspr	r9, SPRN_DEAR
	stw	r9, VCPU_FAULT_DEAR(r4)
..skip_dear:

	andi.	r7, r6, NEED_ESR_MASK
	beq	..skip_esr
	mfspr	r9, SPRN_ESR
	stw	r9, VCPU_FAULT_ESR(r4)
..skip_esr:

	/* Save remaining volatile guest register state to vcpu. */
	stw	r0, VCPU_GPR(r0)(r4)
	stw	r1, VCPU_GPR(r1)(r4)
	stw	r2, VCPU_GPR(r2)(r4)
	stw	r10, VCPU_GPR(r10)(r4)
	stw	r11, VCPU_GPR(r11)(r4)
	stw	r12, VCPU_GPR(r12)(r4)
	stw	r13, VCPU_GPR(r13)(r4)
	stw	r14, VCPU_GPR(r14)(r4) /* We need a NV GPR below. */
	mflr	r3
	stw	r3, VCPU_LR(r4)
	mfxer	r3
	stw	r3, VCPU_XER(r4)
	mfspr	r3, SPRN_SPRG0
	stw	r3, VCPU_GPR(r4)(r4)
	mfspr	r3, SPRN_SRR0
	stw	r3, VCPU_PC(r4)

	/* Restore host stack pointer and PID before IVPR, since the host
	 * exception handlers use them. */
	lwz	r1, VCPU_HOST_STACK(r4)
	lwz	r3, VCPU_HOST_PID(r4)
	mtspr	SPRN_PID, r3

	/* Restore host IVPR before re-enabling interrupts. We cheat and know
	 * that Linux IVPR is always 0xc0000000. */
	lis	r3, 0xc000
	mtspr	SPRN_IVPR, r3

	/* Switch to kernel stack and jump to handler. */
	LOAD_REG_ADDR(r3, kvmppc_handle_exit)
	mtctr	r3
	lwz	r3, HOST_RUN(r1)
	lwz	r2, HOST_R2(r1)
	mr	r14, r4 /* Save vcpu pointer. */

	bctrl	/* kvmppc_handle_exit() */

	/* Restore vcpu pointer and the nonvolatiles we used. */
	mr	r4, r14
	lwz	r14, VCPU_GPR(r14)(r4)

	/* Sometimes instruction emulation must restore complete GPR state. */
	andi.	r5, r3, RESUME_FLAG_NV
	beq	..skip_nv_load
	lwz	r15, VCPU_GPR(r15)(r4)
	lwz	r16, VCPU_GPR(r16)(r4)
	lwz	r17, VCPU_GPR(r17)(r4)
	lwz	r18, VCPU_GPR(r18)(r4)
	lwz	r19, VCPU_GPR(r19)(r4)
	lwz	r20, VCPU_GPR(r20)(r4)
	lwz	r21, VCPU_GPR(r21)(r4)
	lwz	r22, VCPU_GPR(r22)(r4)
	lwz	r23, VCPU_GPR(r23)(r4)
	lwz	r24, VCPU_GPR(r24)(r4)
	lwz	r25, VCPU_GPR(r25)(r4)
	lwz	r26, VCPU_GPR(r26)(r4)
	lwz	r27, VCPU_GPR(r27)(r4)
	lwz	r28, VCPU_GPR(r28)(r4)
	lwz	r29, VCPU_GPR(r29)(r4)
	lwz	r30, VCPU_GPR(r30)(r4)
	lwz	r31, VCPU_GPR(r31)(r4)
..skip_nv_load:

	/* Should we return to the guest? */
	andi.	r5, r3, RESUME_FLAG_HOST
	beq	lightweight_exit

	srawi	r3, r3, 2 /* Shift -ERR back down. */

heavyweight_exit:
	/* Not returning to guest. */

	/* We already saved guest volatile register state; now save the
	 * non-volatiles. */
	stw	r15, VCPU_GPR(r15)(r4)
	stw	r16, VCPU_GPR(r16)(r4)
	stw	r17, VCPU_GPR(r17)(r4)
	stw	r18, VCPU_GPR(r18)(r4)
	stw	r19, VCPU_GPR(r19)(r4)
	stw	r20, VCPU_GPR(r20)(r4)
	stw	r21, VCPU_GPR(r21)(r4)
	stw	r22, VCPU_GPR(r22)(r4)
	stw	r23, VCPU_GPR(r23)(r4)
	stw	r24, VCPU_GPR(r24)(r4)
	stw	r25, VCPU_GPR(r25)(r4)
	stw	r26, VCPU_GPR(r26)(r4)
	stw	r27, VCPU_GPR(r27)(r4)
	stw	r28, VCPU_GPR(r28)(r4)
	stw	r29, VCPU_GPR(r29)(r4)
	stw	r30, VCPU_GPR(r30)(r4)
	stw	r31, VCPU_GPR(r31)(r4)

	/* Load host non-volatile register state from host stack. */
	lwz	r14, HOST_NV_GPR(r14)(r1)
	lwz	r15, HOST_NV_GPR(r15)(r1)
	lwz	r16, HOST_NV_GPR(r16)(r1)
	lwz	r17, HOST_NV_GPR(r17)(r1)
	lwz	r18, HOST_NV_GPR(r18)(r1)
	lwz	r19, HOST_NV_GPR(r19)(r1)
	lwz	r20, HOST_NV_GPR(r20)(r1)
	lwz	r21, HOST_NV_GPR(r21)(r1)
	lwz	r22, HOST_NV_GPR(r22)(r1)
	lwz	r23, HOST_NV_GPR(r23)(r1)
	lwz	r24, HOST_NV_GPR(r24)(r1)
	lwz	r25, HOST_NV_GPR(r25)(r1)
	lwz	r26, HOST_NV_GPR(r26)(r1)
	lwz	r27, HOST_NV_GPR(r27)(r1)
	lwz	r28, HOST_NV_GPR(r28)(r1)
	lwz	r29, HOST_NV_GPR(r29)(r1)
	lwz	r30, HOST_NV_GPR(r30)(r1)
	lwz	r31, HOST_NV_GPR(r31)(r1)

	/* Return to kvm_vcpu_run(). */
	lwz	r4, HOST_STACK_LR(r1)
	addi	r1, r1, HOST_STACK_SIZE
	mtlr	r4
	/* r3 still contains the return code from kvmppc_handle_exit(). */
	blr


/* Registers:
 *  r3: kvm_run pointer
 *  r4: vcpu pointer
 */
_GLOBAL(__kvmppc_vcpu_run)
	stwu	r1, -HOST_STACK_SIZE(r1)
	stw	r1, VCPU_HOST_STACK(r4)	/* Save stack pointer to vcpu. */

	/* Save host state to stack. */
	stw	r3, HOST_RUN(r1)
	mflr	r3
	stw	r3, HOST_STACK_LR(r1)

	/* Save host non-volatile register state to stack. */
	stw	r14, HOST_NV_GPR(r14)(r1)
	stw	r15, HOST_NV_GPR(r15)(r1)
	stw	r16, HOST_NV_GPR(r16)(r1)
	stw	r17, HOST_NV_GPR(r17)(r1)
	stw	r18, HOST_NV_GPR(r18)(r1)
	stw	r19, HOST_NV_GPR(r19)(r1)
	stw	r20, HOST_NV_GPR(r20)(r1)
	stw	r21, HOST_NV_GPR(r21)(r1)
	stw	r22, HOST_NV_GPR(r22)(r1)
	stw	r23, HOST_NV_GPR(r23)(r1)
	stw	r24, HOST_NV_GPR(r24)(r1)
	stw	r25, HOST_NV_GPR(r25)(r1)
	stw	r26, HOST_NV_GPR(r26)(r1)
	stw	r27, HOST_NV_GPR(r27)(r1)
	stw	r28, HOST_NV_GPR(r28)(r1)
	stw	r29, HOST_NV_GPR(r29)(r1)
	stw	r30, HOST_NV_GPR(r30)(r1)
	stw	r31, HOST_NV_GPR(r31)(r1)

	/* Load guest non-volatiles. */
	lwz	r14, VCPU_GPR(r14)(r4)
	lwz	r15, VCPU_GPR(r15)(r4)
	lwz	r16, VCPU_GPR(r16)(r4)
	lwz	r17, VCPU_GPR(r17)(r4)
	lwz	r18, VCPU_GPR(r18)(r4)
	lwz	r19, VCPU_GPR(r19)(r4)
	lwz	r20, VCPU_GPR(r20)(r4)
	lwz	r21, VCPU_GPR(r21)(r4)
	lwz	r22, VCPU_GPR(r22)(r4)
	lwz	r23, VCPU_GPR(r23)(r4)
	lwz	r24, VCPU_GPR(r24)(r4)
	lwz	r25, VCPU_GPR(r25)(r4)
	lwz	r26, VCPU_GPR(r26)(r4)
	lwz	r27, VCPU_GPR(r27)(r4)
	lwz	r28, VCPU_GPR(r28)(r4)
	lwz	r29, VCPU_GPR(r29)(r4)
	lwz	r30, VCPU_GPR(r30)(r4)
	lwz	r31, VCPU_GPR(r31)(r4)

lightweight_exit:
	stw	r2, HOST_R2(r1)

	mfspr	r3, SPRN_PID
	stw	r3, VCPU_HOST_PID(r4)
	lwz	r3, VCPU_SHADOW_PID(r4)
	mtspr	SPRN_PID, r3

	/* Prevent all asynchronous TLB updates. */
	mfmsr	r5
	lis	r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@h
	ori	r6, r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@l
	andc	r6, r5, r6
	mtmsr	r6

	/* Load the guest mappings, leaving the host's "pinned" kernel mappings
	 * in place. */
	mfspr	r10, SPRN_MMUCR			/* Save host MMUCR. */
	li	r5, PPC44x_TLB_SIZE
	lis	r5, tlb_44x_hwater@ha
	lwz	r5, tlb_44x_hwater@l(r5)
	mtctr	r5
	addi	r9, r4, -VCPU_TO_44X + VCPU44x_SHADOW_TLB
	addi	r5, r4, -VCPU_TO_44X + VCPU44x_SHADOW_MOD
	li	r3, 0
1:
	lbzx	r7, r3, r5
	cmpwi	r7, 0
	beq	3f

	/* Load guest entry. */
	mulli	r11, r3, TLBE_BYTES
	add	r11, r11, r9
	lwz	r7, 0(r11)
	mtspr	SPRN_MMUCR, r7
	lwz	r7, 4(r11)
	tlbwe	r7, r3, PPC44x_TLB_PAGEID
	lwz	r7, 8(r11)
	tlbwe	r7, r3, PPC44x_TLB_XLAT
	lwz	r7, 12(r11)
	tlbwe	r7, r3, PPC44x_TLB_ATTRIB
3:
	addi	r3, r3, 1                       /* Increment index. */
	bdnz	1b

	mtspr	SPRN_MMUCR, r10			/* Restore host MMUCR. */

	/* Clear bitmap of modified TLB entries */
	li	r5, PPC44x_TLB_SIZE>>2
	mtctr	r5
	addi	r5, r4, -VCPU_TO_44X + VCPU44x_SHADOW_MOD - 4
	li	r6, 0
1:
	stwu	r6, 4(r5)
	bdnz	1b

	iccci	0, 0 /* XXX hack */

	/* Load some guest volatiles. */
	lwz	r0, VCPU_GPR(r0)(r4)
	lwz	r2, VCPU_GPR(r2)(r4)
	lwz	r9, VCPU_GPR(r9)(r4)
	lwz	r10, VCPU_GPR(r10)(r4)
	lwz	r11, VCPU_GPR(r11)(r4)
	lwz	r12, VCPU_GPR(r12)(r4)
	lwz	r13, VCPU_GPR(r13)(r4)
	lwz	r3, VCPU_LR(r4)
	mtlr	r3
	lwz	r3, VCPU_XER(r4)
	mtxer	r3

	/* Switch the IVPR. XXX If we take a TLB miss after this we're screwed,
	 * so how do we make sure vcpu won't fault? */
	lis	r8, kvmppc_booke_handlers@ha
	lwz	r8, kvmppc_booke_handlers@l(r8)
	mtspr	SPRN_IVPR, r8

	/* Save vcpu pointer for the exception handlers. */
	mtspr	SPRN_SPRG1, r4

	/* Can't switch the stack pointer until after IVPR is switched,
	 * because host interrupt handlers would get confused. */
	lwz	r1, VCPU_GPR(r1)(r4)

	/* XXX handle USPRG0 */
	/* Host interrupt handlers may have clobbered these guest-readable
	 * SPRGs, so we need to reload them here with the guest's values. */
	lwz	r3, VCPU_SPRG4(r4)
	mtspr	SPRN_SPRG4, r3
	lwz	r3, VCPU_SPRG5(r4)
	mtspr	SPRN_SPRG5, r3
	lwz	r3, VCPU_SPRG6(r4)
	mtspr	SPRN_SPRG6, r3
	lwz	r3, VCPU_SPRG7(r4)
	mtspr	SPRN_SPRG7, r3

	/* Finish loading guest volatiles and jump to guest. */
	lwz	r3, VCPU_CTR(r4)
	mtctr	r3
	lwz	r3, VCPU_CR(r4)
	mtcr	r3
	lwz	r5, VCPU_GPR(r5)(r4)
	lwz	r6, VCPU_GPR(r6)(r4)
	lwz	r7, VCPU_GPR(r7)(r4)
	lwz	r8, VCPU_GPR(r8)(r4)
	lwz	r3, VCPU_PC(r4)
	mtsrr0	r3
	lwz	r3, VCPU_MSR(r4)
	oris	r3, r3, KVMPPC_MSR_MASK@h
	ori	r3, r3, KVMPPC_MSR_MASK@l
	mtsrr1	r3

	/* Clear any debug events which occurred since we disabled MSR[DE].
	 * XXX This gives us a 3-instruction window in which a breakpoint
	 * intended for guest context could fire in the host instead. */
	lis	r3, 0xffff
	ori	r3, r3, 0xffff
	mtspr	SPRN_DBSR, r3

	lwz	r3, VCPU_GPR(r3)(r4)
	lwz	r4, VCPU_GPR(r4)(r4)
	rfi