aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/misc/mic/host/mic_intr.c
blob: f9c29bc918bc05c4b5394ddd472fbc3b839aec67 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
/*
 * Intel MIC Platform Software Stack (MPSS)
 *
 * Copyright(c) 2013 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Intel MIC Host driver.
 *
 */
#include <linux/pci.h>
#include <linux/interrupt.h>

#include "../common/mic_dev.h"
#include "mic_device.h"

/*
 * mic_invoke_callback - Invoke callback functions registered for
 * the corresponding source id.
 *
 * @mdev: pointer to the mic_device instance
 * @idx: The interrupt source id.
 *
 * Returns none.
 */
static inline void mic_invoke_callback(struct mic_device *mdev, int idx)
{
	struct mic_intr_cb *intr_cb;
	struct pci_dev *pdev = container_of(mdev->sdev->parent,
		struct pci_dev, dev);

	spin_lock(&mdev->irq_info.mic_intr_lock);
	list_for_each_entry(intr_cb, &mdev->irq_info.cb_list[idx], list)
		if (intr_cb->func)
			intr_cb->func(pdev->irq, intr_cb->data);
	spin_unlock(&mdev->irq_info.mic_intr_lock);
}

/**
 * mic_interrupt - Generic interrupt handler for
 * MSI and INTx based interrupts.
 */
static irqreturn_t mic_interrupt(int irq, void *dev)
{
	struct mic_device *mdev = dev;
	struct mic_intr_info *info = mdev->intr_info;
	u32 mask;
	int i;

	mask = mdev->ops->ack_interrupt(mdev);
	if (!mask)
		return IRQ_NONE;

	for (i = info->intr_start_idx[MIC_INTR_DB];
			i < info->intr_len[MIC_INTR_DB]; i++)
		if (mask & BIT(i))
			mic_invoke_callback(mdev, i);

	return IRQ_HANDLED;
}

/* Return the interrupt offset from the index. Index is 0 based. */
static u16 mic_map_src_to_offset(struct mic_device *mdev,
		int intr_src, enum mic_intr_type type)
{
	if (type >= MIC_NUM_INTR_TYPES)
		return MIC_NUM_OFFSETS;
	if (intr_src >= mdev->intr_info->intr_len[type])
		return MIC_NUM_OFFSETS;

	return mdev->intr_info->intr_start_idx[type] + intr_src;
}

/* Return next available msix_entry. */
static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)
{
	int i;
	struct mic_irq_info *info = &mdev->irq_info;

	for (i = 0; i < info->num_vectors; i++)
		if (!info->mic_msi_map[i])
			return &info->msix_entries[i];
	return NULL;
}

/**
 * mic_register_intr_callback - Register a callback handler for the
 * given source id.
 *
 * @mdev: pointer to the mic_device instance
 * @idx: The source id to be registered.
 * @func: The function to be called when the source id receives
 * the interrupt.
 * @data: Private data of the requester.
 * Return the callback structure that was registered or an
 * appropriate error on failure.
 */
static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,
			u8 idx, irqreturn_t (*func) (int irq, void *dev),
			void *data)
{
	struct mic_intr_cb *intr_cb;
	unsigned long flags;
	int rc;
	intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);

	if (!intr_cb)
		return ERR_PTR(-ENOMEM);

	intr_cb->func = func;
	intr_cb->data = data;
	intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
		0, 0, GFP_KERNEL);
	if (intr_cb->cb_id < 0) {
		rc = intr_cb->cb_id;
		goto ida_fail;
	}

	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
	list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);

	return intr_cb;
ida_fail:
	kfree(intr_cb);
	return ERR_PTR(rc);
}

/**
 * mic_unregister_intr_callback - Unregister the callback handler
 * identified by its callback id.
 *
 * @mdev: pointer to the mic_device instance
 * @idx: The callback structure id to be unregistered.
 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no
 * such callback handler was found.
 */
static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
{
	struct list_head *pos, *tmp;
	struct mic_intr_cb *intr_cb;
	unsigned long flags;
	int i;

	for (i = 0;  i < MIC_NUM_OFFSETS; i++) {
		spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
			intr_cb = list_entry(pos, struct mic_intr_cb, list);
			if (intr_cb->cb_id == idx) {
				list_del(pos);
				ida_simple_remove(&mdev->irq_info.cb_ida,
						  intr_cb->cb_id);
				kfree(intr_cb);
				spin_unlock_irqrestore(
					&mdev->irq_info.mic_intr_lock, flags);
				return i;
			}
		}
		spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
	}
	return MIC_NUM_OFFSETS;
}

/**
 * mic_setup_msix - Initializes MSIx interrupts.
 *
 * @mdev: pointer to mic_device instance
 *
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)
{
	int rc, i;
	int entry_size = sizeof(*mdev->irq_info.msix_entries);

	mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
						    entry_size, GFP_KERNEL);
	if (!mdev->irq_info.msix_entries) {
		rc = -ENOMEM;
		goto err_nomem1;
	}

	for (i = 0; i < MIC_MIN_MSIX; i++)
		mdev->irq_info.msix_entries[i].entry = i;

	rc = pci_enable_msix(pdev, mdev->irq_info.msix_entries,
		MIC_MIN_MSIX);
	if (rc) {
		dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
		goto err_enable_msix;
	}

	mdev->irq_info.num_vectors = MIC_MIN_MSIX;
	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
		mdev->irq_info.num_vectors), GFP_KERNEL);

	if (!mdev->irq_info.mic_msi_map) {
		rc = -ENOMEM;
		goto err_nomem2;
	}

	dev_dbg(mdev->sdev->parent,
		"%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
	return 0;
err_nomem2:
	pci_disable_msix(pdev);
err_enable_msix:
	kfree(mdev->irq_info.msix_entries);
err_nomem1:
	mdev->irq_info.num_vectors = 0;
	return rc;
}

/**
 * mic_setup_callbacks - Initialize data structures needed
 * to handle callbacks.
 *
 * @mdev: pointer to mic_device instance
 */
static int mic_setup_callbacks(struct mic_device *mdev)
{
	int i;

	mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
					       sizeof(*mdev->irq_info.cb_list),
					       GFP_KERNEL);
	if (!mdev->irq_info.cb_list)
		return -ENOMEM;

	for (i = 0; i < MIC_NUM_OFFSETS; i++)
		INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
	ida_init(&mdev->irq_info.cb_ida);
	spin_lock_init(&mdev->irq_info.mic_intr_lock);
	return 0;
}

/**
 * mic_release_callbacks - Uninitialize data structures needed
 * to handle callbacks.
 *
 * @mdev: pointer to mic_device instance
 */
static void mic_release_callbacks(struct mic_device *mdev)
{
	unsigned long flags;
	struct list_head *pos, *tmp;
	struct mic_intr_cb *intr_cb;
	int i;

	for (i = 0; i < MIC_NUM_OFFSETS; i++) {
		spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);

		if (list_empty(&mdev->irq_info.cb_list[i])) {
			spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock,
					       flags);
			break;
		}

		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
			intr_cb = list_entry(pos, struct mic_intr_cb, list);
			list_del(pos);
			ida_simple_remove(&mdev->irq_info.cb_ida,
					  intr_cb->cb_id);
			kfree(intr_cb);
		}
		spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
	}
	ida_destroy(&mdev->irq_info.cb_ida);
	kfree(mdev->irq_info.cb_list);
}

/**
 * mic_setup_msi - Initializes MSI interrupts.
 *
 * @mdev: pointer to mic_device instance
 * @pdev: PCI device structure
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)
{
	int rc;

	rc = pci_enable_msi(pdev);
	if (rc) {
		dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
		return rc;
	}

	mdev->irq_info.num_vectors = 1;
	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
		mdev->irq_info.num_vectors), GFP_KERNEL);

	if (!mdev->irq_info.mic_msi_map) {
		rc = -ENOMEM;
		goto err_nomem1;
	}

	rc = mic_setup_callbacks(mdev);
	if (rc) {
		dev_err(&pdev->dev, "Error setting up callbacks\n");
		goto err_nomem2;
	}

	rc = request_irq(pdev->irq, mic_interrupt, 0 , "mic-msi", mdev);
	if (rc) {
		dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
		goto err_irq_req_fail;
	}

	dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
	return 0;
err_irq_req_fail:
	mic_release_callbacks(mdev);
err_nomem2:
	kfree(mdev->irq_info.mic_msi_map);
err_nomem1:
	pci_disable_msi(pdev);
	mdev->irq_info.num_vectors = 0;
	return rc;
}

/**
 * mic_setup_intx - Initializes legacy interrupts.
 *
 * @mdev: pointer to mic_device instance
 * @pdev: PCI device structure
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)
{
	int rc;

	pci_msi_off(pdev);

	/* Enable intx */
	pci_intx(pdev, 1);
	rc = mic_setup_callbacks(mdev);
	if (rc) {
		dev_err(&pdev->dev, "Error setting up callbacks\n");
		goto err_nomem;
	}

	rc = request_irq(pdev->irq, mic_interrupt,
		IRQF_SHARED, "mic-intx", mdev);
	if (rc)
		goto err;

	dev_dbg(&pdev->dev, "intx irq setup\n");
	return 0;
err:
	mic_release_callbacks(mdev);
err_nomem:
	return rc;
}

/**
 * mic_next_db - Retrieve the next doorbell interrupt source id.
 * The id is picked sequentially from the available pool of
 * doorlbell ids.
 *
 * @mdev: pointer to the mic_device instance.
 *
 * Returns the next doorbell interrupt source.
 */
int mic_next_db(struct mic_device *mdev)
{
	int next_db;

	next_db = mdev->irq_info.next_avail_src %
		mdev->intr_info->intr_len[MIC_INTR_DB];
	mdev->irq_info.next_avail_src++;
	return next_db;
}

#define COOKIE_ID_SHIFT 16
#define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
#define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
#define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)

/**
 * mic_request_irq - request an irq. mic_mutex needs
 * to be held before calling this function.
 *
 * @mdev: pointer to mic_device instance
 * @func: The callback function that handles the interrupt.
 * The function needs to call ack_interrupts
 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
 * @name: The ASCII name of the callee requesting the irq.
 * @data: private data that is returned back when calling the
 * function handler.
 * @intr_src: The source id of the requester. Its the doorbell id
 * for Doorbell interrupts and DMA channel id for DMA interrupts.
 * @type: The type of interrupt. Values defined in mic_intr_type
 *
 * returns: The cookie that is transparent to the caller. Passed
 * back when calling mic_free_irq. An appropriate error code
 * is returned on failure. Caller needs to use IS_ERR(return_val)
 * to check for failure and PTR_ERR(return_val) to obtained the
 * error code.
 *
 */
struct mic_irq *mic_request_irq(struct mic_device *mdev,
	irqreturn_t (*func)(int irq, void *dev),
	const char *name, void *data, int intr_src,
	enum mic_intr_type type)
{
	u16 offset;
	int rc = 0;
	struct msix_entry *msix = NULL;
	unsigned long cookie = 0;
	u16 entry;
	struct mic_intr_cb *intr_cb;
	struct pci_dev *pdev = container_of(mdev->sdev->parent,
		struct pci_dev, dev);

	offset = mic_map_src_to_offset(mdev, intr_src, type);
	if (offset >= MIC_NUM_OFFSETS) {
		dev_err(mdev->sdev->parent,
			"Error mapping index %d to a valid source id.\n",
			intr_src);
		rc = -EINVAL;
		goto err;
	}

	if (mdev->irq_info.num_vectors > 1) {
		msix = mic_get_available_vector(mdev);
		if (!msix) {
			dev_err(mdev->sdev->parent,
				"No MSIx vectors available for use.\n");
			rc = -ENOSPC;
			goto err;
		}

		rc = request_irq(msix->vector, func, 0, name, data);
		if (rc) {
			dev_dbg(mdev->sdev->parent,
				"request irq failed rc = %d\n", rc);
			goto err;
		}
		entry = msix->entry;
		mdev->irq_info.mic_msi_map[entry] |= BIT(offset);
		mdev->intr_ops->program_msi_to_src_map(mdev,
				entry, offset, true);
		cookie = MK_COOKIE(entry, offset);
		dev_dbg(mdev->sdev->parent, "irq: %d assigned for src: %d\n",
			msix->vector, intr_src);
	} else {
		intr_cb = mic_register_intr_callback(mdev,
				offset, func, data);
		if (IS_ERR(intr_cb)) {
			dev_err(mdev->sdev->parent,
				"No available callback entries for use\n");
			rc = PTR_ERR(intr_cb);
			goto err;
		}

		entry = 0;
		if (pci_dev_msi_enabled(pdev)) {
			mdev->irq_info.mic_msi_map[entry] |= (1 << offset);
			mdev->intr_ops->program_msi_to_src_map(mdev,
				entry, offset, true);
		}
		cookie = MK_COOKIE(entry, intr_cb->cb_id);
		dev_dbg(mdev->sdev->parent, "callback %d registered for src: %d\n",
			intr_cb->cb_id, intr_src);
	}
	return (struct mic_irq *)cookie;
err:
	return ERR_PTR(rc);
}

/**
 * mic_free_irq - free irq. mic_mutex
 *  needs to be held before calling this function.
 *
 * @mdev: pointer to mic_device instance
 * @cookie: cookie obtained during a successful call to mic_request_irq
 * @data: private data specified by the calling function during the
 * mic_request_irq
 *
 * returns: none.
 */
void mic_free_irq(struct mic_device *mdev,
	struct mic_irq *cookie, void *data)
{
	u32 offset;
	u32 entry;
	u8 src_id;
	unsigned int irq;
	struct pci_dev *pdev = container_of(mdev->sdev->parent,
		struct pci_dev, dev);

	entry = GET_ENTRY((unsigned long)cookie);
	offset = GET_OFFSET((unsigned long)cookie);
	if (mdev->irq_info.num_vectors > 1) {
		if (entry >= mdev->irq_info.num_vectors) {
			dev_warn(mdev->sdev->parent,
				 "entry %d should be < num_irq %d\n",
				entry, mdev->irq_info.num_vectors);
			return;
		}
		irq = mdev->irq_info.msix_entries[entry].vector;
		free_irq(irq, data);
		mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
		mdev->intr_ops->program_msi_to_src_map(mdev,
			entry, offset, false);

		dev_dbg(mdev->sdev->parent, "irq: %d freed\n", irq);
	} else {
		irq = pdev->irq;
		src_id = mic_unregister_intr_callback(mdev, offset);
		if (src_id >= MIC_NUM_OFFSETS) {
			dev_warn(mdev->sdev->parent, "Error unregistering callback\n");
			return;
		}
		if (pci_dev_msi_enabled(pdev)) {
			mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
			mdev->intr_ops->program_msi_to_src_map(mdev,
				entry, src_id, false);
		}
		dev_dbg(mdev->sdev->parent, "callback %d unregistered for src: %d\n",
			offset, src_id);
	}
}

/**
 * mic_setup_interrupts - Initializes interrupts.
 *
 * @mdev: pointer to mic_device instance
 * @pdev: PCI device structure
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
{
	int rc;

	rc = mic_setup_msix(mdev, pdev);
	if (!rc)
		goto done;

	rc = mic_setup_msi(mdev, pdev);
	if (!rc)
		goto done;

	rc = mic_setup_intx(mdev, pdev);
	if (rc) {
		dev_err(mdev->sdev->parent, "no usable interrupts\n");
		return rc;
	}
done:
	mdev->intr_ops->enable_interrupts(mdev);
	return 0;
}

/**
 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
 *
 * @mdev: pointer to mic_device instance
 * @pdev: PCI device structure
 *
 * returns none.
 */
void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
{
	int i;

	mdev->intr_ops->disable_interrupts(mdev);
	if (mdev->irq_info.num_vectors > 1) {
		for (i = 0; i < mdev->irq_info.num_vectors; i++) {
			if (mdev->irq_info.mic_msi_map[i])
				dev_warn(&pdev->dev, "irq %d may still be in use.\n",
					 mdev->irq_info.msix_entries[i].vector);
		}
		kfree(mdev->irq_info.mic_msi_map);
		kfree(mdev->irq_info.msix_entries);
		pci_disable_msix(pdev);
	} else {
		if (pci_dev_msi_enabled(pdev)) {
			free_irq(pdev->irq, mdev);
			kfree(mdev->irq_info.mic_msi_map);
			pci_disable_msi(pdev);
		} else {
			free_irq(pdev->irq, mdev);
		}
		mic_release_callbacks(mdev);
	}
}

/**
 * mic_intr_restore - Restore MIC interrupt registers.
 *
 * @mdev: pointer to mic_device instance.
 *
 * Restore the interrupt registers to values previously
 * stored in the SW data structures. mic_mutex needs to
 * be held before calling this function.
 *
 * returns None.
 */
void mic_intr_restore(struct mic_device *mdev)
{
	int entry, offset;
	struct pci_dev *pdev = container_of(mdev->sdev->parent,
		struct pci_dev, dev);

	if (!pci_dev_msi_enabled(pdev))
		return;

	for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
		for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
			if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
				mdev->intr_ops->program_msi_to_src_map(mdev,
					entry, offset, true);
		}
	}
}