cndm: Use notifier chain for interrupts

Signed-off-by: Alex Forencich <alex@alexforencich.com>

src/cndm/modules/cndm/Makefile

@@ -6,6 +6,7 @@ ifneq ($(KERNELRELEASE),)
 obj-m += cndm.o
 cndm-y += cndm_main.o
 cndm-y += cndm_devlink.o
+cndm-y += cndm_irq.o
 cndm-y += cndm_dev.o
 cndm-y += cndm_netdev.o
 cndm-y += cndm_ethtool.o

src/cndm/modules/cndm/cndm.h

@@ -22,6 +22,15 @@ Authors:
 #define DRIVER_NAME "cndm"
 #define DRIVER_VERSION "0.1"
 
+#define CNDM_MAX_IRQ 256
+
+struct cndm_irq {
+	int index;
+	int irqn;
+	char name[16+3];
+	struct atomic_notifier_head nh;
+};
+
 struct cndm_dev {
 	struct pci_dev *pdev;
 	struct device *dev;
@@ -31,6 +40,9 @@ struct cndm_dev {
 
 	struct miscdevice misc_dev;
 
+	int irq_count;
+	struct cndm_irq *irq[CNDM_MAX_IRQ];
+
 	struct net_device *ndev[32];
 
 	void __iomem *bar;
@@ -71,6 +83,9 @@ struct cndm_priv {
 	void *txq_region;
 	dma_addr_t txq_region_addr;
 
+	struct cndm_irq *irq;
+	struct notifier_block irq_nb;
+
 	struct cndm_tx_info *tx_info;
 	struct cndm_rx_info *rx_info;
 
@@ -133,8 +148,11 @@ struct cndm_cpl {
 struct devlink *cndm_devlink_alloc(struct device *dev);
 void cndm_devlink_free(struct devlink *devlink);
 
+// cndm_irq.c
+int cndm_irq_init_pcie(struct cndm_dev *cdev);
+void cndm_irq_deinit_pcie(struct cndm_dev *cdev);
+
 // cndm_netdev.c
-irqreturn_t cndm_irq(int irqn, void *data);
 struct net_device *cndm_create_netdev(struct cndm_dev *cdev, int port, void __iomem *hw_addr);
 void cndm_destroy_netdev(struct net_device *ndev);
 

src/cndm/modules/cndm/cndm_irq.c

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL
+/*
+
+Copyright (c) 2025 FPGA Ninja, LLC
+
+Authors:
+- Alex Forencich
+
+*/
+
+#include "cndm.h"
+
+static irqreturn_t cndm_irq_handler(int irqn, void *data)
+{
+	struct cndm_irq *irq = data;
+
+	atomic_notifier_call_chain(&irq->nh, 0, NULL);
+
+	return IRQ_HANDLED;
+}
+
+int cndm_irq_init_pcie(struct cndm_dev *cdev)
+{
+	struct pci_dev *pdev = cdev->pdev;
+	struct device *dev = cdev->dev;
+	int ret = 0;
+	int k;
+
+	cdev->irq_count = pci_alloc_irq_vectors(pdev, 1, CNDM_MAX_IRQ, PCI_IRQ_MSI | PCI_IRQ_MSIX);
+	if (cdev->irq_count < 0) {
+		dev_err(dev, "Failed to allocate IRQs");
+		return -ENOMEM;
+	}
+
+	for (k = 0; k < cdev->irq_count; k++) {
+		struct cndm_irq *irq;
+
+		irq = kzalloc(sizeof(*irq), GFP_KERNEL);
+		if (!irq) {
+			ret = -ENOMEM;
+			goto fail;
+		}
+
+		ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
+
+		ret = pci_request_irq(pdev, k, cndm_irq_handler, NULL,
+				irq, "%s-%d", cdev->name, k);
+		if (ret < 0) {
+			kfree(irq);
+			ret = -ENOMEM;
+			dev_err(dev, "Failed to request IRQ %d", k);
+			goto fail;
+		}
+
+		irq->index = k;
+		irq->irqn = pci_irq_vector(pdev, k);
+		cdev->irq[k] = irq;
+	}
+
+	dev_info(dev, "Configured %d IRQs", cdev->irq_count);
+
+	return 0;
+fail:
+	cndm_irq_deinit_pcie(cdev);
+	return ret;
+}
+
+void cndm_irq_deinit_pcie(struct cndm_dev *cdev)
+{
+	struct pci_dev *pdev = cdev->pdev;
+	int k;
+
+	for (k = 0; k < CNDM_MAX_IRQ; k++)
+	{
+		if (cdev->irq[k]) {
+			pci_free_irq(pdev, k, cdev->irq[k]);
+			kfree(cdev->irq[k]);
+			cdev->irq[k] = NULL;
+		}
+	}
+
+	pci_free_irq_vectors(pdev);
+}

src/cndm/modules/cndm/cndm_main.c

@@ -96,10 +96,10 @@ static int cndm_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto fail_map_bars;
 	}
 
-	ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI | PCI_IRQ_MSIX);
-	if (ret < 0) {
-		dev_err(dev, "Failed to allocate IRQs");
-		goto fail_map_bars;
+	ret = cndm_irq_init_pcie(cdev);
+	if (ret) {
+		dev_err(dev, "Failed to set up interrupts");
+		goto fail_init_irq;
 	}
 
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)
@@ -125,13 +125,6 @@ static int cndm_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			goto fail_netdev;
 		}
 
-		ret = pci_request_irq(pdev, k, cndm_irq, 0, ndev, DRIVER_NAME);
-		if (ret < 0) {
-			dev_err(dev, "Failed to request IRQ");
-			cndm_destroy_netdev(ndev);
-			goto fail_netdev;
-		}
-
 		cdev->ndev[k] = ndev;
 	}
 
@@ -156,13 +149,13 @@ fail_miscdev:
 fail_netdev:
 	for (k = 0; k < 32; k++) {
 		if (cdev->ndev[k]) {
-			pci_free_irq(pdev, k, cdev->ndev[k]);
 			cndm_destroy_netdev(cdev->ndev[k]);
 			cdev->ndev[k] = NULL;
 		}
 	}
 	devlink_unregister(devlink);
-	pci_free_irq_vectors(pdev);
+	cndm_irq_deinit_pcie(cdev);
+fail_init_irq:
 fail_map_bars:
 	if (cdev->bar)
 		pci_iounmap(pdev, cdev->bar);
@@ -191,13 +184,12 @@ static void cndm_pci_remove(struct pci_dev *pdev)
 
 	for (k = 0; k < 32; k++) {
 		if (cdev->ndev[k]) {
-			pci_free_irq(pdev, k, cdev->ndev[k]);
 			cndm_destroy_netdev(cdev->ndev[k]);
 			cdev->ndev[k] = NULL;
 		}
 	}
 	devlink_unregister(devlink);
-	pci_free_irq_vectors(pdev);
+	cndm_irq_deinit_pcie(cdev);
 	if (cdev->bar)
 		pci_iounmap(pdev, cdev->bar);
 	pci_release_regions(pdev);

src/cndm/modules/cndm/cndm_netdev.c

@@ -56,19 +56,18 @@ static const struct net_device_ops cndm_netdev_ops = {
 	.ndo_start_xmit = cndm_start_xmit,
 };
 
-irqreturn_t cndm_irq(int irqn, void *data)
+static int cndm_netdev_irq(struct notifier_block *nb, unsigned long action, void *data)
 {
-	struct net_device *ndev = data;
-	struct cndm_priv *priv = netdev_priv(ndev);
+	struct cndm_priv *priv = container_of(nb, struct cndm_priv, irq_nb);
 
-	netdev_dbg(ndev, "Interrupt");
+	netdev_dbg(priv->ndev, "Interrupt");
 
 	if (priv->port_up) {
 		napi_schedule_irqoff(&priv->tx_napi);
 		napi_schedule_irqoff(&priv->rx_napi);
 	}
 
-	return IRQ_HANDLED;
+	return NOTIFY_DONE;
 }
 
 struct net_device *cndm_create_netdev(struct cndm_dev *cdev, int port, void __iomem *hw_addr)
@@ -210,6 +209,15 @@ struct net_device *cndm_create_netdev(struct cndm_dev *cdev, int port, void __io
 
 	priv->registered = 1;
 
+	priv->irq_nb.notifier_call = cndm_netdev_irq;
+	priv->irq = cdev->irq[port % cdev->irq_count];
+	ret = atomic_notifier_chain_register(&priv->irq->nh, &priv->irq_nb);
+	if (ret) {
+		priv->irq = NULL;
+		goto fail;
+	}
+
+
 	return ndev;
 
 fail:
@@ -227,6 +235,11 @@ void cndm_destroy_netdev(struct net_device *ndev)
 	iowrite32(0x00000000, priv->hw_addr + 0x400);
 	iowrite32(0x00000000, priv->hw_addr + 0x300);
 
+	if (priv->irq)
+		atomic_notifier_chain_unregister(&priv->irq->nh, &priv->irq_nb);
+
+	priv->irq = NULL;
+
 	if (priv->registered)
 		unregister_netdev(ndev);