Mctrain's Blog

What I learned in IT, as well as thought about life

Netmap Setup

| Comments

I’ve recently read the paper netmap: a novel framework for fast packet I/O published in ATC 2012 as its best paper award. And I also try to setup it in my mini-pc, with Realtek whose driver is r8169, which is supported by netmap.

However since they only support r8169 before 3.4 linux kernel, and my kernel version is 3.13.7, so I need to write my own r8169 driver patch. Fortunately the r8169 code is not quite different between these two kernel versions, it is trivial to do so.

Let’s see how to setup the netmap environment and test it:

First we need to download a linux kernel source, for me I want to use the same kernel version with current one, so I use 3.13.7, download, compile and install it:

$ wget https://www.kernel.org/pub/linux/kernel/v3.x/linux-3.13.7.tar.gz
$ tar zxvf linux-3.13.7.tar.gz
$ cd linux-3.13.7
$ make menuconfig
$ cp /boot/config-3.13.7 ./.config
$ make -j4
$ sudo make modules_install
$ sudo make install
$ sudo update-grub2

Then reboot. Next step is make r8169 driver patch for linux-3.13.7/drivers/net/ethernet/realtek/r8169.c

Before showing how to do that, we first need to download the Netmap source code:

$ git clone https://code.google.com/p/netmap/
$ cd netmap/LINUX

You will find in patches folder, there are already many patches for different NICs, e.g., e1000, igb, ixgbe, etc., and when you find the diff--r8169.c--xxxxx--xxxxx, you can see the latest kernel version it supports is 30200--30400, which is linux 3.2.0 to 3.4.0, so based on this patch diff--r8169.c--30200--30400, I make my own patch for the kernel version 30d07 (3.13.7) as follows:

$ diff -Naur ./r8169.c ./r8169.bk.c > r8169.patch
$ cp r8169.patch /path/to/netmap/LINUX/patches/diff-r8169.c--30d07--30d08
$ cat diff-r8169.c--30d07--30d08
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
--- ./r8169.c 2014-03-24 12:45:42.000000000 +0800
+++ ./r8169.bk.c  2014-05-29 14:03:42.489702302 +0800
@@ -925,6 +925,12 @@
  return RTL_R32(OCPAR) & OCPAR_FLAG;
 }

+#if defined(CONFIG_NETMAP) || defined(CONFIG_NETMAP_MODULE)
+static int rtl_open(struct net_device *dev);
+static int rtl8169_close(struct net_device *dev);
+#include <if_re_netmap_linux.h>
+#endif
+
 static u32 ocp_read(struct rtl8169_private *tp, u8 mask, u16 reg)
 {
  void __iomem *ioaddr = tp->mmio_addr;
@@ -5771,6 +5777,12 @@
 {
  unsigned int i;

+#ifdef DEV_NETMAP
+ re_netmap_tx_init(tp);
+ if (re_netmap_rx_init(tp))
+   return 0; // success
+#endif /* DEV_NETMAP */
+
  for (i = 0; i < NUM_RX_DESC; i++) {
    void *data;

@@ -5854,6 +5866,11 @@
  int i;

  napi_disable(&tp->napi);
+
+#ifdef DEV_NETMAP
+ netmap_disable_all_rings(dev);
+#endif /* DEV_NETMAP */
+
  netif_stop_queue(dev);
  synchronize_sched();

@@ -5866,6 +5883,11 @@
  rtl8169_init_ring_indexes(tp);

  napi_enable(&tp->napi);
+
+#ifdef DEV_NETMAP
+ netmap_enable_all_rings(dev);
+#endif /* DEV_NETMAP */
+
  rtl_hw_start(dev);
  netif_wake_queue(dev);
  rtl8169_check_link_status(dev, tp, tp->mmio_addr);
@@ -6122,6 +6144,11 @@
 {
  unsigned int dirty_tx, tx_left;

+#ifdef DEV_NETMAP
+ if (netmap_tx_irq(dev, 0))
+   return;
+#endif /* DEV_NETMAP */
+
  dirty_tx = tp->dirty_tx;
  smp_rmb();
  tx_left = tp->cur_tx - dirty_tx;
@@ -6218,6 +6245,11 @@
  unsigned int cur_rx, rx_left;
  unsigned int count;

+#ifdef DEV_NETMAP
+ if (netmap_rx_irq(dev, 0, &count))
+       return count;
+#endif /* DEV_NETMAP */
+
  cur_rx = tp->cur_rx;

  for (rx_left = min(budget, NUM_RX_DESC); rx_left > 0; rx_left--, cur_rx++) {
@@ -6439,6 +6471,11 @@
  del_timer_sync(&tp->timer);

  napi_disable(&tp->napi);
+
+#ifdef DEV_NETMAP
+ netmap_disable_all_rings(dev);
+#endif /* DEV_NETMAP */
+
  netif_stop_queue(dev);

  rtl8169_hw_reset(tp);
@@ -6544,6 +6581,10 @@
  set_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags);

  napi_enable(&tp->napi);
+
+#ifdef DEV_NETMAP
+ netmap_enable_all_rings(dev);
+#endif /* DEV_NETMAP */

  rtl8169_init_phy(dev, tp);

@@ -6807,6 +6848,9 @@
  unregister_netdev(dev);

  rtl_release_firmware(tp);
+#ifdef DEV_NETMAP
+ netmap_detach(dev);
+#endif /* DEV_NETMAP */

  if (pci_dev_run_wake(pdev))
    pm_runtime_get_noresume(&pdev->dev);
@@ -7186,6 +7230,10 @@
  if (pci_dev_run_wake(pdev))
    pm_runtime_put_noidle(&pdev->dev);

+#ifdef DEV_NETMAP
+ re_netmap_attach(tp);
+#endif /* DEV_NETMAP */
+
  netif_carrier_off(dev);

 out:

After making the patch, we can compile the netmap:

$ make

There may be some compiling errors in if_re_netmap_linux.h, that is because the API of r8169 driver has changed, so just remove the rtl8169_wait_for_quiescence function invoking, and change rtl8169_open to rtl_open, then all errors are gone.

After the make process, there will generate many .ko file, then we can install the netmap and r8169 driver modules:

$ sudo rmmod r8169
$ sudo insmod netmap_lin.ko
$ sudo insmod r8169.ko

and compile the example apps they provided:

$ make apps

To test whether it succeed, we can use the compiled example pkt-gen:

$ cd /path/to/netmap/examples
$ sudo ./pkt-gen -i eth0 -f tx -l 64 -d 10.25.192.61 -D 74:d4:35:47:83:db

In my situation, the rate line can only reach ~300 Kpps, as the README says:

re/r8169 is extremely slow in sending (max 4-500 Kpps)

So the environment is already setup.

Comments