NIC passthrough perf test

lisa/sut_orchestrator/libvirt/libvirt_device_pool.py

@@ -186,24 +186,78 @@ def create_device_pool(
             vendor_id=vendor_id,
             device_id=device_id,
         )
-        primary_nic_iommu = self.get_primary_nic_id()
-        for item in device_list:
-            device = DeviceAddressSchema()
-            domain, bus, slot, fn = self._parse_pci_address_str(addr=item.slot)
-            device.domain = domain
-            device.bus = bus
-            device.slot = slot
-            device.function = fn
+        bdf_list = [i.slot for i in device_list]
+        self._create_pool(pool_type, bdf_list)
+
+    def create_device_pool_from_pci_addresses(
+        self,
+        pool_type: HostDevicePoolType,
+        pci_addr_list: List[str],
+    ) -> None:
+        self.available_host_devices[pool_type] = {}
+        for bdf in pci_addr_list:
+            domain, bus, slot, fn = self._parse_pci_address_str(bdf)
+            device = self._get_pci_address_instance(domain, bus, slot, fn)
             iommu_group = self._get_device_iommu_group(device)
-            is_vfio_pci = self._is_driver_vfio_pci(device)
 
-            if not is_vfio_pci and iommu_group not in primary_nic_iommu:
+            # Get all the devices of that iommu group
+            iommu_path = f"/sys/kernel/iommu_groups/{iommu_group}/devices"
+            bdf_list = [i.strip() for i in self.host_node.tools[Ls].list(iommu_path)]
+            bdf_list.append(bdf.strip())  # append the given device in list
+
+            self._create_pool(pool_type, bdf_list)
+
+    def _create_pool(
+        self,
+        pool_type: HostDevicePoolType,
+        bdf_list: List[str],
+    ) -> None:
+        iommu_grp_of_used_devices = []
+        primary_nic_iommu = self.get_primary_nic_id()
+        for bdf in bdf_list:
+            domain, bus, slot, fn = self._parse_pci_address_str(bdf)
+            dev = self._get_pci_address_instance(domain, bus, slot, fn)
+            is_vfio_pci = self._is_driver_vfio_pci(dev)
+            iommu_group = self._get_device_iommu_group(dev)
+
+            if iommu_group in iommu_grp_of_used_devices:
+                # No need to add this device in pool as one of the devices for this
+                # iommu group is in use
+                continue
+
+            if is_vfio_pci:
+                # Do not consider any device for pool if any device of same iommu group
+                # is already assigned
+                pool = self.available_host_devices.get(pool_type, {})
+                pool.pop(iommu_group, [])
+                self.available_host_devices[pool_type] = pool
+                iommu_grp_of_used_devices.append(iommu_group)
+            elif (
+                iommu_group not in primary_nic_iommu
+                and iommu_group not in iommu_grp_of_used_devices
+            ):
                 pool = self.available_host_devices.get(pool_type, {})
                 devices = pool.get(iommu_group, [])
-                devices.append(device)
+                if dev not in devices:
+                    devices.append(dev)
                 pool[iommu_group] = devices
                 self.available_host_devices[pool_type] = pool
 
+    def _get_pci_address_instance(
+        self,
+        domain: str,
+        bus: str,
+        slot: str,
+        fn: str,
+    ) -> DeviceAddressSchema:
+        device = DeviceAddressSchema()
+        device.domain = domain
+        device.bus = bus
+        device.slot = slot
+        device.function = fn
+
+        return device
+
     def _add_device_passthrough_xml(
         self,
         devices: ET.Element,

lisa/sut_orchestrator/libvirt/schema.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 from dataclasses_json import dataclass_json
 
@@ -48,6 +48,16 @@ class DeviceAddressSchema:
     slot: str = ""
     function: str = ""
 
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, DeviceAddressSchema):
+            return (
+                self.domain == other.domain
+                and self.bus == other.bus
+                and self.slot == other.slot
+                and self.function == other.function
+            )
+        return False
+
 
 # QEMU orchestrator's global configuration options.
 @dataclass_json()

lisa/sut_orchestrator/util/device_pool.py

@@ -1,6 +1,10 @@
-from typing import Any, List, Optional
+from typing import Any, List, Optional, cast
 
-from lisa.sut_orchestrator.util.schema import HostDevicePoolSchema, HostDevicePoolType
+from lisa.sut_orchestrator.util.schema import (
+    HostDevicePoolSchema,
+    HostDevicePoolType,
+    VendorDeviceIdIdentifier,
+)
 from lisa.util import LisaException
 
 
@@ -16,6 +20,13 @@ def create_device_pool(
     ) -> None:
         raise NotImplementedError()
 
+    def create_device_pool_from_pci_addresses(
+        self,
+        pool_type: HostDevicePoolType,
+        pci_addr_list: List[str],
+    ) -> None:
+        raise NotImplementedError()
+
     def get_primary_nic_id(self) -> List[str]:
         raise NotImplementedError()
 
@@ -44,22 +55,40 @@ def configure_device_passthrough_pool(
                         f"Pool type '{pool_type}' is not supported by platform"
                     )
             for config in device_configs:
-                vendor_device_list = config.devices
-                if len(vendor_device_list) > 1:
-                    raise LisaException(
-                        "Device Pool does not support more than one "
-                        "vendor/device id list for given pool type"
-                    )
+                devices = config.devices
+                if isinstance(devices, list) and all(
+                    isinstance(d, VendorDeviceIdIdentifier) for d in devices
+                ):
+                    if len(devices) > 1:
+                        raise LisaException(
+                            "Device Pool does not support more than one "
+                            "vendor/device id list for given pool type"
+                        )
+
+                    vendor_device_id = devices[0]
+                    assert vendor_device_id.vendor_id.strip()
+                    vendor_id = vendor_device_id.vendor_id.strip()
 
-                vendor_device_id = vendor_device_list[0]
-                assert vendor_device_id.vendor_id.strip()
-                vendor_id = vendor_device_id.vendor_id.strip()
+                    assert vendor_device_id.device_id.strip()
+                    device_id = vendor_device_id.device_id.strip()
 
-                assert vendor_device_id.device_id.strip()
-                device_id = vendor_device_id.device_id.strip()
+                    self.create_device_pool(
+                        pool_type=config.type,
+                        vendor_id=vendor_id,
+                        device_id=device_id,
+                    )
+                elif isinstance(devices, dict):
+                    bdf_list = devices.get("pci_bdf", [])
+                    assert bdf_list, "Key not found: 'pci_bdf'"
+                    pci_addr_list: List[str] = cast(List[str], bdf_list)
 
-                self.create_device_pool(
-                    pool_type=config.type,
-                    vendor_id=vendor_id,
-                    device_id=device_id,
-                )
+                    # Create pool from the list of PCI addresses
+                    self.create_device_pool_from_pci_addresses(
+                        pool_type=config.type,
+                        pci_addr_list=pci_addr_list,
+                    )
+                else:
+                    raise LisaException(
+                        f"Unknown device identifier of type: {type(devices)}"
+                        f", value: {devices}"
+                    )

lisa/sut_orchestrator/util/schema.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import List
+from typing import List, Union, cast
 
 from dataclasses_json import dataclass_json
 
@@ -12,17 +12,26 @@ class HostDevicePoolType(Enum):
 
 @dataclass_json()
 @dataclass
-class DeviceIdentifier:
+class VendorDeviceIdIdentifier:
     vendor_id: str = ""
     device_id: str = ""
 
 
+@dataclass_json()
+@dataclass
+class PciAddressIdentifier:
+    # list of bdf like 0000:3b:00.0 - <domain>:<bus>:<slot>.<fn>
+    pci_bdf: List[str] = field(default_factory=list)
+
+
 # Configuration options for device-passthrough for the VM.
 @dataclass_json()
 @dataclass
 class HostDevicePoolSchema:
     type: HostDevicePoolType = HostDevicePoolType.PCI_NIC
-    devices: List[DeviceIdentifier] = field(default_factory=list)
+    devices: Union[List[VendorDeviceIdIdentifier], PciAddressIdentifier] = field(
+        default_factory=lambda: cast(List[VendorDeviceIdIdentifier], [])
+    )
 
 
 @dataclass_json()

lisa/tools/iperf3.py

@@ -119,6 +119,7 @@ def run_as_server_async(
         use_json_format: bool = False,
         one_connection_only: bool = False,
         daemon: bool = True,
+        interface_ip: str = "",
     ) -> Process:
         # -s: run iperf3 as server mode
         # -D: run iperf3 as a daemon
@@ -135,6 +136,8 @@ def run_as_server_async(
             cmd += f" -f {report_unit} "
         if port:
             cmd += f" -p {port} "
+        if interface_ip:
+            cmd += f" -B {interface_ip}"
         process = self.node.execute_async(
             f"{self.command} {cmd}", shell=True, sudo=True
         )

lisa/tools/netperf.py

@@ -31,10 +31,17 @@ def can_install(self) -> bool:
     def dependencies(self) -> List[Type[Tool]]:
         return [Gcc, Git, Make, Texinfo]
 
-    def run_as_server(self, port: int = 30000, daemon: bool = True) -> None:
+    def run_as_server(
+        self,
+        port: int = 30000,
+        daemon: bool = True,
+        interface_ip: str = "",
+    ) -> None:
         cmd = f"netserver -p {port} "
         if not daemon:
             cmd += " -D "
+        if interface_ip:
+            cmd += f" -L {interface_ip}"
         self.node.execute(
             cmd,
             sudo=True,
@@ -50,9 +57,11 @@ def run_as_client_async(
         test_name: str = "TCP_RR",
         seconds: int = 150,
         time_unit: int = 1,
+        interface_ip: str = "",
         send_recv_offset: str = "THROUGHPUT, THROUGHPUT_UNITS, MIN_LATENCY, MAX_LATENCY, MEAN_LATENCY, REQUEST_SIZE, RESPONSE_SIZE, STDDEV_LATENCY",  # noqa: E501
     ) -> Process:
         # -H: Specify the target machine and/or local ip and family
+        # -L: Specify the IP for client interface to be used
         # -p: Specify netserver port number and/or local port
         # -t: Specify test to perform
         # -n: Set the number of processors for CPU util
@@ -61,8 +70,11 @@ def run_as_client_async(
         #     initial guess for units per second. A negative value for time will make
         #     heavy use of the system's timestamping functionality
         # -O: Set the remote send,recv buffer offset
-        cmd = (
-            f"-H {server_ip} -p {port} -t {test_name} -n {core_count} -l {seconds}"
+        cmd: str = ""
+        if interface_ip:
+            cmd += f" -L {interface_ip}"
+        cmd += (
+            f" -H {server_ip} -p {port} -t {test_name} -n {core_count} -l {seconds}"
             f" -D {time_unit} -- -O '{send_recv_offset}'"
         )
         process = self.node.execute_async(f"{self.command} {cmd}", sudo=True)

microsoft/testsuites/performance/common.py

@@ -207,6 +207,7 @@ def perf_tcp_pps(
     test_type: str,
     server: Optional[RemoteNode] = None,
     client: Optional[RemoteNode] = None,
+    use_internal_address: bool = False,
 ) -> None:
     # Either server and client are set explicitly or we use the first two nodes
     # from the environment. We never combine the two options. We need to specify
@@ -226,6 +227,14 @@ def perf_tcp_pps(
         [lambda: client.tools[Netperf], lambda: server.tools[Netperf]]  # type: ignore
     )
 
+    server_interface_ip: str = ""
+    client_interface_ip: str = ""
+    if use_internal_address:
+        assert server.internal_address, "Server Node: internal address is not set"
+        assert client.internal_address, "Client Node: internal address is not set"
+        server_interface_ip = server.internal_address
+        client_interface_ip = client.internal_address
+
     cpu = client.tools[Lscpu]
     core_count = cpu.get_core_count()
     if "maxpps" == test_type:
@@ -235,9 +244,14 @@ def perf_tcp_pps(
     else:
         ports = range(30000, 30001)
     for port in ports:
-        server_netperf.run_as_server(port)
+        server_netperf.run_as_server(port, interface_ip=server_interface_ip)
     for port in ports:
-        client_netperf.run_as_client_async(server.internal_address, core_count, port)
+        client_netperf.run_as_client_async(
+            server_ip=server.internal_address,
+            core_count=core_count,
+            port=port,
+            interface_ip=client_interface_ip,
+        )
     client_sar = client.tools[Sar]
     server_sar = server.tools[Sar]
     server_sar.get_statistics_async()
@@ -446,17 +460,33 @@ def perf_iperf(
     connections: List[int],
     buffer_length_list: List[int],
     udp_mode: bool = False,
+    server: Optional[RemoteNode] = None,
+    client: Optional[RemoteNode] = None,
+    run_with_internal_address: bool = False,
 ) -> None:
-    environment = test_result.environment
-    assert environment, "fail to get environment from testresult"
+    if server is not None or client is not None:
+        assert server is not None, "server need to be specified, if client is set"
+        assert client is not None, "client need to be specified, if server is set"
+    else:
+        environment = test_result.environment
+        assert environment, "fail to get environment from testresult"
+        # set server and client from environment, if not set explicitly
+        client = cast(RemoteNode, environment.nodes[0])
+        server = cast(RemoteNode, environment.nodes[1])
 
-    client = cast(RemoteNode, environment.nodes[0])
-    server = cast(RemoteNode, environment.nodes[1])
     client_iperf3, server_iperf3 = run_in_parallel(
-        [lambda: client.tools[Iperf3], lambda: server.tools[Iperf3]]
+        [lambda: client.tools[Iperf3], lambda: server.tools[Iperf3]]  # type: ignore
     )
     test_case_name = inspect.stack()[1][3]
     iperf3_messages_list: List[Any] = []
+    server_interface_ip = ""
+    client_interface_ip = ""
+    if run_with_internal_address:
+        server_interface_ip = server.internal_address
+        client_interface_ip = client.internal_address
+        assert server_interface_ip, "Server Node: internal address is not set"
+        assert client_interface_ip, "Client Node: internal address is not set"
+
     if udp_mode:
         for node in [client, server]:
             ssh = node.tools[Ssh]
@@ -481,7 +511,13 @@ def perf_iperf(
                 current_server_iperf_instances += 1
                 server_iperf3_process_list.append(
                     server_iperf3.run_as_server_async(
-                        current_server_port, "g", 10, True, True, False
+                        port=current_server_port,
+                        report_unit="g",
+                        report_periodic=10,
+                        use_json_format=True,
+                        one_connection_only=True,
+                        daemon=False,
+                        interface_ip=server_interface_ip,
                     )
                 )
                 current_server_port += 1
@@ -502,6 +538,7 @@ def perf_iperf(
                         parallel_number=num_threads_p,
                         ip_version="4",
                         udp_mode=udp_mode,
+                        client_ip=client_interface_ip,
                     )
                 )
                 current_client_port += 1