diff --git a/nodescraper/plugins/inband/network/ethtool_vendor.py b/nodescraper/plugins/inband/network/ethtool_vendor.py index d43791b4..60aaa4ca 100644 --- a/nodescraper/plugins/inband/network/ethtool_vendor.py +++ b/nodescraper/plugins/inband/network/ethtool_vendor.py @@ -25,6 +25,7 @@ ############################################################################### """Vendor-specific ethtool -S statistics models (Pollara / Thor2 / ConnectX-7).""" +import re from typing import ClassVar, Optional, Union from pydantic import BaseModel, Field, model_validator @@ -32,7 +33,18 @@ class PollaraEthtoolStatistics(BaseModel): - """ifname ionic. Keeping only fields of interest. Skip queue-specific stats for now""" + """ionic (Pollara) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^(rx|tx)_\d+" + queue_error_regex: ClassVar[list[str]] = [ + r"dma_map_err", + r"hwstamp_invalid", + r"alloc_err", + r"csum_error", + r"dropped", + ] + queue_warning_regex: ClassVar[list[str]] = [] rx_csum_error: Optional[int] = None hw_tx_dropped: Optional[int] = None @@ -157,7 +169,27 @@ class PollaraEthtoolStatistics(BaseModel): class Thor2EthtoolStatistics(BaseModel): - """ifname bnxt. Keeping only fields of interest. Skip queue-specific stats for now""" + """bnxt (Thor2) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^\[\d+\]" + queue_error_regex: ClassVar[list[str]] = [ + r"rx_discards", + r"rx_errors", + r"tx_errors", + r"tx_discards", + r"rx_l4_csum_errors", + r"rx_buf_errors", + r"so_txtime_cmpl_errors", + r"missed_irqs", + r"xsk_rx_redirect_fail", + r"xsk_rx_alloc_fail", + r"xsk_rx_no_room", + r"xsk_tx_ring_full", + ] + queue_warning_regex: ClassVar[list[str]] = [ + r"rx_resets", + ] rx_total_l4_csum_errors: Optional[int] = None rx_total_resets: Optional[int] = None @@ -356,7 +388,22 @@ class Thor2EthtoolStatistics(BaseModel): class Cx7EthtoolStatistics(BaseModel): - """ifname mlx. Keeping only fields of interest. Skip queue-specific stats for now""" + """mlx (ConnectX-7) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^(rx|tx|ch)\d+" + queue_error_regex: ClassVar[list[str]] = [ + r"xdp_drop", + r"wqe_err", + r"oversize_pkts_sw_drop", + r"buff_alloc_err", + r"arfs_err", + r"tls_err", + r"xdp_tx_err", + r"dropped", + r"cqe_err", + ] + queue_warning_regex: ClassVar[list[str]] = [] rx_xdp_drop: Optional[int] = None rx_xdp_tx_err: Optional[int] = None @@ -645,15 +692,51 @@ class Cx7EthtoolStatistics(BaseModel): } +def extract_queue_counters(stats: dict[str, str], queue_counter_regex: str) -> dict[str, int]: + """Extract per-queue counters from a raw ``ethtool -S`` stats dict. + + Devices can report hundreds of per-queue counters. These are matched by the + vendor's ``queue_counter_regex`` and retained (rather than dropped like other + non-enumerated fields) so they remain available in the data model for error + detection. + + Args: + stats: Mapping of raw ``ethtool -S`` field name -> string value. + queue_counter_regex: Vendor regex matched against field names to identify + per-queue counters. + + Returns: + Mapping of matching field name -> integer value. The ``netdev`` marker key + and non-integer values are skipped. + """ + pattern = re.compile(queue_counter_regex) + queue_counters: dict[str, int] = {} + for name, value in stats.items(): + if name == "netdev" or not pattern.match(name): + continue + try: + queue_counters[name] = int(value) + except (TypeError, ValueError): + continue + return queue_counters + + class EthtoolStatistics(BaseModel): """Per-netdev ethtool -S row with optional vendor-parsed counters.""" netdev: Optional[str] = None - rdma_ifname: Optional[str] = Field( + driver: Optional[str] = Field( default=None, - description="RDMA interface name from 'rdma link -j' used for vendor prefix selection", + description="Kernel driver (from 'ethtool -i') used for vendor model selection", ) vendor_statistics: Optional[VendorEthtoolStatisticsModel] = None + queue_statistics: dict[str, int] = Field( + default_factory=dict, + description=( + "High-cardinality per-queue ethtool -S counters (raw field name -> value) " + "captured via the vendor queue_counter_regex for error detection" + ), + ) @model_validator(mode="after") def validate_atleast_one_field(self) -> Self: diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 2967df59..eb590651 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer @@ -49,13 +50,13 @@ def analyze_data( """Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex. Args: - data: Network data model with ethtool_info and/or rdma_ethtool_statistics. + data: Network data model with ethtool_info and/or ethtool_statistics. args: Optional analyzer arguments with custom error regex support. Returns: TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ - if not data.ethtool_info and not data.rdma_ethtool_statistics: + if not data.ethtool_info and not data.ethtool_statistics: self.result.message = "No network devices found" self.result.status = ExecutionStatus.WARNING return self.result @@ -110,7 +111,9 @@ def analyze_data( vendor_warning = False vendor_error_fields: set[str] = set() vendor_warning_fields: set[str] = set() - for stat in data.rdma_ethtool_statistics: + vendor_queue_error_fields: set[str] = set() + vendor_queue_warning_fields: set[str] = set() + for stat in data.ethtool_statistics: if stat.vendor_statistics is None: continue @@ -129,24 +132,18 @@ def analyze_data( else: vendor_error = True vendor_error_fields.add(field_name) - # Use a single grouped description per severity so the run summary - # collapses every occurrence into one "Ethtool warning detected" / - # "Ethtool error detected" entry instead of one line per field. The - # specific field is still preserved in the event data below and in - # the consolidated console line emitted after the loop. + # Use a single grouped description per severity desc = ( "Ethtool warning detected" if is_warning_tier else "Ethtool error detected" ) # Per-field events are still recorded for the run summary and event - # log, but console logging is suppressed here to avoid repeating the - # same message once per device. A single consolidated line is emitted - # after the loop instead. + # log, but console logging is suppressed here self._log_event( category=EventCategory.NETWORK, description=desc, data={ "netdev": stat.netdev, - "rdma_ifname": stat.rdma_ifname, + "driver": stat.driver, "error_field": field_name, "error_count": error_value, }, @@ -154,12 +151,68 @@ def analyze_data( console_log=False, ) - if vendor_error: + # Per-queue counters matched against the vendor's adjustable regex + queue_error_patterns = [ + re.compile(pattern) for pattern in getattr(type(vs), "queue_error_regex", []) + ] + queue_warning_patterns = [ + re.compile(pattern) for pattern in getattr(type(vs), "queue_warning_regex", []) + ] + if stat.queue_statistics and (queue_error_patterns or queue_warning_patterns): + for counter_name, counter_value in stat.queue_statistics.items(): + if counter_value <= 0: + continue + if queue_error_patterns and any( + pattern.search(counter_name) for pattern in queue_error_patterns + ): + vendor_error = True + vendor_queue_error_fields.add(f"{stat.netdev} {counter_name}") + self._log_event( + category=EventCategory.NETWORK, + description="Ethtool queue error detected", + data={ + "netdev": stat.netdev, + "driver": stat.driver, + "error_field": counter_name, + "error_count": counter_value, + }, + priority=EventPriority.ERROR, + console_log=False, + ) + elif queue_warning_patterns and any( + pattern.search(counter_name) for pattern in queue_warning_patterns + ): + vendor_warning = True + vendor_queue_warning_fields.add(f"{stat.netdev} {counter_name}") + self._log_event( + category=EventCategory.NETWORK, + description="Ethtool queue warning detected", + data={ + "netdev": stat.netdev, + "driver": stat.driver, + "error_field": counter_name, + "error_count": counter_value, + }, + priority=EventPriority.WARNING, + console_log=False, + ) + + if vendor_error_fields: self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields))) - if vendor_warning: + if vendor_queue_error_fields: + self.logger.error( + "Ethtool queue error detected: %s", + ", ".join(sorted(vendor_queue_error_fields)), + ) + if vendor_warning_fields: self.logger.warning( "Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields)) ) + if vendor_queue_warning_fields: + self.logger.warning( + "Ethtool queue warning detected: %s", + ", ".join(sorted(vendor_queue_warning_fields)), + ) if regex_error or vendor_error: self.result.message = "Network errors detected in statistics" diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index b960e574..942ae93e 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -23,7 +23,6 @@ # SOFTWARE. # ############################################################################### -import json import re from typing import Dict, List, Optional, Tuple @@ -39,6 +38,7 @@ VENDOR_PREFIX_MAP, EthtoolStatistics, VendorEthtoolStatisticsModel, + extract_queue_counters, ) from .networkdata import ( EthtoolInfo, @@ -61,7 +61,7 @@ class NetworkCollector(InBandDataCollector[NetworkDataModel, NetworkCollectorArg CMD_NEIGHBOR = "ip neighbor show" CMD_ETHTOOL_TEMPLATE = "ethtool {interface}" CMD_ETHTOOL_S_TEMPLATE = "ethtool -S {interface}" - CMD_RDMA_LINK_JSON = "rdma link -j" + CMD_ETHTOOL_I_TEMPLATE = "ethtool -i {interface}" CMD_PING = "ping" CMD_WGET = "wget" CMD_CURL = "curl" @@ -453,6 +453,10 @@ def _parse_ethtool(self, interface: str, output: str) -> EthtoolInfo: def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, str]: """Parse 'ethtool -S ' output into a key-value dictionary. + Per-queue counters keep their full raw field name (e.g. "[0]: rx_ucast_packets") + so vendor ``queue_counter_regex`` patterns can match them; scalar counters are + stored under their plain name (e.g. "rx_total_l4_csum_errors"). + Args: output: Raw output from 'ethtool -S ' command interface: Name of the network interface (for netdev key) @@ -462,24 +466,18 @@ def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, st """ stats_dict: Dict[str, str] = {} for line in output.splitlines(): - if ":" not in line: + line = line.strip() + if not line: continue if "NIC statistics" in line: stats_dict["netdev"] = interface - elif "]: " in line and line.strip().startswith("["): - # Format: " [0]: rx_ucast_packets: 162" - bracket_part, rest = line.split("]: ", 1) - index = bracket_part.strip().lstrip("[") - if ": " in rest: - stat_key, stat_value = rest.split(": ", 1) - key = f"{index}_{stat_key.strip()}" - stats_dict[key] = stat_value.strip() - else: - key, value = line.split(":", 1) - stats_dict[key.strip()] = value.strip() - else: - key, value = line.split(":", 1) - stats_dict[key.strip()] = value.strip() + continue + if ": " not in line: + continue + # Split on the last ": " so per-queue lines like "[0]: rx_ucast_packets: 5" + # preserve their full "[0]: rx_ucast_packets" name for vendor queue matching. + name, value = line.rsplit(": ", 1) + stats_dict[name.strip()] = value.strip() return stats_dict def _collect_ethtool_info( @@ -526,47 +524,12 @@ def _collect_ethtool_info( return ethtool_data, skipped - def _collect_rdma_link_json(self) -> Optional[list[dict]]: - """Parse JSON from `rdma link -j`. Returns None on failure, [] when no links.""" - res = self._run_sut_cmd(self.CMD_RDMA_LINK_JSON) - if res.exit_code != 0: - self._log_event( - category=EventCategory.NETWORK, - description="rdma link -j failed (RDMA-scoped ethtool collection skipped)", - data={ - "command": self.CMD_RDMA_LINK_JSON, - "exit_code": res.exit_code, - "stderr": res.stderr, - }, - priority=EventPriority.WARNING, - ) - return None - if not res.stdout.strip(): - return [] - try: - parsed = json.loads(res.stdout) - except json.JSONDecodeError as e: - self._log_event( - category=EventCategory.NETWORK, - description="Failed to parse rdma link -j JSON", - data={"exception": get_exception_traceback(e)}, - priority=EventPriority.WARNING, - ) - return None - if not isinstance(parsed, list): - self._log_event( - category=EventCategory.NETWORK, - description="Unexpected rdma link -j JSON type", - data={"data_type": type(parsed).__name__}, - priority=EventPriority.WARNING, - ) - return None - return parsed + def _collect_ethtool_statistic(self, netdev: str, driver: str) -> Optional[EthtoolStatistics]: + """Run `ethtool -S` for netdev and attach vendor-parsed stats. - def _collect_rdma_scoped_ethtool_statistic( - self, netdev: str, ifname: str - ) -> Optional[EthtoolStatistics]: - """Run `ethtool -S` for netdev and attach vendor-parsed stats (prefix from RDMA ifname).""" + The vendor model is selected by matching ``driver`` (from ``ethtool -i``) + against the known vendor prefixes in ``VENDOR_PREFIX_MAP``. + """ cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=netdev) res = self._run_sut_cmd(cmd_s, sudo=True) if res.exit_code != 0: @@ -586,11 +549,16 @@ def _collect_rdma_scoped_ethtool_statistic( stats_dict = self._parse_ethtool_statistics(res.stdout, netdev) vendor_stats: Optional[VendorEthtoolStatisticsModel] = None + queue_statistics: Dict[str, int] = {} for prefix, vendor_cls in VENDOR_PREFIX_MAP.items(): - if ifname.startswith(prefix): + if driver.startswith(prefix): vendor_fields = set(vendor_cls.model_fields.keys()) stat_fields = set(stats_dict.keys()) - {"netdev"} + queue_statistics = extract_queue_counters( + stats_dict, vendor_cls.queue_counter_regex + ) + missing_fields = vendor_fields - stat_fields if missing_fields: sorted_missing = sorted(missing_fields) @@ -599,7 +567,7 @@ def _collect_rdma_scoped_ethtool_statistic( description=f"Missing fields in ethtool statistic for {netdev}", data={ "netdev": netdev, - "ifname": ifname, + "driver": driver, "missing_fields_count": len(sorted_missing), "missing_fields": sorted_missing[:50], }, @@ -620,18 +588,40 @@ def _collect_rdma_scoped_ethtool_statistic( return EthtoolStatistics( netdev=netdev, - rdma_ifname=ifname or None, + driver=driver or None, vendor_statistics=vendor_stats, + queue_statistics=queue_statistics, ) - def _collect_rdma_scoped_ethtool( - self, exclusions: Optional[List[re.Pattern]] = None + def _get_netdev_driver(self, netdev: str) -> str: + """Return the kernel driver for a netdev via `ethtool -i` ("" on failure).""" + cmd = self.CMD_ETHTOOL_I_TEMPLATE.format(interface=netdev) + res = self._run_sut_cmd(cmd, sudo=True) + if res.exit_code != 0: + return "" + for line in res.stdout.splitlines(): + line = line.strip() + if line.startswith("driver:"): + return line.split(":", 1)[1].strip() + return "" + + def _collect_ethtool_statistics( + self, + interfaces: List[NetworkInterface], + exclusions: Optional[List[re.Pattern]] = None, ) -> tuple[List[str], List[EthtoolStatistics], set[str]]: - """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity). + """Collect ethtool -S for vendor netdevs discovered from `ip addr`. + + Each interface's driver is resolved via `ethtool -i`; only interfaces whose + driver matches a known vendor prefix (see VENDOR_PREFIX_MAP) are collected. The + matched driver selects the respective vendor data model used to parse the + `ethtool -S` counters. Interfaces without a known vendor driver (lo, docker, + veth, ...) are skipped, which avoids noisy `ethtool -S` failures. This does not + depend on RDMA. Args: - exclusions: Compiled regex patterns; netdevs whose name matches any pattern - are skipped (ethtool -S is not run against them). + interfaces: Interfaces discovered from `ip addr` to consider. + exclusions: Compiled regex patterns; matching netdevs are skipped. Returns: Tuple of (netdev list, statistics list, set of netdev names skipped due to @@ -641,38 +631,27 @@ def _collect_rdma_scoped_ethtool( statistics_list: List[EthtoolStatistics] = [] skipped: set[str] = set() - link_data = self._collect_rdma_link_json() - if link_data is None: - return netdev_list, statistics_list, skipped - - for link in link_data: - if not isinstance(link, dict): - self._log_event( - category=EventCategory.NETWORK, - description="Invalid data type for RDMA link entry", - data={"data_type": type(link).__name__}, - priority=EventPriority.WARNING, - ) + for iface in interfaces: + netdev = iface.name + if not netdev: continue - - netdev = link.get("netdev") or "" - ifname = link.get("ifname") or "" - - if netdev: - if exclusions and any(pattern.search(netdev) for pattern in exclusions): - skipped.add(netdev) - continue - netdev_list.append(netdev) - stat = self._collect_rdma_scoped_ethtool_statistic(netdev, ifname) - if stat is not None: - statistics_list.append(stat) + if exclusions and any(pattern.search(netdev) for pattern in exclusions): + skipped.add(netdev) + continue + driver = self._get_netdev_driver(netdev) + if not any(driver.startswith(prefix) for prefix in VENDOR_PREFIX_MAP): + continue + netdev_list.append(netdev) + stat = self._collect_ethtool_statistic(netdev, driver) + if stat is not None: + statistics_list.append(stat) if netdev_list: self._log_event( category=EventCategory.NETWORK, description=( - f"Collected RDMA-scoped ethtool -S for {len(statistics_list)}/" - f"{len(netdev_list)} netdev(s) from rdma link" + f"Collected ethtool -S for {len(statistics_list)}/" + f"{len(netdev_list)} vendor netdev(s)" ), priority=EventPriority.INFO, ) @@ -778,8 +757,8 @@ def collect_data( neighbors = [] ethtool_data: Dict[str, EthtoolInfo] = {} network_accessible: Optional[bool] = None - rdma_ethtool_netdevs: List[str] = [] - rdma_ethtool_statistics: List[EthtoolStatistics] = [] + ethtool_netdevs: List[str] = [] + ethtool_statistics: List[EthtoolStatistics] = [] skipped_devices: set[str] = set() # Compile device-exclusion regex patterns (netdevs matching any pattern are skipped) @@ -835,11 +814,11 @@ def collect_data( if self.system_info.os_family == OSFamily.LINUX: ( - rdma_ethtool_netdevs, - rdma_ethtool_statistics, - rdma_skipped, - ) = self._collect_rdma_scoped_ethtool(compiled_exclusions) - skipped_devices |= rdma_skipped + ethtool_netdevs, + ethtool_statistics, + ethtool_stat_skipped, + ) = self._collect_ethtool_statistics(interfaces, compiled_exclusions) + skipped_devices |= ethtool_stat_skipped # Collect routing table res_route = self._run_sut_cmd(self.CMD_ROUTE) @@ -905,8 +884,8 @@ def collect_data( rules=rules, neighbors=neighbors, ethtool_info=ethtool_data, - rdma_ethtool_netdevs=rdma_ethtool_netdevs, - rdma_ethtool_statistics=rdma_ethtool_statistics, + ethtool_netdevs=ethtool_netdevs, + ethtool_statistics=ethtool_statistics, accessible=network_accessible, ) self.result.status = ExecutionStatus.OK diff --git a/nodescraper/plugins/inband/network/networkdata.py b/nodescraper/plugins/inband/network/networkdata.py index c90a1fc1..25b5f226 100644 --- a/nodescraper/plugins/inband/network/networkdata.py +++ b/nodescraper/plugins/inband/network/networkdata.py @@ -119,7 +119,7 @@ class NetworkDataModel(DataModel): ethtool_info: Dict[str, EthtoolInfo] = Field( default_factory=dict ) # Interface name -> EthtoolInfo mapping - # RDMA-scoped ethtool -S: netdevs from `rdma link -j` with vendor-parsed counters - rdma_ethtool_netdevs: List[str] = Field(default_factory=list) - rdma_ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) + # ethtool -S vendor-parsed counters for netdevs whose driver matches a known vendor + ethtool_netdevs: List[str] = Field(default_factory=list) + ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) accessible: Optional[bool] = None # Network accessibility check via ping diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 740f5998..90d68aa4 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -31,7 +31,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily -from nodescraper.models import TaskResult +from nodescraper.models import CollectorArgs, TaskResult from nodescraper.utils import get_exception_traceback from .rdmadata import ( @@ -45,7 +45,7 @@ ) -class RdmaCollector(InBandDataCollector[RdmaDataModel, None]): +class RdmaCollector(InBandDataCollector[RdmaDataModel, CollectorArgs]): """Collect RDMA status and statistics via rdma link and rdma statistic commands.""" DATA_MODEL = RdmaDataModel @@ -291,7 +291,9 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: ) return None - def collect_data(self, args=None) -> tuple[TaskResult, Optional[RdmaDataModel]]: + def collect_data( + self, args: Optional[CollectorArgs] = None + ) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. Returns: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index 4bf5e8cc..f435ff7a 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -24,6 +24,7 @@ # ############################################################################### from nodescraper.base import InBandDataPlugin +from nodescraper.models import CollectorArgs from .analyzer_args import RdmaAnalyzerArgs from .rdma_analyzer import RdmaAnalyzer @@ -31,10 +32,11 @@ from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, CollectorArgs, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector + COLLECTOR_ARGS = CollectorArgs ANALYZER = RdmaAnalyzer ANALYZER_ARGS = RdmaAnalyzerArgs diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 832125e5..2eb74490 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -192,13 +192,13 @@ def test_empty_ethtool_info(network_analyzer): def test_rdma_ethtool_vendor_error_only(network_analyzer): - """RDMA-scoped vendor ethtool: error-tier counter raises ERROR.""" + """Vendor ethtool: error-tier counter raises ERROR.""" stat = EthtoolStatistics( netdev="eth0", - rdma_ifname="bnxt0", + driver="bnxt_en", vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "Network errors detected" in result.message @@ -209,13 +209,13 @@ def test_rdma_ethtool_vendor_error_only(network_analyzer): def test_rdma_ethtool_vendor_warning_only(network_analyzer): - """RDMA-scoped vendor ethtool: only warning-tier counters -> WARNING status.""" + """Vendor ethtool: only warning-tier counters -> WARNING status.""" stat = EthtoolStatistics( netdev="eth0", - rdma_ifname="bnxt0", + driver="bnxt_en", vendor_statistics=Thor2EthtoolStatistics(rx_pause_frames=2), ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.WARNING assert "warning counters" in result.message @@ -234,9 +234,9 @@ def test_thor2_tx_pause_counters_are_warning_tier(): def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): - """RDMA ethtool row without parsed vendor statistics is ignored by vendor path.""" - stat = EthtoolStatistics(netdev="eth0", rdma_ifname="unknown0", vendor_statistics=None) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + """Ethtool row without parsed vendor statistics is ignored by vendor path.""" + stat = EthtoolStatistics(netdev="eth0", driver="unknown", vendor_statistics=None) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 @@ -399,6 +399,207 @@ def test_custom_error_regex_detected(network_analyzer): assert result.events[0].data["errors"] == {"custom_tx_drops": 9} +def test_queue_counter_error_detected(network_analyzer): + """A non-zero per-queue error counter (e.g. rx_discards) is flagged as ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 100, # benign, ignored + "[0]: rx_discards": 5, # error + "[3]: tx_errors": 2, # error + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + assert "Network errors detected" in result.message + + queue_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert len(queue_events) == 2 + flagged = {e.data["error_field"] for e in queue_events} + assert flagged == {"[0]: rx_discards", "[3]: tx_errors"} + assert all(e.priority == EventPriority.ERROR for e in queue_events) + + +def test_queue_counter_thor2_specific_error_fields(network_analyzer): + """Each curated Thor2 per-queue error pattern flags its non-zero counter as ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 1, + "[1]: rx_errors": 2, + "[2]: tx_errors": 3, + "[3]: rx_l4_csum_errors": 4, + "[4]: rx_buf_errors": 5, + "[5]: so_txtime_cmpl_errors": 6, + "[6]: missed_irqs": 7, + "[7]: xsk_rx_redirect_fail": 8, + "[8]: xsk_rx_alloc_fail": 9, + "[9]: xsk_rx_no_room": 10, + "[10]: xsk_tx_ring_full": 11, + # benign counters must NOT be flagged + "[0]: rx_ucast_packets": 12345, + "[0]: tx_ucast_bytes": 67890, + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + + error_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + flagged = {e.data["error_field"] for e in error_events} + assert flagged == { + "[0]: rx_discards", + "[1]: rx_errors", + "[2]: tx_errors", + "[3]: rx_l4_csum_errors", + "[4]: rx_buf_errors", + "[5]: so_txtime_cmpl_errors", + "[6]: missed_irqs", + "[7]: xsk_rx_redirect_fail", + "[8]: xsk_rx_alloc_fail", + "[9]: xsk_rx_no_room", + "[10]: xsk_tx_ring_full", + } + assert "[0]: rx_ucast_packets" not in flagged + assert "[0]: tx_ucast_bytes" not in flagged + assert all(e.priority == EventPriority.ERROR for e in error_events) + + +def test_queue_counter_benign_not_reported(network_analyzer): + """Non-zero benign per-queue counters (packets/bytes) do not raise errors.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 12345, + "[0]: tx_ucast_bytes": 67890, + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + + +def test_queue_counter_zero_not_reported(network_analyzer): + """A per-queue error counter that is zero does not raise an error.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={"[0]: rx_discards": 0, "[1]: tx_errors": 0}, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + + +def test_queue_counter_warning_detected(network_analyzer): + """A non-zero warning-tier per-queue counter (e.g. rx_resets) -> WARNING.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 100, # benign, ignored + "[0]: rx_resets": 3, # warning-tier + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.WARNING + assert "warning counters" in result.message + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert len(warn_events) == 1 + assert warn_events[0].data["error_field"] == "[0]: rx_resets" + assert warn_events[0].priority == EventPriority.WARNING + + +def test_queue_counter_error_precedence_over_warning(network_analyzer): + """When both error and warning per-queue counters are non-zero, status is ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 5, # error-tier + "[0]: rx_resets": 3, # warning-tier + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert {e.data["error_field"] for e in err_events} == {"[0]: rx_discards"} + assert {e.data["error_field"] for e in warn_events} == {"[0]: rx_resets"} + + +def test_queue_counter_empty_warning_list_skips_warning_check(network_analyzer, monkeypatch): + """An empty queue_warning_regex skips warning checking; error checking still runs.""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_warning_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_resets": 3, # would be a warning, but warning list is empty + "[0]: rx_discards": 5, # still flagged as an error + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert warn_events == [] + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert {e.data["error_field"] for e in err_events} == {"[0]: rx_discards"} + + +def test_queue_counter_empty_error_list_skips_error_check(network_analyzer, monkeypatch): + """An empty queue_error_regex skips error checking; warning checking still runs.""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_error_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 5, # would be an error, but error list is empty + "[0]: rx_resets": 3, # still flagged as a warning + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.WARNING + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert err_events == [] + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert {e.data["error_field"] for e in warn_events} == {"[0]: rx_resets"} + + +def test_queue_counter_both_lists_empty_skips_all_checks(network_analyzer, monkeypatch): + """Empty error and warning lists skip per-queue checking entirely (status OK).""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_error_regex", []) + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_warning_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={"[0]: rx_discards": 5, "[0]: rx_resets": 3}, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + + def test_custom_warning_priority_regex(network_analyzer): """A user-supplied WARNING-priority custom pattern yields WARNING status via the regex path.""" ethtool = EthtoolInfo( diff --git a/test/unit/plugin/test_network_collector.py b/test/unit/plugin/test_network_collector.py index a7b1faae..909e1319 100644 --- a/test/unit/plugin/test_network_collector.py +++ b/test/unit/plugin/test_network_collector.py @@ -30,6 +30,12 @@ from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.network.ethtool_vendor import ( + Cx7EthtoolStatistics, + PollaraEthtoolStatistics, + Thor2EthtoolStatistics, + extract_queue_counters, +) from nodescraper.plugins.inband.network.network_collector import NetworkCollector from nodescraper.plugins.inband.network.networkdata import ( EthtoolInfo, @@ -510,7 +516,10 @@ def test_parse_ethtool_empty_output(collector): def test_parse_ethtool_statistics(collector): - """Test parsing ethtool -S output (statistics) for error/health analysis.""" + """Test parsing ethtool -S output (statistics) for error/health analysis. + + Per-queue bracket lines keep their full raw name so vendor queue patterns match. + """ output = """NIC statistics: [0]: rx_ucast_packets: 162692536538787551 [0]: rx_errors: 0 @@ -519,13 +528,51 @@ def test_parse_ethtool_statistics(collector): rx_total_buf_errors: 0""" stats = collector._parse_ethtool_statistics(output, "abc1p1") assert stats.get("netdev") == "abc1p1" - assert stats.get("0_rx_ucast_packets") == "162692536538787551" - assert stats.get("0_rx_errors") == "0" - assert stats.get("1_rx_ucast_packets") == "79657418409137764" + assert stats.get("[0]: rx_ucast_packets") == "162692536538787551" + assert stats.get("[0]: rx_errors") == "0" + assert stats.get("[1]: rx_ucast_packets") == "79657418409137764" assert stats.get("rx_total_l4_csum_errors") == "0" assert stats.get("rx_total_buf_errors") == "0" +def test_extract_queue_counters_thor2(): + """Thor2 per-queue counters are the bracket-indexed "[]" lines.""" + stats = { + "netdev": "benic1p1", + "[0]: rx_ucast_packets": "5", + "[15]: tx_errors": "0", + "rx_total_l4_csum_errors": "0", + } + queue = extract_queue_counters(stats, Thor2EthtoolStatistics.queue_counter_regex) + assert queue == {"[0]: rx_ucast_packets": 5, "[15]: tx_errors": 0} + assert "rx_total_l4_csum_errors" not in queue + assert "netdev" not in queue + + +def test_extract_queue_counters_pollara(): + """Pollara per-queue counters start with "rx_" / "tx_".""" + stats = { + "rx_0_pkts": "3", + "tx_12_bytes": "9", + "rx_csum_error": "0", + "frames_tx_pri_0": "1", + } + queue = extract_queue_counters(stats, PollaraEthtoolStatistics.queue_counter_regex) + assert queue == {"rx_0_pkts": 3, "tx_12_bytes": 9} + + +def test_extract_queue_counters_cx7(): + """CX7 per-queue counters start with "rx" / "tx".""" + stats = { + "rx0_packets": "7", + "tx3_bytes": "8", + "rx_out_of_buffer": "0", + "rx_prio0_pause": "2", + } + queue = extract_queue_counters(stats, Cx7EthtoolStatistics.queue_counter_regex) + assert queue == {"rx0_packets": 7, "tx3_bytes": 8} + + def test_network_data_model_creation(collector): """Test creating NetworkDataModel with all components""" interface = NetworkInterface( @@ -650,13 +697,10 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert accessible is False -def test_collect_data_includes_rdma_ethtool(collector, conn_mock): - """RDMA-scoped ethtool -S is stored on NetworkDataModel when rdma link succeeds.""" - import json - +def test_collect_data_includes_ethtool_statistics(collector, conn_mock): + """ethtool -S is collected for vendor NICs via driver detection (no RDMA).""" collector.system_info.os_family = OSFamily.LINUX - rdma_link = [{"netdev": "eth0", "ifname": "bnxt0"}] ethtool_s_bnxt = "NIC statistics:\n tx_pfc_frames: 0\n rx_pause_frames: 0\n" def run_sut_cmd_side_effect(cmd, **kwargs): @@ -668,8 +712,8 @@ def run_sut_cmd_side_effect(cmd, **kwargs): return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) elif "neighbor show" in cmd: return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) - elif "rdma link -j" in cmd: - return MagicMock(exit_code=0, stdout=json.dumps(rdma_link), command=cmd) + elif "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock(exit_code=0, stdout="driver: bnxt_en\nversion: 1.0\n", command=cmd) elif "ethtool -S" in cmd and "eth0" in cmd: return MagicMock(exit_code=0, stdout=ethtool_s_bnxt, command=cmd) elif "ethtool" in cmd: @@ -684,8 +728,46 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert result.status == ExecutionStatus.OK assert data is not None - assert "eth0" in data.rdma_ethtool_netdevs - assert len(data.rdma_ethtool_statistics) == 1 - assert data.rdma_ethtool_statistics[0].netdev == "eth0" - assert data.rdma_ethtool_statistics[0].rdma_ifname == "bnxt0" - assert data.rdma_ethtool_statistics[0].vendor_statistics is not None + # lo has no vendor driver and is skipped; eth0 (bnxt_en) is collected + assert data.ethtool_netdevs == ["eth0"] + assert len(data.ethtool_statistics) == 1 + assert data.ethtool_statistics[0].netdev == "eth0" + assert data.ethtool_statistics[0].driver == "bnxt_en" + assert data.ethtool_statistics[0].vendor_statistics is not None + + +def test_collect_data_skips_non_vendor_netdev(collector, conn_mock): + """ethtool -S is not run for netdevs whose driver is not a known vendor.""" + collector.system_info.os_family = OSFamily.LINUX + + ethtool_s_called = {"value": False} + + def run_sut_cmd_side_effect(cmd, **kwargs): + if "addr show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ADDR_OUTPUT, command=cmd) + elif "route show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ROUTE_OUTPUT, command=cmd) + elif "rule show" in cmd: + return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) + elif "neighbor show" in cmd: + return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) + elif "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock(exit_code=0, stdout="driver: e1000e\n", command=cmd) + elif "ethtool -S" in cmd: + ethtool_s_called["value"] = True + return MagicMock(exit_code=0, stdout="NIC statistics:\n", command=cmd) + elif "ethtool" in cmd: + return MagicMock(exit_code=1, stdout="", command=cmd) + elif "lldpcli" in cmd or "lldpctl" in cmd: + return MagicMock(exit_code=1, stdout="", command=cmd) + return MagicMock(exit_code=1, stdout="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.ethtool_netdevs == [] + assert data.ethtool_statistics == [] + assert ethtool_s_called["value"] is False