Skip to content

Ethtool Queue Enhancement #260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sunnyhe2 wants to merge 2 commits into
base: development
Choose a base branch
from
Open

Ethtool Queue Enhancement #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
94e8691
added queue fields
sunnyhe2 Jul 13, 2026
07cf9d3
Merge branch 'development' into sunny_rdmaethool_enhance
alexandraBara Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 88 additions & 5 deletions nodescraper/plugins/inband/network/ethtool_vendor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,26 @@
###############################################################################
"""Vendor-specific ethtool -S statistics models (Pollara / Thor2 / ConnectX-7)."""

import re
from typing import ClassVar, Optional, Union

from pydantic import BaseModel, Field, model_validator
from typing_extensions import Self


class PollaraEthtoolStatistics(BaseModel):
"""ifname ionic. Keeping only fields of interest. Skip queue-specific stats for now"""
"""ionic (Pollara) ethtool -S counters."""

#: Regex matched against raw ``ethtool -S`` field names to identify per-queue
queue_counter_regex: ClassVar[str] = r"^(rx|tx)_\d+"
queue_error_regex: ClassVar[list[str]] = [
r"dma_map_err",
r"hwstamp_invalid",
r"alloc_err",
r"csum_error",
r"dropped",
]
queue_warning_regex: ClassVar[list[str]] = []

rx_csum_error: Optional[int] = None
hw_tx_dropped: Optional[int] = None
Expand Down Expand Up @@ -157,7 +169,27 @@ class PollaraEthtoolStatistics(BaseModel):


class Thor2EthtoolStatistics(BaseModel):
"""ifname bnxt. Keeping only fields of interest. Skip queue-specific stats for now"""
"""bnxt (Thor2) ethtool -S counters."""

#: Regex matched against raw ``ethtool -S`` field names to identify per-queue
queue_counter_regex: ClassVar[str] = r"^\[\d+\]"
queue_error_regex: ClassVar[list[str]] = [
r"rx_discards",
r"rx_errors",
r"tx_errors",
r"tx_discards",
r"rx_l4_csum_errors",
r"rx_buf_errors",
r"so_txtime_cmpl_errors",
r"missed_irqs",
r"xsk_rx_redirect_fail",
r"xsk_rx_alloc_fail",
r"xsk_rx_no_room",
r"xsk_tx_ring_full",
]
queue_warning_regex: ClassVar[list[str]] = [
r"rx_resets",
]

rx_total_l4_csum_errors: Optional[int] = None
rx_total_resets: Optional[int] = None
Expand Down Expand Up @@ -356,7 +388,22 @@ class Thor2EthtoolStatistics(BaseModel):


class Cx7EthtoolStatistics(BaseModel):
"""ifname mlx. Keeping only fields of interest. Skip queue-specific stats for now"""
"""mlx (ConnectX-7) ethtool -S counters."""

#: Regex matched against raw ``ethtool -S`` field names to identify per-queue
queue_counter_regex: ClassVar[str] = r"^(rx|tx|ch)\d+"
queue_error_regex: ClassVar[list[str]] = [
r"xdp_drop",
r"wqe_err",
r"oversize_pkts_sw_drop",
r"buff_alloc_err",
r"arfs_err",
r"tls_err",
r"xdp_tx_err",
r"dropped",
r"cqe_err",
]
queue_warning_regex: ClassVar[list[str]] = []

rx_xdp_drop: Optional[int] = None
rx_xdp_tx_err: Optional[int] = None
Expand Down Expand Up @@ -645,15 +692,51 @@ class Cx7EthtoolStatistics(BaseModel):
}


def extract_queue_counters(stats: dict[str, str], queue_counter_regex: str) -> dict[str, int]:
"""Extract per-queue counters from a raw ``ethtool -S`` stats dict.

Devices can report hundreds of per-queue counters. These are matched by the
vendor's ``queue_counter_regex`` and retained (rather than dropped like other
non-enumerated fields) so they remain available in the data model for error
detection.

Args:
stats: Mapping of raw ``ethtool -S`` field name -> string value.
queue_counter_regex: Vendor regex matched against field names to identify
per-queue counters.

Returns:
Mapping of matching field name -> integer value. The ``netdev`` marker key
and non-integer values are skipped.
"""
pattern = re.compile(queue_counter_regex)
queue_counters: dict[str, int] = {}
for name, value in stats.items():
if name == "netdev" or not pattern.match(name):
continue
try:
queue_counters[name] = int(value)
except (TypeError, ValueError):
continue
return queue_counters


class EthtoolStatistics(BaseModel):
"""Per-netdev ethtool -S row with optional vendor-parsed counters."""

netdev: Optional[str] = None
rdma_ifname: Optional[str] = Field(
driver: Optional[str] = Field(
default=None,
description="RDMA interface name from 'rdma link -j' used for vendor prefix selection",
description="Kernel driver (from 'ethtool -i') used for vendor model selection",
)
vendor_statistics: Optional[VendorEthtoolStatisticsModel] = None
queue_statistics: dict[str, int] = Field(
default_factory=dict,
description=(
"High-cardinality per-queue ethtool -S counters (raw field name -> value) "
"captured via the vendor queue_counter_regex for error detection"
),
)

@model_validator(mode="after")
def validate_atleast_one_field(self) -> Self:
Expand Down
81 changes: 67 additions & 14 deletions nodescraper/plugins/inband/network/network_analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
# SOFTWARE.
#
###############################################################################
import re
from typing import Optional

from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer
Expand All @@ -49,13 +50,13 @@ def analyze_data(
"""Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex.

Args:
data: Network data model with ethtool_info and/or rdma_ethtool_statistics.
data: Network data model with ethtool_info and/or ethtool_statistics.
args: Optional analyzer arguments with custom error regex support.

Returns:
TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR.
"""
if not data.ethtool_info and not data.rdma_ethtool_statistics:
if not data.ethtool_info and not data.ethtool_statistics:
self.result.message = "No network devices found"
self.result.status = ExecutionStatus.WARNING
return self.result
Expand Down Expand Up @@ -110,7 +111,9 @@ def analyze_data(
vendor_warning = False
vendor_error_fields: set[str] = set()
vendor_warning_fields: set[str] = set()
for stat in data.rdma_ethtool_statistics:
vendor_queue_error_fields: set[str] = set()
vendor_queue_warning_fields: set[str] = set()
for stat in data.ethtool_statistics:
if stat.vendor_statistics is None:
continue

Expand All @@ -129,37 +132,87 @@ def analyze_data(
else:
vendor_error = True
vendor_error_fields.add(field_name)
# Use a single grouped description per severity so the run summary
# collapses every occurrence into one "Ethtool warning detected" /
# "Ethtool error detected" entry instead of one line per field. The
# specific field is still preserved in the event data below and in
# the consolidated console line emitted after the loop.
# Use a single grouped description per severity
desc = (
"Ethtool warning detected" if is_warning_tier else "Ethtool error detected"
)
# Per-field events are still recorded for the run summary and event
# log, but console logging is suppressed here to avoid repeating the
# same message once per device. A single consolidated line is emitted
# after the loop instead.
# log, but console logging is suppressed here
self._log_event(
category=EventCategory.NETWORK,
description=desc,
data={
"netdev": stat.netdev,
"rdma_ifname": stat.rdma_ifname,
"driver": stat.driver,
"error_field": field_name,
"error_count": error_value,
},
priority=priority,
console_log=False,
)

if vendor_error:
# Per-queue counters matched against the vendor's adjustable regex
queue_error_patterns = [
re.compile(pattern) for pattern in getattr(type(vs), "queue_error_regex", [])
]
queue_warning_patterns = [
re.compile(pattern) for pattern in getattr(type(vs), "queue_warning_regex", [])
]
if stat.queue_statistics and (queue_error_patterns or queue_warning_patterns):
for counter_name, counter_value in stat.queue_statistics.items():
if counter_value <= 0:
continue
if queue_error_patterns and any(
pattern.search(counter_name) for pattern in queue_error_patterns
):
vendor_error = True
vendor_queue_error_fields.add(f"{stat.netdev} {counter_name}")
self._log_event(
category=EventCategory.NETWORK,
description="Ethtool queue error detected",
data={
"netdev": stat.netdev,
"driver": stat.driver,
"error_field": counter_name,
"error_count": counter_value,
},
priority=EventPriority.ERROR,
console_log=False,
)
elif queue_warning_patterns and any(
pattern.search(counter_name) for pattern in queue_warning_patterns
):
vendor_warning = True
vendor_queue_warning_fields.add(f"{stat.netdev} {counter_name}")
self._log_event(
category=EventCategory.NETWORK,
description="Ethtool queue warning detected",
data={
"netdev": stat.netdev,
"driver": stat.driver,
"error_field": counter_name,
"error_count": counter_value,
},
priority=EventPriority.WARNING,
console_log=False,
)

if vendor_error_fields:
self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields)))
if vendor_warning:
if vendor_queue_error_fields:
self.logger.error(
"Ethtool queue error detected: %s",
", ".join(sorted(vendor_queue_error_fields)),
)
if vendor_warning_fields:
self.logger.warning(
"Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields))
)
if vendor_queue_warning_fields:
self.logger.warning(
"Ethtool queue warning detected: %s",
", ".join(sorted(vendor_queue_warning_fields)),
)

if regex_error or vendor_error:
self.result.message = "Network errors detected in statistics"
Expand Down
Loading
Loading