gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-02 07:03:18 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

mgr/dashboard: fix stale metrics in hardware dashboards

Browse Source 
Switch all stat panels in the hardware and hardware compression
dashboards from lastNotNull to last reduceFunction. When a host is
unreachable or node-proxy stops, metric silence is ambiguous — it
could mean host down, agent crash, or scrape failure. lastNotNull
freezes the last known value (e.g. 4 NVMe drives), causing
inconsistency with the global Overview which reflects live count
reductions.

Use last so panels show N/A when no current data arrives.

Fixes: https://tracker.ceph.com/issues/78360

Signed-off-by: Afreen Misbah <afreen@ibm.com>
This commit is contained in:
Afreen Misbah 2026-07-17 21:21:40 +05:30
parent af595ea7fb
commit 8321ca25d7
3 changed files with 121 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
monitoring/ceph-mixin/dashboards/hardware.libsonnet
View File

@ -78,7 +78,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Health',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 4, x: 0, y: 1 }
gridPosition={ h: 4, w: 4, x: 0, y: 1 },
reducerFunction='last',
noValue='N/A',
)
.addTarget($.addTargetSchema(
'max(ceph_hardware_health)',
@ -92,6 +94,7 @@ local g = import 'grafonnet/grafana.libsonnet';
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 999, result: { color: 'semi-dark-red', index: 1, text: 'NOT OK' } } },
],
noValue: 'N/A',
thresholds: {
mode: 'absolute',
steps: [{ color: 'green' }, { color: 'red', value: 1 }],
@ -219,7 +222,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Hosts',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 2, x: 0, y: 5 }
gridPosition={ h: 4, w: 2, x: 0, y: 5 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'count(count by(hostname) (ceph_hardware_health))',
legendFormat='Hosts'
@ -231,7 +236,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Drives',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 2, x: 2, y: 5 }
gridPosition={ h: 4, w: 2, x: 2, y: 5 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'count(ceph_hardware_storage_capacity_bytes)',
legendFormat='Drives'
@ -266,7 +273,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Power',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 2 }
gridPosition={ h: 3, w: 2, x: 0, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="power"})',
legendFormat='Power'
@ -279,6 +288,7 @@ local g = import 'grafonnet/grafana.libsonnet';
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
noValue: 'N/A',
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
@ -290,7 +300,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='MB Temperature (max)',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 2, y: 2 }
gridPosition={ h: 4, w: 3, x: 2, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})',
legendFormat='Motherboard'
@ -306,7 +318,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='CPU Temperature',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 5, y: 2 }
gridPosition={ h: 4, w: 3, x: 5, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})',
legendFormat='CPU'
@ -322,7 +336,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='DIMM Temperature (max)',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 8, y: 2 }
gridPosition={ h: 4, w: 3, x: 8, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})',
legendFormat='DIMM'
@ -338,7 +354,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='PSU Fans',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 11, y: 2 }
gridPosition={ h: 4, w: 3, x: 11, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'count(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
legendFormat='PSU Fans'
@ -353,7 +371,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='AVG PSU Temperature',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 14, y: 2 }
gridPosition={ h: 4, w: 3, x: 14, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})',
legendFormat='PSU'
@ -369,7 +389,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='NVMe Drives',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 17, y: 2 }
gridPosition={ h: 4, w: 3, x: 17, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'count(ceph_hardware_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})',
legendFormat='NVMe'
@ -385,7 +407,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='NVMe Temperature (max)',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 20, y: 2 }
gridPosition={ h: 4, w: 3, x: 20, y: 2 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})',
legendFormat='NVMe'
@ -401,7 +425,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Network',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 5 }
gridPosition={ h: 3, w: 2, x: 0, y: 5 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="network"})',
legendFormat='Network'
@ -414,6 +440,7 @@ local g = import 'grafonnet/grafana.libsonnet';
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
noValue: 'N/A',
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
@ -425,7 +452,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Cooling',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 8 }
gridPosition={ h: 3, w: 2, x: 0, y: 8 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="fans"})',
legendFormat='Cooling'
@ -438,6 +467,7 @@ local g = import 'grafonnet/grafana.libsonnet';
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
noValue: 'N/A',
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
@ -449,7 +479,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='Drives',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 11 }
gridPosition={ h: 3, w: 2, x: 0, y: 11 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="storage"})',
legendFormat='Drives'
@ -462,6 +494,7 @@ local g = import 'grafonnet/grafana.libsonnet';
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
noValue: 'N/A',
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
@ -559,7 +592,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='FAN: $fan_speeds (RPM)',
unit='locale',
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 0, y: 3 }
gridPosition={ h: 5, w: 4, x: 0, y: 3 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}',
legendFormat='{{fan_name}}'
@ -577,7 +612,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='PSU1 Fan Speed (RPM)',
unit='locale',
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 0, y: 8 }
gridPosition={ h: 5, w: 4, x: 0, y: 8 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}',
legendFormat='{{fan_name}}'
@ -592,7 +629,9 @@ local g = import 'grafonnet/grafana.libsonnet';
title='PSU2 Fan Speed (RPM)',
unit='locale',
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 4, y: 8 }
gridPosition={ h: 5, w: 4, x: 4, y: 8 },
reducerFunction='last',
noValue='N/A',
).addTarget($.addTargetSchema(
'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}',
legendFormat='{{fan_name}}'

50
monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json
View File

@ -109,7 +109,8 @@
}
]
},
"unit": "bytes"
"unit": "bytes",
"noValue": "N/A"
},
"overrides": []
},
@ -128,7 +129,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -193,7 +194,8 @@
}
]
},
"unit": "none"
"unit": "none",
"noValue": "N/A"
},
"overrides": []
},
@ -212,7 +214,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -269,7 +271,8 @@
}
]
},
"unit": "percent"
"unit": "percent",
"noValue": "N/A"
},
"overrides": []
},
@ -288,7 +291,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -351,7 +354,8 @@
}
]
},
"unit": "percent"
"unit": "percent",
"noValue": "N/A"
},
"overrides": []
},
@ -370,7 +374,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -422,7 +426,8 @@
"mode": "palette-classic"
},
"mappings": [],
"unit": "bytes"
"unit": "bytes",
"noValue": "N/A"
},
"overrides": [
{
@ -486,7 +491,7 @@
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -549,7 +554,8 @@
}
]
},
"unit": "bytes"
"unit": "bytes",
"noValue": "N/A"
},
"overrides": []
},
@ -568,7 +574,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -616,7 +622,8 @@
}
]
},
"unit": "bytes"
"unit": "bytes",
"noValue": "N/A"
},
"overrides": []
},
@ -635,7 +642,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -683,7 +690,8 @@
}
]
},
"unit": "bytes"
"unit": "bytes",
"noValue": "N/A"
},
"overrides": []
},
@ -702,7 +710,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -750,7 +758,8 @@
}
]
},
"unit": "short"
"unit": "short",
"noValue": "N/A"
},
"overrides": []
},
@ -769,7 +778,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -829,7 +838,8 @@
}
]
},
"unit": "bytes"
"unit": "bytes",
"noValue": "N/A"
},
"overrides": []
},
@ -848,7 +858,7 @@
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false

51
monitoring/ceph-mixin/dashboards_out/hardware.json
View File

@ -78,6 +78,7 @@
"type": "range"
}
],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -107,7 +108,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -415,6 +416,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [ ]
@ -437,7 +439,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -467,6 +469,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [ ]
@ -489,7 +492,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -623,6 +626,7 @@
"type": "range"
}
],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -652,7 +656,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -682,6 +686,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -716,7 +721,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -748,6 +753,7 @@
"mappings": [ ],
"max": 85,
"min": 0,
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -782,7 +788,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -814,6 +820,7 @@
"mappings": [ ],
"max": 88,
"min": 0,
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -848,7 +855,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -878,6 +885,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [ ]
@ -900,7 +908,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -936,6 +944,7 @@
"mappings": [ ],
"max": 100,
"min": 0,
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -970,7 +979,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1004,6 +1013,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -1038,7 +1048,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1074,6 +1084,7 @@
"mappings": [ ],
"max": 85,
"min": 0,
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -1108,7 +1119,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1159,6 +1170,7 @@
"type": "range"
}
],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -1188,7 +1200,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1239,6 +1251,7 @@
"type": "range"
}
],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -1268,7 +1281,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1319,6 +1332,7 @@
"type": "range"
}
],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [
@ -1348,7 +1362,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1555,6 +1569,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [ ]
@ -1578,7 +1593,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1614,6 +1629,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [ ]
@ -1636,7 +1652,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false
@ -1670,6 +1686,7 @@
"decimals": 0,
"links": [ ],
"mappings": [ ],
"noValue": "N/A",
"thresholds": {
"mode": "absolute",
"steps": [ ]
@ -1692,7 +1709,7 @@
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"last"
],
"fields": "",
"values": false