feat: Add Consensus Peer Height Metrics (#5517)

Eric-Warehime · web-flow · commit 787f4a11a16c · 2025-12-05T16:08:48.000Z
--- <img width="1132" height="345" alt="Screenshot 2025-12-04 at 3 48 42 PM" src="https://github.com/user-attachments/assets/f6a93751-389f-4911-8c14-a8b389268545" /> #### PR checklist - [ ] Tests written/updated - [ ] Changelog entry added in `.changelog` (we use [unclog](https://github.com/informalsystems/unclog) to manage our changelog) - [ ] Updated relevant documentation (`docs/` or `spec/`) and code comments  --- > [!NOTE] > Introduce a per-peer consensus height metric and emit updates on NewRoundStep via the reactor. > > - **Consensus**: > - **Metrics**: Add `PeerHeight` gauge (`metrics.go`, `metrics.gen.go`) labeled by `peer_id` to track each peer's current consensus height. > - **Reactor**: On `NewRoundStepMessage`, enqueue to `statsMsgQueue` and update `Metrics.PeerHeight` per peer in `peerStatsRoutine`; minor refactor in switch variable naming. > - **Changelog**: Add improvement entry for publishing peer height metric; minor formatting fixes. > > <sup>Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit f1d7bd9. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot).</sup>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,7 +11,7 @@
 
 ### IMPROVEMENTS
 
--  `[mempool]` perf(mempool/cache): Optimize LRUTxCache.Remove to reduce lock contention and map access
+- `[mempool]` perf(mempool/cache): Optimize LRUTxCache.Remove to reduce lock contention and map access
    ([\#5244](https://github.com/cometbft/cometbft/pull/5244))
 - `[e2e]` add support for testing different keytypes, including BLS
   ([\#3513](https://github.com/cometbft/cometbft/pull/3513))
@@ -63,7 +63,7 @@
   ([\#2988](https://github.com/cometbft/cometbft/issues/2988))
 - `[types]` Significantly speedup types.MakePartSet and types.AddPart, which are used in creating a block proposal
   ([\#3117](https://github.com/cometbft/cometbft/issues/3117))
-- `[types] Make a new method `GetByAddressMut` for `ValSet`, which does not copy the returned validator.
+- `[types] Make a new method`GetByAddressMut` for `ValSet`, which does not copy the returned validator.
   ([\#3119](https://github.com/cometbft/cometbft/issues/3119))
 - `[consensus]` Make Vote messages only take one peerstate mutex
   ([\#3156](https://github.com/cometbft/cometbft/issues/3156))
@@ -82,6 +82,8 @@
 - `[consensus]` Make mempool updates asynchronous from consensus Commit's,
   reducing latency for reaching consensus timeouts.
   ([#3008](https://github.com/cometbft/cometbft/pull/3008))
+- [consensus] Add peer height metric publication to the consensus reactor's peer state.
+  ([#5517](https://github.com/cometbft/cometbft/pull/5517))
 
 ### BUG-FIXES
 
@@ -148,6 +150,7 @@ encouraged to upgrade as soon as possible.
 *December 20 2024*
 
 This release:
+
 - fixes a bug that caused a node produce errors caused by the sending of next PEX requests too soon.
 As a consequence of this incorrect behavior a node would be marked as BAD.
 - Adds a proper description of `ExtendedVoteInfo` and `VoteInfo` in the spec.
@@ -604,7 +607,7 @@ gossip.
 This release includes the second part of ABCI++, called ABCI 2.0.
 ABCI 2.0 introduces ABCI methods `ExtendVote` and `VerifyVoteExtension`.
 These new methods allow the application to add data (opaque to CometBFT),
-called _vote extensions_ to precommit votes sent by validators.
+called *vote extensions* to precommit votes sent by validators.
 These vote extensions are made available to the proposer(s) of the next height.
 Additionally, ABCI 2.0 coalesces `BeginBlock`, `DeliverTx`, and `EndBlock`
 into one method, `FinalizeBlock`, whose `Request*` and `Response*`
@@ -688,7 +691,7 @@ for people who forked CometBFT and interact directly with the indexers kvstore.
 - `[light]` Fixed an edge case where a light client would panic when attempting
   to query a node that (1) has started from a non-zero height and (2) does
   not yet have any data. The light client will now, correctly, not panic
-  _and_ keep the node in its list of providers in the same way it would if
+  *and* keep the node in its list of providers in the same way it would if
   it queried a node starting from height zero that does not yet have data
   ([\#575](https://github.com/cometbft/cometbft/issues/575))
 - `[abci]` Restore the snake_case naming in JSON serialization of
@@ -853,7 +856,7 @@ See below for more details.
   [\#77](https://github.com/cometbft/cometbft/pull/77). @jmalicevic
   ([\#382](https://github.com/cometbft/cometbft/pull/382))
 - `[consensus]` ([\#386](https://github.com/cometbft/cometbft/pull/386)) Short-term fix for the case when `needProofBlock` cannot find previous block meta by defaulting to the creation of a new proof block. (@adizere)
-  - Special thanks to the [Vega.xyz](https://vega.xyz/) team, and in particular to Zohar (@ze97286), for reporting the problem and working with us to get to a fix.
+    - Special thanks to the [Vega.xyz](https://vega.xyz/) team, and in particular to Zohar (@ze97286), for reporting the problem and working with us to get to a fix.
 - `[docker]` enable cross platform build using docker buildx
   ([\#9073](https://github.com/tendermint/tendermint/pull/9073))
 - `[consensus]` fix round number of `enterPropose`
@@ -942,7 +945,7 @@ to this release!
 - `[consensus]` Short-term fix for the case when `needProofBlock` cannot find
   previous block meta by defaulting to the creation of a new proof block.
   ([\#386](https://github.com/cometbft/cometbft/pull/386): @adizere)
-  - Special thanks to the [Vega.xyz](https://vega.xyz/) team, and in particular
+    - Special thanks to the [Vega.xyz](https://vega.xyz/) team, and in particular
     to Zohar (@ze97286), for reporting the problem and working with us to get to
     a fix.
 - `[p2p]` Correctly use non-blocking `TrySendEnvelope` method when attempting to
@@ -965,7 +968,7 @@ to this release!
 ### FEATURES
 
 - `[rpc]` Add `match_event` query parameter to indicate to the RPC that it
-  should match events _within_ attributes, not only within a height
+  should match events *within* attributes, not only within a height
   ([tendermint/tendermint\#9759](https://github.com/tendermint/tendermint/pull/9759))
 
 ### IMPROVEMENTS
diff --git a/consensus/metrics.gen.go b/consensus/metrics.gen.go
diff --git a/consensus/metrics.go b/consensus/metrics.go
@@ -143,6 +143,11 @@ type Metrics struct {
 	// correspond to earlier heights and rounds than this node is currently
 	// in.
 	LateVotes metrics.Counter `metrics_labels:"vote_type"`
+
+	// PeerHeight is the consensus reactor's view of what height their peers are currently on.
+	// It is reported with a separate tag for every peer we are connected to, and updated when their height updates
+	// in our consensus state.
+	PeerHeight metrics.Gauge `metrics_labels:"peer_id"`
 }
 
 func (m *Metrics) MarkProposalProcessed(accepted bool) {
diff --git a/consensus/reactor.go b/consensus/reactor.go
@@ -276,6 +276,7 @@ func (conR *Reactor) Receive(e p2p.Envelope) {
 				return
 			}
 			ps.ApplyNewRoundStepMessage(msg)
+			conR.conS.statsMsgQueue <- msgInfo{msg, e.Src.ID()}
 		case *NewValidBlockMessage:
 			ps.ApplyNewValidBlockMessage(msg)
 		case *HasVoteMessage:
@@ -984,7 +985,7 @@ func (conR *Reactor) peerStatsRoutine() {
 			if !ok {
 				panic(fmt.Sprintf("Peer %v has no state", peer))
 			}
-			switch msg.Msg.(type) {
+			switch concreteMsg := msg.Msg.(type) {
 			case *VoteMessage:
 				if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 {
 					conR.Switch.MarkPeerAsGood(peer)
@@ -993,6 +994,8 @@ func (conR *Reactor) peerStatsRoutine() {
 				if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 {
 					conR.Switch.MarkPeerAsGood(peer)
 				}
+			case *NewRoundStepMessage:
+				conR.Metrics.PeerHeight.With("peer_id", string(msg.PeerID)).Set(float64(concreteMsg.Height))
 			}
 		case <-conR.conS.Quit():
 			return
@@ -1669,7 +1672,6 @@ func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error {
 			Field:  "Height",
 			Reason: fmt.Sprintf("%v should be lower than initial height %v", m.Height, initialHeight),
 		}
-
 	}
 
 	if m.Height == initialHeight && m.LastCommitRound != -1 {

Original file line number	Diff line number	Diff line change
`@@ -276,6 +276,7 @@ func (conR *Reactor) Receive(e p2p.Envelope) {`
`276`	`276`	`return`
`277`	`277`	`}`
`278`	`278`	`ps.ApplyNewRoundStepMessage(msg)`
	`279`	`+ conR.conS.statsMsgQueue <- msgInfo{msg, e.Src.ID()}`
`279`	`280`	`case *NewValidBlockMessage:`
`280`	`281`	`ps.ApplyNewValidBlockMessage(msg)`
`281`	`282`	`case *HasVoteMessage:`
`@@ -984,7 +985,7 @@ func (conR *Reactor) peerStatsRoutine() {`
`984`	`985`	`if !ok {`
`985`	`986`	`panic(fmt.Sprintf("Peer %v has no state", peer))`
`986`	`987`	`}`
`987`		`- switch msg.Msg.(type) {`
	`988`	`+ switch concreteMsg := msg.Msg.(type) {`
`988`	`989`	`case *VoteMessage:`
`989`	`990`	`if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 {`
`990`	`991`	`conR.Switch.MarkPeerAsGood(peer)`
`@@ -993,6 +994,8 @@ func (conR *Reactor) peerStatsRoutine() {`
`993`	`994`	`if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 {`
`994`	`995`	`conR.Switch.MarkPeerAsGood(peer)`
`995`	`996`	`}`
	`997`	`+ case *NewRoundStepMessage:`
	`998`	`+ conR.Metrics.PeerHeight.With("peer_id", string(msg.PeerID)).Set(float64(concreteMsg.Height))`
`996`	`999`	`}`
`997`	`1000`	`case <-conR.conS.Quit():`
`998`	`1001`	`return`
`@@ -1669,7 +1672,6 @@ func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error {`
`1669`	`1672`	`Field: "Height",`
`1670`	`1673`	`Reason: fmt.Sprintf("%v should be lower than initial height %v", m.Height, initialHeight),`
`1671`	`1674`	`}`
`1672`		`-`
`1673`	`1675`	`}`
`1674`	`1676`
`1675`	`1677`	`if m.Height == initialHeight && m.LastCommitRound != -1 {`