From b2d1f58ef76a58a98d675292a6f1ddf18316e32f Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Wed, 15 Jul 2026 16:32:55 -0400 Subject: [PATCH 01/20] fix: send replication requests up to maxRoundLimit instead of dropping --- simplex/epoch.go | 28 +++++++++++------ simplex/replication_request_test.go | 49 +++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/simplex/epoch.go b/simplex/epoch.go index 0483976c..da40aa72 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -3071,15 +3071,27 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co } response := &common.VerifiedReplicationResponse{} - if len(req.Seqs) > int(e.MaxRoundWindow) || len(req.Rounds) > int(e.MaxRoundWindow) { - e.Logger.Info("Replication request exceeds maximum allowed seqs and rounds", - zap.Stringer("from", from), - zap.Int("num seqs", len(req.Seqs)), - zap.Int("num rounds", len(req.Rounds)), + seqs := req.Seqs + slices.Sort(seqs) + if len(seqs) > int(e.MaxRoundWindow) { + e.Logger.Debug("Truncating replication request seqs", + zap.Stringer("from", from), + zap.Int("nums seqs", len(req.Seqs)), zap.Uint64("max round window", e.MaxRoundWindow)) - return nil + seqs = seqs[:e.MaxRoundWindow] + } + rounds := req.Rounds + slices.Sort(rounds) + if len(rounds) > int(e.MaxRoundWindow) { + e.Logger.Debug("Truncating replication request rounds", + zap.Stringer("from", from), + zap.Int("num rounds", len(req.Rounds)), + zap.Uint64("max round window", e.MaxRoundWindow)) + rounds = rounds[:e.MaxRoundWindow] } + + if req.LatestRound > 0 { latestRound := e.getLatestVerifiedQuorumRound() if latestRound != nil && latestRound.GetRound() > req.LatestRound { @@ -3095,8 +3107,6 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co } } - seqs := req.Seqs - slices.Sort(seqs) seqData := make([]common.VerifiedQuorumRound, len(seqs)) for i, seq := range seqs { quorumRound := e.locateQuorumRecord(seq) @@ -3109,9 +3119,7 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co seqData[i] = *quorumRound } - rounds := req.Rounds roundData := make([]common.VerifiedQuorumRound, 0, len(rounds)) - slices.Sort(rounds) for _, roundNum := range rounds { quorumRound := e.locateQuorumRecordByRound(roundNum) if quorumRound == nil { diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index f1f34d82..626c15f5 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -410,3 +410,52 @@ func TestMalformedReplicationResponse(t *testing.T) { }, nodes[1]) require.NoError(t, err) } + + +// TestReplicationRequestTruncated ensures a request with more seqs or rounds +// than MaxRoundWindow is answered with the lowest MaxRoundWindow entries +// rather than being dropped. +func TestReplicationRequestTruncated(t *testing.T) { + bb := testutil.NewTestBlockBuilder() + nodes := []common.NodeID{{1}, {2}, {3}, {4}} + comm := NewListenerComm(nodes) + ctx := context.Background() + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + conf.ReplicationEnabled = true + + numBlocks := 2 * conf.MaxRoundWindow + seqs := createBlocks(t, nodes, numBlocks) + for _, data := range seqs { + require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) + } + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + + // request 2*MaxRoundWindow seqs, deliberately unsorted (highest first), + // to also pin down that truncation happens after sorting + requested := make([]uint64, 0, numBlocks) + for i := numBlocks; i > 0; i-- { + requested = append(requested, i-1) + } + + req := &common.Message{ + ReplicationRequest: &common.ReplicationRequest{ + Seqs: requested, + }, + } + require.NoError(t, e.HandleMessage(req, nodes[1])) + + msg := <-comm.in + resp := msg.VerifiedReplicationResponse + + // we should get exactly the lowest MaxRoundWindow seqs: 0..MaxRoundWindow-1 + require.Len(t, resp.Data, int(conf.MaxRoundWindow)) + for i, data := range resp.Data { + require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Seq) + require.Equal(t, seqs[i].Finalization, *data.Finalization) + require.Equal(t, seqs[i].VerifiedBlock, data.VerifiedBlock) + } +} \ No newline at end of file From bc88bcfb995d8a63d6f4b776fc98a111528dc291 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Wed, 15 Jul 2026 16:39:13 -0400 Subject: [PATCH 02/20] add tests for seqs and rounds --- simplex/epoch.go | 16 ++-- simplex/replication_request_test.go | 142 +++++++++++++++++++--------- 2 files changed, 104 insertions(+), 54 deletions(-) diff --git a/simplex/epoch.go b/simplex/epoch.go index da40aa72..70dbd27d 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -3074,24 +3074,22 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co seqs := req.Seqs slices.Sort(seqs) if len(seqs) > int(e.MaxRoundWindow) { - e.Logger.Debug("Truncating replication request seqs", - zap.Stringer("from", from), - zap.Int("nums seqs", len(req.Seqs)), + e.Logger.Debug("Truncating replication request seqs", + zap.Stringer("from", from), + zap.Int("nums seqs", len(req.Seqs)), zap.Uint64("max round window", e.MaxRoundWindow)) seqs = seqs[:e.MaxRoundWindow] } - rounds := req.Rounds + rounds := req.Rounds slices.Sort(rounds) if len(rounds) > int(e.MaxRoundWindow) { - e.Logger.Debug("Truncating replication request rounds", - zap.Stringer("from", from), - zap.Int("num rounds", len(req.Rounds)), + e.Logger.Debug("Truncating replication request rounds", + zap.Stringer("from", from), + zap.Int("num rounds", len(req.Rounds)), zap.Uint64("max round window", e.MaxRoundWindow)) rounds = rounds[:e.MaxRoundWindow] } - - if req.LatestRound > 0 { latestRound := e.getLatestVerifiedQuorumRound() if latestRound != nil && latestRound.GetRound() > req.LatestRound { diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 626c15f5..b6da63b9 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -411,51 +411,103 @@ func TestMalformedReplicationResponse(t *testing.T) { require.NoError(t, err) } - -// TestReplicationRequestTruncated ensures a request with more seqs or rounds +// TestReplicationRequestTruncated ensures a request with more seqs // than MaxRoundWindow is answered with the lowest MaxRoundWindow entries // rather than being dropped. func TestReplicationRequestTruncated(t *testing.T) { - bb := testutil.NewTestBlockBuilder() - nodes := []common.NodeID{{1}, {2}, {3}, {4}} - comm := NewListenerComm(nodes) - ctx := context.Background() - conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) - conf.ReplicationEnabled = true - - numBlocks := 2 * conf.MaxRoundWindow - seqs := createBlocks(t, nodes, numBlocks) - for _, data := range seqs { - require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) - } - - e, err := simplex.NewEpoch(conf) - require.NoError(t, err) - t.Cleanup(e.Stop) - require.NoError(t, e.Start()) - - // request 2*MaxRoundWindow seqs, deliberately unsorted (highest first), - // to also pin down that truncation happens after sorting - requested := make([]uint64, 0, numBlocks) - for i := numBlocks; i > 0; i-- { - requested = append(requested, i-1) - } - - req := &common.Message{ - ReplicationRequest: &common.ReplicationRequest{ - Seqs: requested, - }, - } - require.NoError(t, e.HandleMessage(req, nodes[1])) - - msg := <-comm.in - resp := msg.VerifiedReplicationResponse - - // we should get exactly the lowest MaxRoundWindow seqs: 0..MaxRoundWindow-1 - require.Len(t, resp.Data, int(conf.MaxRoundWindow)) - for i, data := range resp.Data { - require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Seq) - require.Equal(t, seqs[i].Finalization, *data.Finalization) - require.Equal(t, seqs[i].VerifiedBlock, data.VerifiedBlock) - } -} \ No newline at end of file + bb := testutil.NewTestBlockBuilder() + nodes := []common.NodeID{{1}, {2}, {3}, {4}} + comm := NewListenerComm(nodes) + ctx := context.Background() + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + conf.ReplicationEnabled = true + + numBlocks := 2 * conf.MaxRoundWindow + seqs := createBlocks(t, nodes, numBlocks) + for _, data := range seqs { + require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) + } + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + + // request 2*MaxRoundWindow seqs, unsorted (highest first), + // to also pin down that truncation happens after sorting + requested := make([]uint64, 0, numBlocks) + for i := numBlocks; i > 0; i-- { + requested = append(requested, i-1) + } + + req := &common.Message{ + ReplicationRequest: &common.ReplicationRequest{ + Seqs: requested, + }, + } + require.NoError(t, e.HandleMessage(req, nodes[1])) + + msg := <-comm.in + resp := msg.VerifiedReplicationResponse + + // we should get exactly the lowest MaxRoundWindow seqs: 0..MaxRoundWindow-1 + require.Len(t, resp.Data, int(conf.MaxRoundWindow)) + for i, data := range resp.Data { + require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Seq) + require.Equal(t, seqs[i].Finalization, *data.Finalization) + require.Equal(t, seqs[i].VerifiedBlock, data.VerifiedBlock) + } + +} + +// TestReplicationRequestRoundsTruncated ensures a request with more rounds +// than MaxRoundWindow is answered with the lowest MaxRoundWindow rounds +// rather than being dropped. +func TestReplicationRequestRoundsTruncated(t *testing.T) { + bb := testutil.NewTestBlockBuilder() + nodes := []common.NodeID{{1}, {2}, {3}, {4}} + comm := NewListenerComm(nodes) + noop := testutil.NewNoopComm(nodes) + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], noop, bb) + conf.ReplicationEnabled = true + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + + numRounds := 2 * conf.MaxRoundWindow + rounds := make(map[uint64]common.VerifiedQuorumRound, numRounds) + for i := uint64(0); i < numRounds; i++ { + block, notarization := advanceRoundFromNotarization(t, e, bb) + rounds[i] = common.VerifiedQuorumRound{ + VerifiedBlock: block, + Notarization: notarization, + } + } + require.Equal(t, numRounds, e.Metadata().Round) + + // request 2*MaxRoundWindow rounds, unsorted (highest first), + // to also pin down that truncation happens after sorting + requested := make([]uint64, 0, numRounds) + for i := numRounds; i > 0; i-- { + requested = append(requested, i-1) + } + + e.Comm = comm + require.NoError(t, e.HandleMessage(&common.Message{ + ReplicationRequest: &common.ReplicationRequest{ + Rounds: requested, + }, + }, nodes[1])) + + msg := <-comm.in + resp := msg.VerifiedReplicationResponse + + // we should get exactly the lowest MaxRoundWindow rounds: 0..MaxRoundWindow-1 + require.Len(t, resp.Data, int(conf.MaxRoundWindow)) + for i, data := range resp.Data { + require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Round) + require.Equal(t, rounds[uint64(i)].Notarization, data.Notarization) + } +} From 70ee54a0af7d109f4deaea06a1c0b84ae568ea67 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 16 Jul 2026 10:28:00 -0400 Subject: [PATCH 03/20] added unsorted seqs and rounds test --- simplex/epoch.go | 2 +- simplex/replication_request_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/simplex/epoch.go b/simplex/epoch.go index 70dbd27d..ab58f7f4 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -3076,7 +3076,7 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co if len(seqs) > int(e.MaxRoundWindow) { e.Logger.Debug("Truncating replication request seqs", zap.Stringer("from", from), - zap.Int("nums seqs", len(req.Seqs)), + zap.Int("num seqs", len(req.Seqs)), zap.Uint64("max round window", e.MaxRoundWindow)) seqs = seqs[:e.MaxRoundWindow] } diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index b6da63b9..c2561cf6 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -3,6 +3,7 @@ package simplex_test import ( "bytes" "context" + "slices" "testing" "time" @@ -443,6 +444,7 @@ func TestReplicationRequestTruncated(t *testing.T) { req := &common.Message{ ReplicationRequest: &common.ReplicationRequest{ Seqs: requested, + Rounds: slices.Clone(requested), }, } require.NoError(t, e.HandleMessage(req, nodes[1])) From ab8cace2c2652a7cd1c52acee02d7f87a688aa03 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 16 Jul 2026 10:30:51 -0400 Subject: [PATCH 04/20] go format --- simplex/replication_request_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index c2561cf6..576bc77c 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -443,7 +443,7 @@ func TestReplicationRequestTruncated(t *testing.T) { req := &common.Message{ ReplicationRequest: &common.ReplicationRequest{ - Seqs: requested, + Seqs: requested, Rounds: slices.Clone(requested), }, } From db90c4a5c072ee84f787c9bc7c1afee311f67d4a Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 16 Jul 2026 14:01:09 -0400 Subject: [PATCH 05/20] added test for unsorted seqs and rounds above limit --- common/msg.go | 1 + simplex/replication_request_test.go | 70 ++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/common/msg.go b/common/msg.go index 1c287b11..422bc0f9 100644 --- a/common/msg.go +++ b/common/msg.go @@ -367,6 +367,7 @@ func (q *VerifiedQuorumRound) GetRound() uint64 { return 0 } + type VerifiedFinalizedBlock struct { VerifiedBlock VerifiedBlock Finalization Finalization diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 576bc77c..022ed6d4 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -3,7 +3,6 @@ package simplex_test import ( "bytes" "context" - "slices" "testing" "time" @@ -412,19 +411,21 @@ func TestMalformedReplicationResponse(t *testing.T) { require.NoError(t, err) } -// TestReplicationRequestTruncated ensures a request with more seqs -// than MaxRoundWindow is answered with the lowest MaxRoundWindow entries +// TestReplicationRequestSeqsAndRoundsTruncated ensures a request with more seqs and more rounds +// than MaxRoundWindow is answered with both truncated to the lowest MaxRoundWindow entries // rather than being dropped. -func TestReplicationRequestTruncated(t *testing.T) { +func TestReplicationRequestSeqsAndRoundsTruncated(t *testing.T) { bb := testutil.NewTestBlockBuilder() nodes := []common.NodeID{{1}, {2}, {3}, {4}} comm := NewListenerComm(nodes) + noop := testutil.NewNoopComm(nodes) ctx := context.Background() - conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], noop, bb) conf.ReplicationEnabled = true - numBlocks := 2 * conf.MaxRoundWindow - seqs := createBlocks(t, nodes, numBlocks) + // index maxRoundsWindow + 2 blocks: 0...11 + numIndexed := conf.MaxRoundWindow + 2 + seqs := createBlocks(t, nodes, numIndexed) for _, data := range seqs { require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) } @@ -434,30 +435,55 @@ func TestReplicationRequestTruncated(t *testing.T) { t.Cleanup(e.Stop) require.NoError(t, e.Start()) - // request 2*MaxRoundWindow seqs, unsorted (highest first), - // to also pin down that truncation happens after sorting - requested := make([]uint64, 0, numBlocks) - for i := numBlocks; i > 0; i-- { - requested = append(requested, i-1) + // notarize a few rounds past the indexed blocks: rounds 12...16 + // not finalized + numNotarized := uint64(5) + for i := uint64(0); i < numNotarized; i++ { + advanceRoundFromNotarization(t, e, bb) } - req := &common.Message{ + // oversized, unsortedseqs seqs request: 0 ...2*MaxRoundWindow + // truncation keeps seqs 0...9 + requestedSeqs := make([]uint64, 0, 2*conf.MaxRoundWindow) + for i := 2 * conf.MaxRoundWindow; i > 0; i-- { + requestedSeqs = append(requestedSeqs, i-1) + } + + // oversized, unsorted rounds request: 4..24 + // truncation keeps rounds 4...13 + roundStart := numIndexed - 8 // 4 + requestedRounds := make([]uint64, 0, 2*conf.MaxRoundWindow) + for i := roundStart + 2*conf.MaxRoundWindow; i > roundStart; i-- { + requestedRounds = append(requestedRounds, i-1) + } + + e.Comm = comm + require.NoError(t, e.HandleMessage(&common.Message{ ReplicationRequest: &common.ReplicationRequest{ - Seqs: requested, - Rounds: slices.Clone(requested), + Seqs: requestedSeqs, + Rounds: requestedRounds, }, - } - require.NoError(t, e.HandleMessage(req, nodes[1])) + }, nodes[1])) msg := <-comm.in resp := msg.VerifiedReplicationResponse - // we should get exactly the lowest MaxRoundWindow seqs: 0..MaxRoundWindow-1 - require.Len(t, resp.Data, int(conf.MaxRoundWindow)) - for i, data := range resp.Data { - require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Seq) + require.Len(t, resp.Data, int(conf.MaxRoundWindow)+2) + + // firts maxRoundWindow items: 0...9 in order + for i := uint64(0); i < conf.MaxRoundWindow; i++ { + data := resp.Data[i] + require.Equal(t, i, data.VerifiedBlock.BlockHeader().Seq) + require.NotNil(t, data.Finalization) require.Equal(t, seqs[i].Finalization, *data.Finalization) - require.Equal(t, seqs[i].VerifiedBlock, data.VerifiedBlock) + } + + // last two items: notarized rounds 12, 13 -- 14..16 truncated as well + for i, expectedRound := range []uint64{numIndexed, numIndexed + 1} { + data := resp.Data[int(conf.MaxRoundWindow)+i] + require.Equal(t, expectedRound, data.VerifiedBlock.BlockHeader().Round) + require.NotNil(t, data.Notarization) + require.Nil(t, data.Finalization) } } From 8678cde4e6e8f27bd3cc39e381582caa1f1eb262 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 16 Jul 2026 14:02:54 -0400 Subject: [PATCH 06/20] go format --- common/msg.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/msg.go b/common/msg.go index 422bc0f9..1c287b11 100644 --- a/common/msg.go +++ b/common/msg.go @@ -367,7 +367,6 @@ func (q *VerifiedQuorumRound) GetRound() uint64 { return 0 } - type VerifiedFinalizedBlock struct { VerifiedBlock VerifiedBlock Finalization Finalization From 2558b9fd0df003ffd4fcfed1fbc4b6a52adc220d Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 16 Jul 2026 15:53:01 -0400 Subject: [PATCH 07/20] added size limit for replication messages --- common/msg.go | 28 +++++++++ simplex/epoch.go | 54 ++++++++++++++++- simplex/replication_request_test.go | 94 +++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/common/msg.go b/common/msg.go index 1c287b11..81c1b96a 100644 --- a/common/msg.go +++ b/common/msg.go @@ -367,6 +367,34 @@ func (q *VerifiedQuorumRound) GetRound() uint64 { return 0 } +// quoromRoundSizeSlack is added to the estimated size of each VerifiedQuoromRound +// to account for encoding overhead (in avalanchego each item adds protobuf +// field tags and length prefixes on top of the block, QC, and header bytes) +// it is tens of bytes in practivce, 512 is a generous bound +const quoromRoundSizeSlack = 512 + +func (q *VerifiedQuorumRound) EstimateSize() (int, error) { + size := quoromRoundSizeSlack + + if q.VerifiedBlock != nil { + blockBytes, err := q.VerifiedBlock.Bytes() + if err != nil { + return 0, err + } + size += len(blockBytes) + } + if q.Notarization != nil { + size += len(q.Notarization.Vote.Bytes()) + len(q.Notarization.QC.Bytes()) + } + if q.Finalization != nil { + size += len(q.Finalization.Finalization.Bytes()) + len(q.Finalization.QC.Bytes()) + } + if q.EmptyNotarization != nil { + size += len(q.EmptyNotarization.Vote.Bytes()) + len(q.EmptyNotarization.QC.Bytes()) + } + return size, nil +} + type VerifiedFinalizedBlock struct { VerifiedBlock VerifiedBlock Finalization Finalization diff --git a/simplex/epoch.go b/simplex/epoch.go index ab58f7f4..f7e53679 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -31,6 +31,9 @@ const ( DefaultFinalizeVoteRebroadcastTimeout = 6 * time.Second EmptyVoteTimeoutID = "rebroadcast_empty_vote" maxItemCountPerRequest = 10 // max number of rounds or sequences that fit in one replication request + // DefaultMaxReplicationResponseSize is the max size of a replication response. avalanchego rejects messages larger + // than 2 MiB. we cap at 80% (4/5) same as avalanchego (see utils/constants/network.go) + DefaultMaxReplicationResponseSize = 2 * 1024 * 1024 * 4 / 5 ) type EmptyVoteSet struct { @@ -61,6 +64,7 @@ func NewRound(block common.VerifiedBlock) *Round { type EpochConfig struct { MaxProposalWait time.Duration MaxRoundWindow uint64 + MaxReplicationResponseSize int MaxRebroadcastWait time.Duration FinalizeRebroadcastTimeout time.Duration QCDeserializer common.QCDeserializer @@ -265,6 +269,9 @@ func (e *Epoch) maybeAssignDefaultConfig() error { if e.MaxRebroadcastWait == 0 { e.MaxRebroadcastWait = DefaultEmptyVoteRebroadcastTimeout } + if e.MaxReplicationResponseSize == 0 { + e.MaxReplicationResponseSize = DefaultMaxReplicationResponseSize + } if e.RandomSource == nil { source, err := NewRandomSource() if err != nil { @@ -3089,19 +3096,33 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co zap.Uint64("max round window", e.MaxRoundWindow)) rounds = rounds[:e.MaxRoundWindow] } + remainingBytes := e.MaxReplicationResponseSize if req.LatestRound > 0 { latestRound := e.getLatestVerifiedQuorumRound() if latestRound != nil && latestRound.GetRound() > req.LatestRound { - response.LatestRound = latestRound + size, err := latestRound.EstimateSize() + if err != nil { + e.Logger.Error("Failed estimating size of latest round", zap.Error(err)) + } else { + response.LatestRound = latestRound + remainingBytes -= size + } } } if req.LatestFinalizedSeq > 0 { if e.lastBlock != nil && e.lastBlock.Finalization.Finalization.Seq > req.LatestFinalizedSeq { - response.LatestFinalizedSeq = &common.VerifiedQuorumRound{ + latestFinalizedSeq := &common.VerifiedQuorumRound{ VerifiedBlock: e.lastBlock.VerifiedBlock, Finalization: &e.lastBlock.Finalization, } + size, err := latestFinalizedSeq.EstimateSize() + if err != nil { + e.Logger.Error("Failed estimating size of latest finalized seq", zap.Error(err)) + } else { + response.LatestFinalizedSeq = latestFinalizedSeq + remainingBytes -= size + } } } @@ -3113,6 +3134,21 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co seqData = seqData[:i] break } + size, err := quorumRound.EstimateSize() + if err != nil { + e.Logger.Error("Failed estimating size of quorom round", zap.Uint64("seq", seq), zap.Error(err)) + seqData = seqData[:i] + break + } + if size > remainingBytes { + e.Logger.Debug("Replication response reached size limit", + zap.Stringer("from", from), + zap.Uint64("Stopped at Seq", seq), + zap.Int("max size", e.MaxReplicationResponseSize)) + seqData = seqData[:i] + break + } + remainingBytes -= size seqData[i] = *quorumRound } @@ -3124,6 +3160,20 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co // we cannot break early since empty votes may continue } + size, err := quorumRound.EstimateSize() + if err != nil { + e.Logger.Error("Failed estimating size of quorom round", zap.Uint64("round", roundNum), zap.Error(err)) + break + } + if size > remainingBytes { + e.Logger.Debug("Replication response reached size limit", + zap.Stringer("from", from), + zap.Uint64("Stopped at Round", roundNum), + zap.Int("max size", e.MaxReplicationResponseSize)) + break + } + remainingBytes -= size + roundData = append(roundData, *quorumRound) } diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 022ed6d4..538d92b6 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -539,3 +539,97 @@ func TestReplicationRequestRoundsTruncated(t *testing.T) { require.Equal(t, rounds[uint64(i)].Notarization, data.Notarization) } } + +// TestReplicationRequestSizeLimited ensures a replication response is capped +// at MaxReplicationResponseSize estimated bytes, starting wit the lowest seqs first. +// Sequences that do not fit are requested again by the requester +// so a partial response is not lost. +func TestReplicationRequestSizeLimited(t *testing.T) { + bb := testutil.NewTestBlockBuilder() + nodes := []common.NodeID{{1}, {2}, {3}, {4}} + comm := NewListenerComm(nodes) + ctx := context.Background() + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + conf.ReplicationEnabled = true + + numBlocks := uint64(8) // within MaxRoundWindow + seqs := createBlocks(t, nodes, numBlocks) + for _, data := range seqs { + require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) + } + + // measure one quorum round and budget for roughly three of them + oneRound, err := (&common.VerifiedQuorumRound{ + VerifiedBlock: seqs[0].VerifiedBlock, + Finalization: &seqs[0].Finalization, + }).EstimateSize() + require.NoError(t, err) + conf.MaxReplicationResponseSize = 3*oneRound + oneRound/2 + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + + requested := make([]uint64, numBlocks) + for i := range requested { + requested[i] = uint64(i) + } + require.NoError(t, e.HandleMessage(&common.Message{ + ReplicationRequest: &common.ReplicationRequest{ + Seqs: requested, + }, + }, nodes[1])) + + msg := <-comm.in + resp := msg.VerifiedReplicationResponse + + // only the lowest seqs that fit under the budget are sent + require.Len(t, resp.Data, 3) + total := 0 + for i, data := range resp.Data { + require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Seq) + size, err := data.EstimateSize() + require.NoError(t, err) + total += size + } + require.LessOrEqual(t, total, conf.MaxReplicationResponseSize) +} + +// TestReplicationRequestSizeLimitedLatestFinalizedSeq ensures the latest-seq/round are +// sent even when they alone exceed the response size limit. A single item that +// cannot fit in a message could not have been sent through the network in the +// first place. +func TestReplicationRequestSizeLimitedLatestFinalizedSeq(t *testing.T) { + bb := testutil.NewTestBlockBuilder() + nodes := []common.NodeID{{1}, {2}, {3}, {4}} + comm := NewListenerComm(nodes) + ctx := context.Background() + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + conf.ReplicationEnabled = true + conf.MaxReplicationResponseSize = 1 + + numBlocks := uint64(4) + seqs := createBlocks(t, nodes, numBlocks) + for _, data := range seqs { + require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) + } + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + + // ask for data and the hint: the hint must arrive, the data must not + require.NoError(t, e.HandleMessage(&common.Message{ + ReplicationRequest: &common.ReplicationRequest{ + Seqs: []uint64{0, 1, 2}, + LatestFinalizedSeq: 1, + }, + }, nodes[1])) + + msg := <-comm.in + resp := msg.VerifiedReplicationResponse + require.NotNil(t, resp.LatestFinalizedSeq) + require.Empty(t, resp.Data) +} From 5593649181cfb3eeee11f8389542410daa158469 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 16 Jul 2026 16:01:32 -0400 Subject: [PATCH 08/20] go format --- simplex/replication_request_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 538d92b6..8581c6eb 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -620,7 +620,7 @@ func TestReplicationRequestSizeLimitedLatestFinalizedSeq(t *testing.T) { t.Cleanup(e.Stop) require.NoError(t, e.Start()) - // ask for data and the hint: the hint must arrive, the data must not + // ask for data and the latest finalized seq: the latest finalized seq must arrive, the data must not require.NoError(t, e.HandleMessage(&common.Message{ ReplicationRequest: &common.ReplicationRequest{ Seqs: []uint64{0, 1, 2}, From 26caa0face1478dd0ae180260945117c2793962a Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Fri, 17 Jul 2026 16:15:04 -0400 Subject: [PATCH 09/20] add Size function for all sub-messages instead of using Bytes() --- common/api.go | 3 ++ common/metadata.go | 3 ++ common/msg.go | 47 +++++++++++++++++------------ common/msg_test.go | 37 +++++++++++++++++++++++ external.go | 16 ++++++++++ instance_test.go | 22 ++++++++++++++ msm/misc.go | 2 ++ msm/util_test.go | 6 ++++ simplex/epoch.go | 34 ++++++--------------- simplex/replication_request_test.go | 8 ++--- testutil/block.go | 16 ++++++++++ testutil/random_network/block.go | 8 +++++ testutil/util.go | 3 ++ 13 files changed, 157 insertions(+), 48 deletions(-) diff --git a/common/api.go b/common/api.go index c12dcd2b..3bc5b44f 100644 --- a/common/api.go +++ b/common/api.go @@ -119,6 +119,9 @@ type VerifiedBlock interface { // Bytes returns a byte encoding of the block Bytes() ([]byte, error) + //Size returns the number of the bytes encoding of the block + Size() int + // SealingBlockInfo returns a non-nil value for a block that is not a sealing block and that is not the first ever simplex block. SealingBlockInfo() *SealingBlockInfo } diff --git a/common/metadata.go b/common/metadata.go index 5c3f3c5f..06e9f941 100644 --- a/common/metadata.go +++ b/common/metadata.go @@ -77,6 +77,9 @@ func (bh *BlockHeader) Bytes() []byte { return buff } +func (bh *BlockHeader) Size() int { + return BlockHeaderLen +} func (bh *BlockHeader) FromBytes(buff []byte) error { if len(buff) != BlockHeaderLen { diff --git a/common/msg.go b/common/msg.go index 81c1b96a..846bd5d7 100644 --- a/common/msg.go +++ b/common/msg.go @@ -49,15 +49,17 @@ type ToBeSignedEmptyVote struct { EmptyVoteMetadata } +const emptyVoteLen = 1 + 8 + 8 // Version + Epoch + Round + func (v *ToBeSignedEmptyVote) Bytes() []byte { - bytes := make([]byte, 1+8+8) // Version + Epoch + Round + bytes := make([]byte, emptyVoteLen) binary.BigEndian.PutUint64(bytes[1:9], v.EmptyVoteMetadata.Epoch) binary.BigEndian.PutUint64(bytes[9:17], v.EmptyVoteMetadata.Round) return bytes } func (v *ToBeSignedEmptyVote) FromBytes(buff []byte) error { - if len(buff) != 17 { + if len(buff) != emptyVoteLen { return fmt.Errorf("invalid buffer length, expected 17, got %d", len(buff)) } @@ -71,6 +73,10 @@ func (v *ToBeSignedEmptyVote) FromBytes(buff []byte) error { return nil } +func (v *ToBeSignedEmptyVote) Size() int { + return emptyVoteLen +} + func (v *ToBeSignedEmptyVote) Sign(signer Signer) ([]byte, error) { context := "ToBeSignedEmptyVote" msg := v.Bytes() @@ -191,6 +197,10 @@ func (f *Finalization) Verify(nodes Nodes) error { return verifyContextQC(f.QC, f.Finalization.Bytes(), context, nodes) } +func (f *Finalization) Size() int { + return f.Finalization.Size() + f.QC.Size() +} + // Notarization represents a block that has reached a quorum of votes. type Notarization struct { Vote ToBeSignedVote @@ -202,6 +212,10 @@ func (n *Notarization) Verify(nodes Nodes) error { return verifyContextQC(n.QC, n.Vote.Bytes(), context, nodes) } +func (n *Notarization) Size() int { + return n.Vote.Size() + n.QC.Size() +} + type BlockMessage struct { Block Block Vote Vote @@ -221,6 +235,9 @@ func (en *EmptyNotarization) Verify(nodes Nodes) error { context := "ToBeSignedEmptyVote" return verifyContextQC(en.QC, en.Vote.Bytes(), context, nodes) } +func (en *EmptyNotarization) Size() int { + return en.Vote.Size() + en.QC.Size() +} type SignedMessage struct { Payload []byte @@ -236,6 +253,8 @@ type QuorumCertificate interface { Verify(msg []byte, nodes Nodes) error // Bytes returns a raw representation of the given QuorumCertificate. Bytes() []byte + // Size returns the number of bytes + Size() int } type ReplicationRequest struct { @@ -367,32 +386,22 @@ func (q *VerifiedQuorumRound) GetRound() uint64 { return 0 } -// quoromRoundSizeSlack is added to the estimated size of each VerifiedQuoromRound -// to account for encoding overhead (in avalanchego each item adds protobuf -// field tags and length prefixes on top of the block, QC, and header bytes) -// it is tens of bytes in practivce, 512 is a generous bound -const quoromRoundSizeSlack = 512 - -func (q *VerifiedQuorumRound) EstimateSize() (int, error) { - size := quoromRoundSizeSlack +func (q *VerifiedQuorumRound) Size() int { + var size int if q.VerifiedBlock != nil { - blockBytes, err := q.VerifiedBlock.Bytes() - if err != nil { - return 0, err - } - size += len(blockBytes) + size += q.VerifiedBlock.Size() } if q.Notarization != nil { - size += len(q.Notarization.Vote.Bytes()) + len(q.Notarization.QC.Bytes()) + size += q.Notarization.Size() } if q.Finalization != nil { - size += len(q.Finalization.Finalization.Bytes()) + len(q.Finalization.QC.Bytes()) + size += q.Finalization.Size() } if q.EmptyNotarization != nil { - size += len(q.EmptyNotarization.Vote.Bytes()) + len(q.EmptyNotarization.QC.Bytes()) + size += q.EmptyNotarization.Size() } - return size, nil + return size } type VerifiedFinalizedBlock struct { diff --git a/common/msg_test.go b/common/msg_test.go index 32567e68..ab296f44 100644 --- a/common/msg_test.go +++ b/common/msg_test.go @@ -207,3 +207,40 @@ func TestQuorumRoundMalformed(t *testing.T) { } } + +func TestSizeMatchesBytes(t *testing.T) { + qc := testutil.TestQC{ + {Signer: common.NodeID{1}, Value: []byte("signature1")}, + {Signer: common.NodeID{2}, Value: []byte("signature2")}, + {Signer: common.NodeID{3}, Value: []byte("signature3")}, + } + + bh := common.BlockHeader{ + ProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Epoch: 2, + Round: 3, + Seq: 4, + Prev: common.Digest{3}, + }, + Digest: common.Digest{6}, + } + require.Equal(t, len(bh.Bytes()), bh.Size()) + + + emptyVote := common.ToBeSignedEmptyVote{ + EmptyVoteMetadata: common.EmptyVoteMetadata{Round: 7, Epoch: 8}, + } + require.Equal(t, len(emptyVote.Bytes()), emptyVote.Size()) + + + notarization := common.Notarization{Vote: common.ToBeSignedVote{BlockHeader: bh}, QC: qc} + require.Equal(t, len(notarization.Vote.Bytes())+len(notarization.QC.Bytes()), notarization.Size()) + + finalization := common.Finalization{Finalization: common.ToBeSignedFinalization{BlockHeader: bh}, QC: qc} + require.Equal(t, len(finalization.Finalization.Bytes())+len(finalization.QC.Bytes()), finalization.Size()) + + emptyNotarization := common.EmptyNotarization{Vote: emptyVote, QC: qc} + require.Equal(t, len(emptyNotarization.Vote.Bytes())+len(emptyNotarization.QC.Bytes()), emptyNotarization.Size()) + +} diff --git a/external.go b/external.go index ad19ea40..f3af48ef 100644 --- a/external.go +++ b/external.go @@ -6,6 +6,7 @@ package simplex import ( "context" + "github.com/StephenButtolph/canoto" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" ) @@ -37,6 +38,21 @@ func (p *ParsedBlock) Bytes() ([]byte, error) { } return rawBlock.MarshalCanoto(), nil } +func (p *ParsedBlock) Size() int { + (&p.Metadata).CalculateCanotoCache() + metadataSize := (&p.Metadata).CachedCanotoSize() + var size uint64 + if metadataSize != 0 { + size += uint64(len(canotoTag_RawBlock__Metadata)) + canoto.SizeUint(metadataSize) + metadataSize + } + if p.InnerBlock != nil { + innerBlockSize := uint64(p.InnerBlock.Size()) + if innerBlockSize != 0 { + size += uint64(len(canotoTag_RawBlock__InnerBlockBytes)) + canoto.SizeUint(innerBlockSize) + innerBlockSize + } + } + return int(size) +} func (p *ParsedBlock) BlockHeader() common.BlockHeader { var md *common.ProtocolMetadata diff --git a/instance_test.go b/instance_test.go index bbb2c7dd..83c9986b 100644 --- a/instance_test.go +++ b/instance_test.go @@ -384,6 +384,25 @@ func TestInstanceRestartAcrossEpochs(t *testing.T) { // The restarted node keeps extending the chain. waitForNumBlocks(t, storage, storage.NumBlocks()+2) } +func TestParseBlockSizeMatchesBytes(t *testing.T) { + pb := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: []byte{1, 2, 3}, + SimplexBlacklist: []byte{4, 5}, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 7, + TS: time.UnixMilli(8), + Payload: []byte("payload"), + }, + }, + } + bytes, err := pb.Bytes() + require.NoError(t, err) + require.Equal(t, len(bytes), pb.Size()) +} // requireTipIsSealing asserts whether the last block in storage is a sealing block. func requireTipIsSealing(t *testing.T, storage *MockStorage, want bool) { @@ -527,6 +546,9 @@ func (b *testInnerBlock) Digest() [32]byte { bytes, _ := b.Bytes() return sha256.Sum256(bytes) } +func (b *testInnerBlock) Size() int { + return 16 + len(b.Payload) +} func (b *testInnerBlock) Height() uint64 { return b.Height_ } func (b *testInnerBlock) Timestamp() time.Time { return b.TS } diff --git a/msm/misc.go b/msm/misc.go index 920bee8d..06471542 100644 --- a/msm/misc.go +++ b/msm/misc.go @@ -52,6 +52,8 @@ type VMBlock interface { // Bytes returns the byte representation of this block. Bytes() ([]byte, error) + // Size returns the number of bytes of the Bytes representation + Size() int } type bitmask big.Int diff --git a/msm/util_test.go b/msm/util_test.go index e2d8f975..dd0f963a 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -48,6 +48,9 @@ type InnerBlock struct { func (i *InnerBlock) Bytes() ([]byte, error) { return i.bytes, nil } +func (i *InnerBlock) Size() int { + return len(i.bytes) +} func (i *InnerBlock) Digest() [32]byte { return sha256.Sum256(i.bytes) @@ -73,6 +76,9 @@ type fakeVMBlock struct { func (f *fakeVMBlock) Bytes() ([]byte, error) { panic("implement me") } +func (f *fakeVMBlock) Size() int { + return 0 +} func (f *fakeVMBlock) Digest() [32]byte { return [32]byte{} } func (f *fakeVMBlock) Height() uint64 { return f.height } diff --git a/simplex/epoch.go b/simplex/epoch.go index f7e53679..0a2bc928 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -3101,13 +3101,9 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co if req.LatestRound > 0 { latestRound := e.getLatestVerifiedQuorumRound() if latestRound != nil && latestRound.GetRound() > req.LatestRound { - size, err := latestRound.EstimateSize() - if err != nil { - e.Logger.Error("Failed estimating size of latest round", zap.Error(err)) - } else { - response.LatestRound = latestRound - remainingBytes -= size - } + size := latestRound.Size() + response.LatestRound = latestRound + remainingBytes -= size } } if req.LatestFinalizedSeq > 0 { @@ -3116,13 +3112,10 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co VerifiedBlock: e.lastBlock.VerifiedBlock, Finalization: &e.lastBlock.Finalization, } - size, err := latestFinalizedSeq.EstimateSize() - if err != nil { - e.Logger.Error("Failed estimating size of latest finalized seq", zap.Error(err)) - } else { - response.LatestFinalizedSeq = latestFinalizedSeq - remainingBytes -= size - } + size := latestFinalizedSeq.Size() + response.LatestFinalizedSeq = latestFinalizedSeq + remainingBytes -= size + } } @@ -3134,12 +3127,7 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co seqData = seqData[:i] break } - size, err := quorumRound.EstimateSize() - if err != nil { - e.Logger.Error("Failed estimating size of quorom round", zap.Uint64("seq", seq), zap.Error(err)) - seqData = seqData[:i] - break - } + size := quorumRound.Size() if size > remainingBytes { e.Logger.Debug("Replication response reached size limit", zap.Stringer("from", from), @@ -3160,11 +3148,7 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co // we cannot break early since empty votes may continue } - size, err := quorumRound.EstimateSize() - if err != nil { - e.Logger.Error("Failed estimating size of quorom round", zap.Uint64("round", roundNum), zap.Error(err)) - break - } + size := quorumRound.Size() if size > remainingBytes { e.Logger.Debug("Replication response reached size limit", zap.Stringer("from", from), diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 8581c6eb..1711ff48 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" "github.com/ava-labs/simplex/simplex" "github.com/ava-labs/simplex/testutil" "github.com/stretchr/testify/require" @@ -559,11 +560,10 @@ func TestReplicationRequestSizeLimited(t *testing.T) { } // measure one quorum round and budget for roughly three of them - oneRound, err := (&common.VerifiedQuorumRound{ + oneRound := (&common.VerifiedQuorumRound{ VerifiedBlock: seqs[0].VerifiedBlock, Finalization: &seqs[0].Finalization, - }).EstimateSize() - require.NoError(t, err) + }).Size() conf.MaxReplicationResponseSize = 3*oneRound + oneRound/2 e, err := simplex.NewEpoch(conf) @@ -589,7 +589,7 @@ func TestReplicationRequestSizeLimited(t *testing.T) { total := 0 for i, data := range resp.Data { require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Seq) - size, err := data.EstimateSize() + size := data.Size() require.NoError(t, err) total += size } diff --git a/testutil/block.go b/testutil/block.go index 8df92e5a..a8175bbd 100644 --- a/testutil/block.go +++ b/testutil/block.go @@ -108,6 +108,14 @@ func (t *TestBlock) Bytes() ([]byte, error) { return rawBytes, nil } +func (t *TestBlock) Size() int { + bytes, err := t.Bytes() + if err != nil { + return 0 + } + return len(bytes) +} + type EncodedTestBlock struct { Data []byte Metadata []byte @@ -161,6 +169,14 @@ func (i *InnerBlock) Bytes() ([]byte, error) { return i.Content, nil } +func (i *InnerBlock) Size() int { + bytes, err := i.Bytes() + if err != nil { + return 0 + } + return len(bytes) +} + func (i *InnerBlock) Digest() [32]byte { return sha256.Sum256(i.Content) } diff --git a/testutil/random_network/block.go b/testutil/random_network/block.go index 0a9ef623..2c5a4b70 100644 --- a/testutil/random_network/block.go +++ b/testutil/random_network/block.go @@ -84,6 +84,14 @@ func (b *Block) Bytes() ([]byte, error) { return asn1.Marshal(encodedB) } +func (b *Block) Size() int { + bytes, err := b.Bytes() + if err != nil { + return 0 + } + return len(bytes) +} + func (b *Block) containsTX(txID txID) bool { for _, tx := range b.txs { if tx.ID == txID { diff --git a/testutil/util.go b/testutil/util.go index b4694d86..1150d2ed 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -176,6 +176,9 @@ func (t TestQC) Bytes() []byte { } return bytes } +func (t TestQC) Size() int { + return len(t.Bytes()) +} type TestSigner struct { } From 7303b3aedc0b994f92c82c62887da8b92fcad889 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Fri, 17 Jul 2026 16:15:26 -0400 Subject: [PATCH 10/20] go format --- common/msg_test.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/common/msg_test.go b/common/msg_test.go index ab296f44..c18fd3f0 100644 --- a/common/msg_test.go +++ b/common/msg_test.go @@ -218,22 +218,20 @@ func TestSizeMatchesBytes(t *testing.T) { bh := common.BlockHeader{ ProtocolMetadata: common.ProtocolMetadata{ Version: 1, - Epoch: 2, - Round: 3, - Seq: 4, - Prev: common.Digest{3}, + Epoch: 2, + Round: 3, + Seq: 4, + Prev: common.Digest{3}, }, Digest: common.Digest{6}, } require.Equal(t, len(bh.Bytes()), bh.Size()) - emptyVote := common.ToBeSignedEmptyVote{ - EmptyVoteMetadata: common.EmptyVoteMetadata{Round: 7, Epoch: 8}, - } + EmptyVoteMetadata: common.EmptyVoteMetadata{Round: 7, Epoch: 8}, + } require.Equal(t, len(emptyVote.Bytes()), emptyVote.Size()) - notarization := common.Notarization{Vote: common.ToBeSignedVote{BlockHeader: bh}, QC: qc} require.Equal(t, len(notarization.Vote.Bytes())+len(notarization.QC.Bytes()), notarization.Size()) From 836cfa27a048f7dd7fe51bc3f9b64fc4cc78961d Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Fri, 17 Jul 2026 16:20:22 -0400 Subject: [PATCH 11/20] fix removed unused import --- simplex/replication_request_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 1711ff48..ce4d0936 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ava-labs/simplex/common" - metadata "github.com/ava-labs/simplex/msm" "github.com/ava-labs/simplex/simplex" "github.com/ava-labs/simplex/testutil" "github.com/stretchr/testify/require" From 15bb351e4816729a6313e23b55ba9f3fb18e634d Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Tue, 21 Jul 2026 13:51:19 -0400 Subject: [PATCH 12/20] fixed and cleaned tests --- simplex/replication_request_test.go | 81 +++++------------------------ 1 file changed, 13 insertions(+), 68 deletions(-) diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 022ed6d4..5d9d1776 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -438,8 +438,13 @@ func TestReplicationRequestSeqsAndRoundsTruncated(t *testing.T) { // notarize a few rounds past the indexed blocks: rounds 12...16 // not finalized numNotarized := uint64(5) + notarized := make([]common.VerifiedQuorumRound, 0, numNotarized) for i := uint64(0); i < numNotarized; i++ { - advanceRoundFromNotarization(t, e, bb) + block, notarization := advanceRoundFromNotarization(t, e, bb) + notarized = append(notarized, common.VerifiedQuorumRound{ + VerifiedBlock: block, + Notarization: notarization, + }) } // oversized, unsortedseqs seqs request: 0 ...2*MaxRoundWindow @@ -468,74 +473,14 @@ func TestReplicationRequestSeqsAndRoundsTruncated(t *testing.T) { msg := <-comm.in resp := msg.VerifiedReplicationResponse - require.Len(t, resp.Data, int(conf.MaxRoundWindow)+2) - - // firts maxRoundWindow items: 0...9 in order + expected := make([]common.VerifiedQuorumRound, 0, conf.MaxRoundWindow+2) for i := uint64(0); i < conf.MaxRoundWindow; i++ { - data := resp.Data[i] - require.Equal(t, i, data.VerifiedBlock.BlockHeader().Seq) - require.NotNil(t, data.Finalization) - require.Equal(t, seqs[i].Finalization, *data.Finalization) - } - - // last two items: notarized rounds 12, 13 -- 14..16 truncated as well - for i, expectedRound := range []uint64{numIndexed, numIndexed + 1} { - data := resp.Data[int(conf.MaxRoundWindow)+i] - require.Equal(t, expectedRound, data.VerifiedBlock.BlockHeader().Round) - require.NotNil(t, data.Notarization) - require.Nil(t, data.Finalization) - } - -} - -// TestReplicationRequestRoundsTruncated ensures a request with more rounds -// than MaxRoundWindow is answered with the lowest MaxRoundWindow rounds -// rather than being dropped. -func TestReplicationRequestRoundsTruncated(t *testing.T) { - bb := testutil.NewTestBlockBuilder() - nodes := []common.NodeID{{1}, {2}, {3}, {4}} - comm := NewListenerComm(nodes) - noop := testutil.NewNoopComm(nodes) - conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], noop, bb) - conf.ReplicationEnabled = true - - e, err := simplex.NewEpoch(conf) - require.NoError(t, err) - t.Cleanup(e.Stop) - require.NoError(t, e.Start()) - - numRounds := 2 * conf.MaxRoundWindow - rounds := make(map[uint64]common.VerifiedQuorumRound, numRounds) - for i := uint64(0); i < numRounds; i++ { - block, notarization := advanceRoundFromNotarization(t, e, bb) - rounds[i] = common.VerifiedQuorumRound{ - VerifiedBlock: block, - Notarization: notarization, - } + expected = append(expected, common.VerifiedQuorumRound{ + VerifiedBlock: seqs[i].VerifiedBlock, + Finalization: &seqs[i].Finalization, + }) } - require.Equal(t, numRounds, e.Metadata().Round) + expected = append(expected, notarized[:2]...) + require.Equal(t, expected, resp.Data) - // request 2*MaxRoundWindow rounds, unsorted (highest first), - // to also pin down that truncation happens after sorting - requested := make([]uint64, 0, numRounds) - for i := numRounds; i > 0; i-- { - requested = append(requested, i-1) - } - - e.Comm = comm - require.NoError(t, e.HandleMessage(&common.Message{ - ReplicationRequest: &common.ReplicationRequest{ - Rounds: requested, - }, - }, nodes[1])) - - msg := <-comm.in - resp := msg.VerifiedReplicationResponse - - // we should get exactly the lowest MaxRoundWindow rounds: 0..MaxRoundWindow-1 - require.Len(t, resp.Data, int(conf.MaxRoundWindow)) - for i, data := range resp.Data { - require.Equal(t, uint64(i), data.VerifiedBlock.BlockHeader().Round) - require.Equal(t, rounds[uint64(i)].Notarization, data.Notarization) - } } From a59a7ecb31451cf3938e7d33228849a607a23fdd Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Tue, 21 Jul 2026 15:01:28 -0400 Subject: [PATCH 13/20] changed the order of the test, setup then create the epoch --- simplex/replication_request_test.go | 56 ++++++++++++++++------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 5d9d1776..b5a66303 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -418,36 +418,40 @@ func TestReplicationRequestSeqsAndRoundsTruncated(t *testing.T) { bb := testutil.NewTestBlockBuilder() nodes := []common.NodeID{{1}, {2}, {3}, {4}} comm := NewListenerComm(nodes) - noop := testutil.NewNoopComm(nodes) ctx := context.Background() - conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], noop, bb) + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) conf.ReplicationEnabled = true // index maxRoundsWindow + 2 blocks: 0...11 numIndexed := conf.MaxRoundWindow + 2 - seqs := createBlocks(t, nodes, numIndexed) + numNotarized := uint64(5) + blocks := createBlocks(t, nodes, numIndexed+numNotarized) + seqs := blocks[:numIndexed] for _, data := range seqs { require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) } - e, err := simplex.NewEpoch(conf) - require.NoError(t, err) - t.Cleanup(e.Stop) - require.NoError(t, e.Start()) - // notarize a few rounds past the indexed blocks: rounds 12...16 // not finalized - numNotarized := uint64(5) + quorom := common.Quorum(len(nodes)) notarized := make([]common.VerifiedQuorumRound, 0, numNotarized) - for i := uint64(0); i < numNotarized; i++ { - block, notarization := advanceRoundFromNotarization(t, e, bb) + + for _, data := range blocks[numIndexed:] { + blockBytes, err := data.VerifiedBlock.Bytes() + require.NoError(t, err) + require.NoError(t, wal.Append(common.BlockRecord(data.VerifiedBlock.BlockHeader(), blockBytes))) + + notarization, err := testutil.NewNotarization(conf.Logger, &testutil.TestSignatureAggregator{N: len(nodes)}, data.VerifiedBlock, nodes[:quorom]) + require.NoError(t, err) + require.NoError(t, wal.Append(common.NewQuorumRecord(notarization.QC.Bytes(), notarization.Vote.Bytes(), common.NotarizationRecordType))) + notarized = append(notarized, common.VerifiedQuorumRound{ - VerifiedBlock: block, - Notarization: notarization, + VerifiedBlock: data.VerifiedBlock, + Notarization: ¬arization, }) } - // oversized, unsortedseqs seqs request: 0 ...2*MaxRoundWindow + // oversized, unsorted seqs seqs request: 0 ...2*MaxRoundWindow // truncation keeps seqs 0...9 requestedSeqs := make([]uint64, 0, 2*conf.MaxRoundWindow) for i := 2 * conf.MaxRoundWindow; i > 0; i-- { @@ -462,7 +466,20 @@ func TestReplicationRequestSeqsAndRoundsTruncated(t *testing.T) { requestedRounds = append(requestedRounds, i-1) } - e.Comm = comm + expected := make([]common.VerifiedQuorumRound, 0, conf.MaxRoundWindow+2) + for i := uint64(0); i < conf.MaxRoundWindow; i++ { + expected = append(expected, common.VerifiedQuorumRound{ + VerifiedBlock: seqs[i].VerifiedBlock, + Finalization: &seqs[i].Finalization, + }) + } + expected = append(expected, notarized[:2]...) + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + require.NoError(t, e.HandleMessage(&common.Message{ ReplicationRequest: &common.ReplicationRequest{ Seqs: requestedSeqs, @@ -472,15 +489,6 @@ func TestReplicationRequestSeqsAndRoundsTruncated(t *testing.T) { msg := <-comm.in resp := msg.VerifiedReplicationResponse - - expected := make([]common.VerifiedQuorumRound, 0, conf.MaxRoundWindow+2) - for i := uint64(0); i < conf.MaxRoundWindow; i++ { - expected = append(expected, common.VerifiedQuorumRound{ - VerifiedBlock: seqs[i].VerifiedBlock, - Finalization: &seqs[i].Finalization, - }) - } - expected = append(expected, notarized[:2]...) require.Equal(t, expected, resp.Data) } From 70ad8e3434bd4d481f1dd5400eda16fd879572d7 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Tue, 21 Jul 2026 18:02:11 -0400 Subject: [PATCH 14/20] moved Size() from ParsedBlock to StateMachineBlock --- adapters.go | 2 +- external.go | 39 -------------------- instance_test.go | 6 ++-- external.canoto.go => msm/block.canoto.go | 6 ++-- msm/block.go | 44 +++++++++++++++++++++++ 5 files changed, 51 insertions(+), 46 deletions(-) rename external.canoto.go => msm/block.canoto.go (98%) diff --git a/adapters.go b/adapters.go index 76f083c6..dad6124f 100644 --- a/adapters.go +++ b/adapters.go @@ -234,7 +234,7 @@ type blockDeserializer struct { } func (bp *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) (common.Block, error) { - var rawBlock RawBlock + var rawBlock metadata.RawBlock if err := rawBlock.UnmarshalCanoto(bytes); err != nil { return nil, err } diff --git a/external.go b/external.go index 4e3a62a8..1bb28c83 100644 --- a/external.go +++ b/external.go @@ -6,54 +6,15 @@ package simplex import ( "context" - "github.com/StephenButtolph/canoto" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" ) -type RawBlock struct { - Metadata metadata.StateMachineMetadata `canoto:"value,1"` - InnerBlockBytes []byte `canoto:"bytes,2"` - - canotoData canotoData_RawBlock -} - type ParsedBlock struct { metadata.StateMachineBlock msm *metadata.StateMachine } -func (p *ParsedBlock) Bytes() ([]byte, error) { - var innerBlockBytes []byte - if p.InnerBlock != nil { - rawInnerBlock, err := p.InnerBlock.Bytes() - if err != nil { - return nil, err - } - innerBlockBytes = rawInnerBlock - } - rawBlock := &RawBlock{ - Metadata: p.Metadata, - InnerBlockBytes: innerBlockBytes, - } - return rawBlock.MarshalCanoto(), nil -} -func (p *ParsedBlock) Size() int { - (&p.Metadata).CalculateCanotoCache() - metadataSize := (&p.Metadata).CachedCanotoSize() - var size uint64 - if metadataSize != 0 { - size += uint64(len(canotoTag_RawBlock__Metadata)) + canoto.SizeUint(metadataSize) + metadataSize - } - if p.InnerBlock != nil { - innerBlockSize := uint64(p.InnerBlock.Size()) - if innerBlockSize != 0 { - size += uint64(len(canotoTag_RawBlock__InnerBlockBytes)) + canoto.SizeUint(innerBlockSize) + innerBlockSize - } - } - return int(size) -} - func (p *ParsedBlock) BlockHeader() common.BlockHeader { var md *common.ProtocolMetadata var err error diff --git a/instance_test.go b/instance_test.go index 957cb11f..25ac9bda 100644 --- a/instance_test.go +++ b/instance_test.go @@ -818,7 +818,7 @@ func (m *MockStorage) blockAt(seq uint64) (metadata.StateMachineBlock, bool) { } func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { - raw := &RawBlock{} + raw := &metadata.RawBlock{} require.NoError(m.t, raw.UnmarshalCanoto(encoded)) var inner avalanchego.VMBlock if len(raw.InnerBlockBytes) > 0 { @@ -942,10 +942,10 @@ func (n *inMemNetwork) dispatch(inst *Instance, m netMsg) { // toRawBlock re-encodes a verified block into the wire RawBlock the receiving // instance parses in HandleBlockMessage. -func toRawBlock(t *testing.T, vb common.VerifiedBlock) *RawBlock { +func toRawBlock(t *testing.T, vb common.VerifiedBlock) *metadata.RawBlock { bytes, err := vb.Bytes() require.NoError(t, err) - raw := &RawBlock{} + raw := &metadata.RawBlock{} require.NoError(t, raw.UnmarshalCanoto(bytes)) return raw } diff --git a/external.canoto.go b/msm/block.canoto.go similarity index 98% rename from external.canoto.go rename to msm/block.canoto.go index 2c3df92f..960373f3 100644 --- a/external.canoto.go +++ b/msm/block.canoto.go @@ -1,9 +1,9 @@ // Code generated by canoto. DO NOT EDIT. // versions: // canoto v0.19.0 -// source: external.go +// source: block.go -package simplex +package metadata import ( "io" @@ -63,7 +63,7 @@ func (*RawBlock) CanotoSpec(types ...reflect.Type) *canoto.Spec { return s } -// UnmarshalCanoto unmarshals a Canoto-rawBlock byte slice into the struct. +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. // // During parsing, the canoto cache is saved. func (c *RawBlock) UnmarshalCanoto(bytes []byte) error { diff --git a/msm/block.go b/msm/block.go index 944638ec..239efdc7 100644 --- a/msm/block.go +++ b/msm/block.go @@ -9,8 +9,11 @@ import ( "github.com/ava-labs/simplex/avalanchego" "github.com/ava-labs/simplex/common" + "github.com/StephenButtolph/canoto" ) +//go:generate go run github.com/StephenButtolph/canoto/canoto block.go + type BlockType uint8 // A StateMachineBlock is a representation of a parsed OuterBlock, containing the inner block and the metadata. @@ -21,6 +24,14 @@ type StateMachineBlock struct { Metadata StateMachineMetadata } +// RawBlock is the serialized form of a StateMachineBlock. +type RawBlock struct { + Metadata StateMachineMetadata `canoto:"value,1"` + InnerBlockBytes []byte `canoto:"bytes,2"` + + canotoData canotoData_RawBlock +} + // Clone returns a shallow copy of the block, skipping the canoto caches // so it is safe to call while the original is being marshaled. func (smb *StateMachineBlock) Clone() StateMachineBlock { @@ -120,3 +131,36 @@ func (smb *StateMachineBlock) SealingBlockInfo() *common.SealingBlockInfo { PrevSealingBlockHash: smb.Metadata.SimplexEpochInfo.PrevSealingBlockHash, } } + + +func (smb *StateMachineBlock) Bytes() ([]byte, error){ + var innerBlockBytes []byte + if smb.InnerBlock != nil { + rawInnerBlock, err := smb.InnerBlock.Bytes() + if err != nil { + return nil, err + } + innerBlockBytes = rawInnerBlock + } + rawBlock := &RawBlock{ + Metadata: smb.Metadata, + InnerBlockBytes: innerBlockBytes, + } + return rawBlock.MarshalCanoto(), nil +} + +func (smb *StateMachineBlock) Size() int { + (&smb.Metadata).CalculateCanotoCache() + metadataSize := (&smb.Metadata).CachedCanotoSize() + var size uint64 + if metadataSize != 0 { + size += uint64(len(canotoTag_RawBlock__Metadata)) + canoto.SizeUint(metadataSize) + metadataSize + } + if smb.InnerBlock != nil { + innerBlockSize := uint64(smb.InnerBlock.Size()) + if innerBlockSize != 0 { + size += uint64(len(canotoTag_RawBlock__InnerBlockBytes)) + canoto.SizeUint(innerBlockSize) + innerBlockSize + } + } + return int(size) +} \ No newline at end of file From 6ecd3a06984f8bdc1f1f6e2c6c50f93e318d275c Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Thu, 23 Jul 2026 17:07:23 -0400 Subject: [PATCH 15/20] removed dependency on canoto, size is computed at first serialization, cashing the full block size --- avalanchego/misc.go | 2 -- instance_test.go | 26 +++++++++++++++++++++++--- msm/block.go | 34 +++++++++++++++------------------- msm/util_test.go | 6 ------ testutil/block.go | 8 -------- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/avalanchego/misc.go b/avalanchego/misc.go index c0ce44f2..2f537a4b 100644 --- a/avalanchego/misc.go +++ b/avalanchego/misc.go @@ -52,8 +52,6 @@ type VMBlock interface { // Bytes returns the byte representation of this block. Bytes() ([]byte, error) - // Size returns the number of bytes of the Bytes representation - Size() int } type Bitmask big.Int diff --git a/instance_test.go b/instance_test.go index 25ac9bda..36eb8675 100644 --- a/instance_test.go +++ b/instance_test.go @@ -386,6 +386,7 @@ func TestInstanceRestartAcrossEpochs(t *testing.T) { waitForNumBlocks(t, storage, storage.NumBlocks()+2) } func TestParseBlockSizeMatchesBytes(t *testing.T) { + // Case 1: Bytes() first, Size() second, size returns the cached length. pb := &ParsedBlock{ StateMachineBlock: metadata.StateMachineBlock{ Metadata: metadata.StateMachineMetadata{ @@ -403,6 +404,28 @@ func TestParseBlockSizeMatchesBytes(t *testing.T) { bytes, err := pb.Bytes() require.NoError(t, err) require.Equal(t, len(bytes), pb.Size()) + + // Case 2: Size() first on a non serialized block. it will + // compute the size and match a later Byte() call. + pb2 := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: []byte{1, 2, 3}, + SimplexBlacklist: []byte{4, 5}, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 9, + TS: time.UnixMilli(10), + Payload: []byte("other payload"), + }, + }, + } + size := pb2.Size() + require.NotZero(t, size) + bytes2, err := pb2.Bytes() + require.NoError(t, err) + require.Equal(t, len(bytes2), size) } // requireTipIsSealing asserts whether the last block in storage is a sealing block. @@ -547,9 +570,6 @@ func (b *testInnerBlock) Digest() [32]byte { bytes, _ := b.Bytes() return sha256.Sum256(bytes) } -func (b *testInnerBlock) Size() int { - return 16 + len(b.Payload) -} func (b *testInnerBlock) Height() uint64 { return b.Height_ } func (b *testInnerBlock) Timestamp() time.Time { return b.TS } diff --git a/msm/block.go b/msm/block.go index 239efdc7..d7f0f24a 100644 --- a/msm/block.go +++ b/msm/block.go @@ -9,7 +9,6 @@ import ( "github.com/ava-labs/simplex/avalanchego" "github.com/ava-labs/simplex/common" - "github.com/StephenButtolph/canoto" ) //go:generate go run github.com/StephenButtolph/canoto/canoto block.go @@ -22,14 +21,16 @@ type StateMachineBlock struct { InnerBlock avalanchego.VMBlock // Metadata contains the state machine metadata associated with this block. Metadata StateMachineMetadata + // size is the number of bytes of the serialized block, set at serialization + size int } // RawBlock is the serialized form of a StateMachineBlock. type RawBlock struct { - Metadata StateMachineMetadata `canoto:"value,1"` - InnerBlockBytes []byte `canoto:"bytes,2"` + Metadata StateMachineMetadata `canoto:"value,1"` + InnerBlockBytes []byte `canoto:"bytes,2"` - canotoData canotoData_RawBlock + canotoData canotoData_RawBlock } // Clone returns a shallow copy of the block, skipping the canoto caches @@ -132,8 +133,7 @@ func (smb *StateMachineBlock) SealingBlockInfo() *common.SealingBlockInfo { } } - -func (smb *StateMachineBlock) Bytes() ([]byte, error){ +func (smb *StateMachineBlock) Bytes() ([]byte, error) { var innerBlockBytes []byte if smb.InnerBlock != nil { rawInnerBlock, err := smb.InnerBlock.Bytes() @@ -146,21 +146,17 @@ func (smb *StateMachineBlock) Bytes() ([]byte, error){ Metadata: smb.Metadata, InnerBlockBytes: innerBlockBytes, } - return rawBlock.MarshalCanoto(), nil + encoded := rawBlock.MarshalCanoto() + smb.size = len(encoded) + return encoded, nil } +// Size returns the number of bytes of the bytes encoding of the block. func (smb *StateMachineBlock) Size() int { - (&smb.Metadata).CalculateCanotoCache() - metadataSize := (&smb.Metadata).CachedCanotoSize() - var size uint64 - if metadataSize != 0 { - size += uint64(len(canotoTag_RawBlock__Metadata)) + canoto.SizeUint(metadataSize) + metadataSize - } - if smb.InnerBlock != nil { - innerBlockSize := uint64(smb.InnerBlock.Size()) - if innerBlockSize != 0 { - size += uint64(len(canotoTag_RawBlock__InnerBlockBytes)) + canoto.SizeUint(innerBlockSize) + innerBlockSize + if smb.size == 0 { + if _, err := smb.Bytes(); err != nil { + return 0 } } - return int(size) -} \ No newline at end of file + return smb.size +} diff --git a/msm/util_test.go b/msm/util_test.go index 4522cbd8..86b45f5c 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -49,9 +49,6 @@ type InnerBlock struct { func (i *InnerBlock) Bytes() ([]byte, error) { return i.bytes, nil } -func (i *InnerBlock) Size() int { - return len(i.bytes) -} func (i *InnerBlock) Digest() [32]byte { return sha256.Sum256(i.bytes) @@ -77,9 +74,6 @@ type fakeVMBlock struct { func (f *fakeVMBlock) Bytes() ([]byte, error) { panic("implement me") } -func (f *fakeVMBlock) Size() int { - return 0 -} func (f *fakeVMBlock) Digest() [32]byte { return [32]byte{} } func (f *fakeVMBlock) Height() uint64 { return f.height } diff --git a/testutil/block.go b/testutil/block.go index a8175bbd..26950f69 100644 --- a/testutil/block.go +++ b/testutil/block.go @@ -169,14 +169,6 @@ func (i *InnerBlock) Bytes() ([]byte, error) { return i.Content, nil } -func (i *InnerBlock) Size() int { - bytes, err := i.Bytes() - if err != nil { - return 0 - } - return len(bytes) -} - func (i *InnerBlock) Digest() [32]byte { return sha256.Sum256(i.Content) } From 9b52490833dd26c9ca715a8810c44f850a5d430f Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Fri, 24 Jul 2026 15:54:22 -0400 Subject: [PATCH 16/20] moved size() to parsedBlock and added a lock --- external.go | 17 +++++++++++++++++ instance.go | 10 +++++----- instance_test.go | 34 ++++++++++++++++++++++++++++++++++ msm/block.go | 16 +--------------- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/external.go b/external.go index 1bb28c83..f5216b1a 100644 --- a/external.go +++ b/external.go @@ -5,6 +5,7 @@ package simplex import ( "context" + "sync" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" @@ -13,6 +14,10 @@ import ( type ParsedBlock struct { metadata.StateMachineBlock msm *metadata.StateMachine + // sizeLock guards size, so Size() can be invoked concurrently + sizeLock sync.Mutex + // size caches the length of the Bytes encoding, computed on first use + size int } func (p *ParsedBlock) BlockHeader() common.BlockHeader { @@ -46,3 +51,15 @@ func (p *ParsedBlock) Verify(ctx context.Context) (common.VerifiedBlock, error) } return p, nil } +func (p *ParsedBlock) Size() int { + p.sizeLock.Lock() + defer p.sizeLock.Unlock() + if p.size == 0 { + bytes, err := p.Bytes() + if err != nil { + return 0 + } + p.size = len(bytes) + } + return p.size +} diff --git a/instance.go b/instance.go index d8666232..a306ab7e 100644 --- a/instance.go +++ b/instance.go @@ -100,7 +100,7 @@ func (i *Instance) Start(ctx context.Context) error { lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, &ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) if err != nil { return fmt.Errorf("error determining latest epoch and validator set: %w", err) } @@ -417,7 +417,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, &ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) if err != nil { return simplex.EpochConfig{}, err } @@ -581,7 +581,7 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) error { return i.startAtEpoch(epochChange.validators, epochChange.epochNum) } -func constructEpochAndValidatorSet(logger common.Logger, lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock ParsedBlock, storage Storage) (common.Nodes, uint64, error) { +func constructEpochAndValidatorSet(logger common.Logger, lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock *ParsedBlock, storage Storage) (common.Nodes, uint64, error) { epochNum := lastBlock.BlockHeader().Epoch var validatorSet metadata.NodeBLSMappings @@ -613,7 +613,7 @@ func constructEpochAndValidatorSet(logger common.Logger, lastNonSimplexInnerBloc if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { return nil, 0, fmt.Errorf("expected sealing block at seq %d, but got a non-sealing block", sealingBlockSeq) } - validatorSet = constructValidatorSetFromSealingBlock(ParsedBlock{StateMachineBlock: sealingBlock}) + validatorSet = constructValidatorSetFromSealingBlock(&ParsedBlock{StateMachineBlock: sealingBlock}) nodes = validatorSetToNodes(validatorSet) logger.Debug("Determined epoch and validator set from sealing block in storage", zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) @@ -633,7 +633,7 @@ func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { return nodes } -func constructValidatorSetFromSealingBlock(lastBlock ParsedBlock) metadata.NodeBLSMappings { +func constructValidatorSetFromSealingBlock(lastBlock *ParsedBlock) metadata.NodeBLSMappings { var validatorSet metadata.NodeBLSMappings vdrs := lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor.AggregatedMembership.Members for _, vdr := range vdrs { diff --git a/instance_test.go b/instance_test.go index 36eb8675..386463b7 100644 --- a/instance_test.go +++ b/instance_test.go @@ -426,6 +426,40 @@ func TestParseBlockSizeMatchesBytes(t *testing.T) { bytes2, err := pb2.Bytes() require.NoError(t, err) require.Equal(t, len(bytes2), size) + + // case 3: cincurrent Size() calls on a block that was never serialized. + // the goroutines rase to compute the size, the lock must make this + // safe and every call must return the correct value + + pb3 := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: []byte{1, 2, 3}, + SimplexBlacklist: []byte{4, 5}, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 11, + TS: time.UnixMilli(12), + Payload: []byte("concurrent"), + }, + }, + } + var wg sync.WaitGroup + sizes := make([]int, 4) + for i := range sizes { + wg.Add(1) + go func() { + defer wg.Done() + sizes[i] = pb3.Size() + }() + } + wg.Wait() + bytes3, err := pb3.Bytes() + require.NoError(t, err) + for _, size := range sizes { + require.Equal(t, len(bytes3), size) + } } // requireTipIsSealing asserts whether the last block in storage is a sealing block. diff --git a/msm/block.go b/msm/block.go index d7f0f24a..bf865407 100644 --- a/msm/block.go +++ b/msm/block.go @@ -21,8 +21,6 @@ type StateMachineBlock struct { InnerBlock avalanchego.VMBlock // Metadata contains the state machine metadata associated with this block. Metadata StateMachineMetadata - // size is the number of bytes of the serialized block, set at serialization - size int } // RawBlock is the serialized form of a StateMachineBlock. @@ -146,17 +144,5 @@ func (smb *StateMachineBlock) Bytes() ([]byte, error) { Metadata: smb.Metadata, InnerBlockBytes: innerBlockBytes, } - encoded := rawBlock.MarshalCanoto() - smb.size = len(encoded) - return encoded, nil -} - -// Size returns the number of bytes of the bytes encoding of the block. -func (smb *StateMachineBlock) Size() int { - if smb.size == 0 { - if _, err := smb.Bytes(); err != nil { - return 0 - } - } - return smb.size + return rawBlock.MarshalCanoto(), nil } From fd6534d9ec54d575b43fa755f0f73d615318dd49 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Mon, 27 Jul 2026 12:54:07 -0400 Subject: [PATCH 17/20] fixed comments, fixed race --- common/api.go | 2 +- external.go | 2 +- msm/block.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/api.go b/common/api.go index 3bc5b44f..44e51232 100644 --- a/common/api.go +++ b/common/api.go @@ -119,7 +119,7 @@ type VerifiedBlock interface { // Bytes returns a byte encoding of the block Bytes() ([]byte, error) - //Size returns the number of the bytes encoding of the block + // Size returns the number of the bytes encoding of the block Size() int // SealingBlockInfo returns a non-nil value for a block that is not a sealing block and that is not the first ever simplex block. diff --git a/external.go b/external.go index f5216b1a..87af5b45 100644 --- a/external.go +++ b/external.go @@ -14,7 +14,7 @@ import ( type ParsedBlock struct { metadata.StateMachineBlock msm *metadata.StateMachine - // sizeLock guards size, so Size() can be invoked concurrently + // sizeLock locks size, so Size() can be invoked concurrently sizeLock sync.Mutex // size caches the length of the Bytes encoding, computed on first use size int diff --git a/msm/block.go b/msm/block.go index bf865407..dc4aeebd 100644 --- a/msm/block.go +++ b/msm/block.go @@ -141,7 +141,7 @@ func (smb *StateMachineBlock) Bytes() ([]byte, error) { innerBlockBytes = rawInnerBlock } rawBlock := &RawBlock{ - Metadata: smb.Metadata, + Metadata: smb.Metadata.Clone(), InnerBlockBytes: innerBlockBytes, } return rawBlock.MarshalCanoto(), nil From 914795b5823129edc20429bc87a35852cdcda614 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Mon, 27 Jul 2026 13:31:16 -0400 Subject: [PATCH 18/20] sizeLock -> lock --- external.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external.go b/external.go index 87af5b45..1204fbec 100644 --- a/external.go +++ b/external.go @@ -14,8 +14,8 @@ import ( type ParsedBlock struct { metadata.StateMachineBlock msm *metadata.StateMachine - // sizeLock locks size, so Size() can be invoked concurrently - sizeLock sync.Mutex + // lock guards size, so Size() can be invoked concurrently + lock sync.Mutex // size caches the length of the Bytes encoding, computed on first use size int } From e5589ebe7ee446ce76e8da1c47cf581cc2d76190 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Mon, 27 Jul 2026 13:37:23 -0400 Subject: [PATCH 19/20] fixed sizeLock mentions --- external.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external.go b/external.go index 1204fbec..13f59853 100644 --- a/external.go +++ b/external.go @@ -52,8 +52,8 @@ func (p *ParsedBlock) Verify(ctx context.Context) (common.VerifiedBlock, error) return p, nil } func (p *ParsedBlock) Size() int { - p.sizeLock.Lock() - defer p.sizeLock.Unlock() + p.lock.Lock() + defer p.lock.Unlock() if p.size == 0 { bytes, err := p.Bytes() if err != nil { From 72048e6de2f0adf13ea984c3d5603bd9955c6238 Mon Sep 17 00:00:00 2001 From: jadalsmail Date: Mon, 27 Jul 2026 17:19:42 -0400 Subject: [PATCH 20/20] fix latestFinalizedSeq and latestRound over the size limit, added unit test --- external.go | 3 ++ simplex/epoch.go | 18 ++++++----- simplex/replication_request_test.go | 46 +++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/external.go b/external.go index 13f59853..a2e0b220 100644 --- a/external.go +++ b/external.go @@ -56,6 +56,9 @@ func (p *ParsedBlock) Size() int { defer p.lock.Unlock() if p.size == 0 { bytes, err := p.Bytes() + // TODO(#465): Bytes() fails when serializing the inner block when it is not nil and can't be serialized. + // So returning 0 here is not correct, because it will be silently swallowed up in the addition of other blocks. + // once Bytes() no longer returns an error (https://github.com/ava-labs/Simplex/issues/465), remove this branch. if err != nil { return 0 } diff --git a/simplex/epoch.go b/simplex/epoch.go index 0a2bc928..0cb637ef 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -3098,14 +3098,6 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co } remainingBytes := e.MaxReplicationResponseSize - if req.LatestRound > 0 { - latestRound := e.getLatestVerifiedQuorumRound() - if latestRound != nil && latestRound.GetRound() > req.LatestRound { - size := latestRound.Size() - response.LatestRound = latestRound - remainingBytes -= size - } - } if req.LatestFinalizedSeq > 0 { if e.lastBlock != nil && e.lastBlock.Finalization.Finalization.Seq > req.LatestFinalizedSeq { latestFinalizedSeq := &common.VerifiedQuorumRound{ @@ -3118,6 +3110,16 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co } } + if req.LatestRound > 0 { + latestRound := e.getLatestVerifiedQuorumRound() + if latestRound != nil && latestRound.GetRound() > req.LatestRound { + size := latestRound.Size() + if size <= remainingBytes || response.LatestFinalizedSeq == nil { + response.LatestRound = latestRound + remainingBytes -= size + } + } + } seqData := make([]common.VerifiedQuorumRound, len(seqs)) for i, seq := range seqs { diff --git a/simplex/replication_request_test.go b/simplex/replication_request_test.go index 834e830a..da7bb1ad 100644 --- a/simplex/replication_request_test.go +++ b/simplex/replication_request_test.go @@ -585,3 +585,49 @@ func TestReplicationRequestSizeLimitedLatestFinalizedSeq(t *testing.T) { require.NotNil(t, resp.LatestFinalizedSeq) require.Empty(t, resp.Data) } + +// + +func TestReplicationRequestSizeLimitedBothLatest(t *testing.T) { + bb := testutil.NewTestBlockBuilder() + nodes := []common.NodeID{{1}, {2}, {3}, {4}} + comm := NewListenerComm(nodes) + ctx := context.Background() + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + conf.ReplicationEnabled = true + conf.MaxReplicationResponseSize = 1 + + // blocks 0..3 indexed to storage (provides lastBlock for LatestFinalizedSeq) + // block 4 notarized via a Wal record (provides a round for LatestRound) + blocks := createBlocks(t, nodes, 5) + for _, data := range blocks[:4] { + require.NoError(t, conf.Storage.Index(ctx, data.VerifiedBlock, data.Finalization)) + } + notarizedBlock := blocks[4].VerifiedBlock + blockBytes, err := notarizedBlock.Bytes() + require.NoError(t, err) + require.NoError(t, wal.Append(common.BlockRecord(notarizedBlock.BlockHeader(), blockBytes))) + notarization, err := testutil.NewNotarization(conf.Logger, + &testutil.TestSignatureAggregator{N: len(nodes)}, + notarizedBlock, nodes[:common.Quorum(len(nodes))]) + require.NoError(t, err) + require.NoError(t, wal.Append(common.NewQuorumRecord(notarization.QC.Bytes(), + notarization.Vote.Bytes(), common.NotarizationRecordType))) + + e, err := simplex.NewEpoch(conf) + require.NoError(t, err) + t.Cleanup(e.Stop) + require.NoError(t, e.Start()) + + require.NoError(t, e.HandleMessage(&common.Message{ + ReplicationRequest: &common.ReplicationRequest{ + LatestRound: 1, + LatestFinalizedSeq: 1, + }, + }, nodes[1])) + msg := <-comm.in + resp := msg.VerifiedReplicationResponse + require.NotNil(t, resp.LatestFinalizedSeq) + require.Nil(t, resp.LatestRound) + require.Empty(t, resp.Data) +}