feat: add safe package for unified goroutine recovery

pkg/api/pin.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/ethersphere/bee/v2/pkg/file/redundancy"
 	"github.com/ethersphere/bee/v2/pkg/jsonhttp"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/storage"
 	"github.com/ethersphere/bee/v2/pkg/storer"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
@@ -238,7 +239,9 @@ func (s *Service) pinIntegrityHandler(w http.ResponseWriter, r *http.Request) {
 
 	out := make(chan storer.PinStat)
 
-	go s.pinIntegrity.Check(r.Context(), logger, querie.Ref.String(), out)
+	safe.Go(logger, "pin-integrity-check", func() {
+		s.pinIntegrity.Check(r.Context(), logger, querie.Ref.String(), out)
+	})
 
 	flusher, ok := w.(http.Flusher)
 	if !ok {

pkg/file/joiner/joiner.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/file/redundancy"
 	"github.com/ethersphere/bee/v2/pkg/file/redundancy/getter"
 	"github.com/ethersphere/bee/v2/pkg/replicas"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/storage"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
 	"golang.org/x/sync/errgroup"
@@ -279,7 +280,7 @@ func (j *joiner) readAtOffset(
 		currentReadSize = min(currentReadSize, subtrieSpan)
 
 		func(address swarm.Address, b []byte, cur, subTrieSize, off, bufferOffset, bytesToRead, subtrieSpanLimit int64) {
-			eg.Go(func() error {
+			eg.Go(safe.RunFunc(nil, "joiner-read-at-offset", func() error {
 				ch, err := g.Get(j.ctx, addr)
 				if err != nil {
 					return err
@@ -295,7 +296,7 @@ func (j *joiner) readAtOffset(
 
 				j.readAtOffset(b, chunkData, cur, subtrieSpan, off, bufferOffset, currentReadSize, bytesRead, subtrieParity, eg)
 				return nil
-			})
+			}))
 		}(addr, b, cur, subtrieSpan, off, bufferOffset, currentReadSize, subtrieSpanLimit)
 
 		bufferOffset += currentReadSize

pkg/hive/hive.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/p2p"
 	"github.com/ethersphere/bee/v2/pkg/p2p/protobuf"
 	"github.com/ethersphere/bee/v2/pkg/ratelimit"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/settlement/swap/chequebook"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
 	ma "github.com/multiformats/go-multiaddr"
@@ -313,7 +314,9 @@ func (s *Service) startCheckPeersHandler() {
 				return
 			case newPeers := <-s.peersChan:
 				s.wg.Go(func() {
-					s.checkAndAddPeers(ctx, newPeers)
+					safe.Run(s.logger, "hive-check-and-add-peers", func() {
+						s.checkAndAddPeers(ctx, newPeers)
+					})
 				})
 			}
 		}

pkg/postage/listener/listener.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/log"
 	"github.com/ethersphere/bee/v2/pkg/postage"
 	"github.com/ethersphere/bee/v2/pkg/postage/batchservice"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/transaction"
 	"github.com/ethersphere/bee/v2/pkg/util/syncutil"
 	"github.com/prometheus/client_golang/prometheus"
@@ -250,7 +251,7 @@ func (l *listener) Listen(ctx context.Context, from uint64, updater postage.Even
 	lastConfirmedBlock := uint64(0)
 
 	l.wg.Add(1)
-	listenf := func() error {
+	listenf := safe.RunFunc(l.logger, "postage-listener-func", func() error {
 		defer l.wg.Done()
 		for {
 			// if for whatever reason we are stuck for too long we terminate
@@ -350,7 +351,7 @@ func (l *listener) Listen(ctx context.Context, from uint64, updater postage.Even
 			totalTimeMetric(l.metrics.PageProcessDuration, start)
 			l.metrics.PagesProcessed.Inc()
 		}
-	}
+	})
 
 	go func() {
 		err := listenf()

pkg/pss/pss.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/log"
 	"github.com/ethersphere/bee/v2/pkg/postage"
 	"github.com/ethersphere/bee/v2/pkg/pushsync"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
 	"github.com/ethersphere/bee/v2/pkg/topology"
 )
@@ -180,7 +181,9 @@ func (p *pss) TryUnwrap(c swarm.Chunk) {
 		wg.Add(1)
 		go func(hh Handler) {
 			defer wg.Done()
-			hh(ctx, msg)
+			safe.Run(p.logger, "pss-handler", func() {
+				hh(ctx, msg)
+			})
 		}(*hh)
 	}
 	go func() {

pkg/puller/puller.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/puller/intervalstore"
 	"github.com/ethersphere/bee/v2/pkg/pullsync"
 	"github.com/ethersphere/bee/v2/pkg/rate"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/storage"
 	"github.com/ethersphere/bee/v2/pkg/storer"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
@@ -409,12 +410,16 @@ func (p *Puller) syncPeerBin(parentCtx context.Context, peer *syncPeer, bin uint
 	if cursor > 0 {
 		peer.wg.Add(1)
 		p.wg.Add(1)
-		go sync(true, peer.address, cursor)
+		safe.Go(p.logger, "puller-sync-historical", func() {
+			sync(true, peer.address, cursor)
+		})
 	}
 
 	peer.wg.Add(1)
 	p.wg.Add(1)
-	go sync(false, peer.address, cursor+1)
+	safe.Go(p.logger, "puller-sync-live", func() {
+		sync(false, peer.address, cursor+1)
+	})
 }
 
 func (p *Puller) Close() error {

pkg/pusher/pusher.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/log"
 	"github.com/ethersphere/bee/v2/pkg/postage"
 	"github.com/ethersphere/bee/v2/pkg/pushsync"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/stabilization"
 	storage "github.com/ethersphere/bee/v2/pkg/storage"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
@@ -236,7 +237,9 @@ func (s *Service) chunksWorker(startupStabilizer stabilization.Subscriber) {
 			select {
 			case sem <- struct{}{}:
 				wg.Add(1)
-				go push(op)
+				safe.Go(s.logger, "pusher-push-worker", func() {
+					push(op)
+				})
 			case <-s.quit:
 				return
 			}

pkg/pushsync/pushsync.go

@@ -22,6 +22,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/postage"
 	"github.com/ethersphere/bee/v2/pkg/pricer"
 	"github.com/ethersphere/bee/v2/pkg/pushsync/pb"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/skippeers"
 	"github.com/ethersphere/bee/v2/pkg/soc"
 	"github.com/ethersphere/bee/v2/pkg/stabilization"
@@ -242,7 +243,9 @@ func (ps *PushSync) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream)
 	chunk.WithStamp(stamp)
 
 	if cac.Valid(chunk) {
-		go ps.unwrap(chunk)
+		safe.Go(ps.logger, "pushsync-unwrap-chunk", func() {
+			ps.unwrap(chunk)
+		})
 	} else if chunk, err := soc.FromChunk(chunk); err == nil {
 		addr, err := chunk.Address()
 		if err != nil {
@@ -424,7 +427,9 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo
 					if inflight == 0 {
 						if ps.fullNode {
 							if cac.Valid(ch) {
-								go ps.unwrap(ch)
+								safe.Go(ps.logger, "pushsync-unwrap-ch", func() {
+									ps.unwrap(ch)
+								})
 							}
 							return nil, topology.ErrWantSelf
 						}
@@ -477,7 +482,9 @@ func (ps *PushSync) pushToClosest(ctx context.Context, ch swarm.Chunk, origin bo
 			ps.metrics.TotalSendAttempts.Inc()
 			inflight++
 
-			go ps.push(ctx, resultChan, peer, ch, action)
+			safe.Go(ps.logger, "pushsync-push", func() {
+				ps.push(ctx, resultChan, peer, ch, action)
+			})
 
 		case result := <-resultChan:
 			inflight--

pkg/replicas/getter.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/ethersphere/bee/v2/pkg/file/redundancy"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/soc"
 	"github.com/ethersphere/bee/v2/pkg/storage"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
@@ -70,15 +71,20 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e
 
 	// concurrently call to retrieve chunk using original CAC address
 	g.wg.Go(func() {
-		ch, err := g.Getter.Get(ctx, addr)
+		err := safe.RunFunc(nil, "replicas-get-original", func() error {
+			ch, err := g.Getter.Get(ctx, addr)
+			if err != nil {
+				return err
+			}
+
+			select {
+			case resultC <- ch:
+			case <-ctx.Done():
+			}
+			return nil
+		})()
 		if err != nil {
 			errc <- err
-			return
-		}
-
-		select {
-		case resultC <- ch:
-		case <-ctx.Done():
 		}
 	})
 	// counters
@@ -129,21 +135,25 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e
 			}
 
 			g.wg.Go(func() {
-				ch, err := g.Getter.Get(ctx, swarm.NewAddress(so.addr))
+				err := safe.RunFunc(nil, "replicas-get-replica", func() error {
+					ch, err := g.Getter.Get(ctx, swarm.NewAddress(so.addr))
+					if err != nil {
+						return err
+					}
+
+					soc, err := soc.FromChunk(ch)
+					if err != nil {
+						return err
+					}
+
+					select {
+					case resultC <- soc.WrappedChunk():
+					case <-ctx.Done():
+					}
+					return nil
+				})()
 				if err != nil {
 					errc <- err
-					return
-				}
-
-				soc, err := soc.FromChunk(ch)
-				if err != nil {
-					errc <- err
-					return
-				}
-
-				select {
-				case resultC <- soc.WrappedChunk():
-				case <-ctx.Done():
 				}
 			})
 			n++

pkg/replicas/putter.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/ethersphere/bee/v2/pkg/file/redundancy"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/soc"
 	"github.com/ethersphere/bee/v2/pkg/storage"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
@@ -44,10 +45,13 @@ func (p *putter) Put(ctx context.Context, ch swarm.Chunk) (err error) {
 	wg := sync.WaitGroup{}
 	for r := range rr.c {
 		wg.Go(func() {
-			sch, err := soc.New(r.id, ch).Sign(signer)
-			if err == nil {
-				err = p.putter.Put(ctx, sch)
-			}
+			err := safe.RunFunc(nil, "replicas-put", func() error {
+				sch, err := soc.New(r.id, ch).Sign(signer)
+				if err != nil {
+					return err
+				}
+				return p.putter.Put(ctx, sch)
+			})()
 			errc <- err
 		})
 	}

pkg/retrieval/retrieval.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/p2p/protobuf"
 	"github.com/ethersphere/bee/v2/pkg/pricer"
 	pb "github.com/ethersphere/bee/v2/pkg/retrieval/pb"
+	"github.com/ethersphere/bee/v2/pkg/safe"
 	"github.com/ethersphere/bee/v2/pkg/skippeers"
 	"github.com/ethersphere/bee/v2/pkg/soc"
 	storage "github.com/ethersphere/bee/v2/pkg/storage"
@@ -257,13 +258,13 @@ func (s *Service) RetrieveChunk(ctx context.Context, chunkAddr, sourcePeerAddr s
 
 				inflight++
 
-				go func() {
+				safe.Go(loggerV1, "retrieval-retrieve-chunk", func() {
 					span, _, ctx := s.tracer.FollowSpanFromContext(spanCtx, "retrieve-chunk", s.logger, trace.WithAttributes(
 						attribute.String("address", chunkAddr.String()),
 					))
 					defer span.End()
 					s.retrieveChunk(ctx, quit, chunkAddr, peer, resultC, action, span)
-				}()
+				})
 
 			case res := <-resultC:
 

pkg/safe/safe.go

@@ -0,0 +1,62 @@
+// Copyright 2026 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package safe
+
+import (
+	"fmt"
+	"runtime/debug"
+
+	"github.com/ethersphere/bee/v2/pkg/log"
+)
+
+// Go runs the given function in a new goroutine and recovers from panic in it.
+// Panics are logged using the provided logger (if non-nil).
+// If the logger is nil, the function is run without panic recovery.
+func Go(logger log.Logger, name string, fn func()) {
+	go func() {
+		if logger != nil {
+			defer func() {
+				if r := recover(); r != nil {
+					logger.Error(nil, "goroutine panic recovered", "name", name, "panic", fmt.Sprintf("%v", r), "stack", string(debug.Stack()))
+				}
+			}()
+		}
+		fn()
+	}()
+}
+
+// Run runs the given function synchronously and recovers from panic in it.
+// Panics are logged using the provided logger (if non-nil).
+// If the logger is nil, the function is run without panic recovery.
+func Run(logger log.Logger, name string, fn func()) {
+	if logger != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				logger.Error(nil, "panic recovered", "name", name, "panic", fmt.Sprintf("%v", r), "stack", string(debug.Stack()))
+			}
+		}()
+	}
+	fn()
+}
+
+// RunFunc returns a function wrapped with panic recovery, suitable for use in errgroup.Go.
+// Panics are logged using the provided logger (if non-nil), and the returned function returns a non-nil error.
+// Do not try to "unwrap" r to an error, since it is a panic and we don't
+// want to lose the fact that it was a panic.
+func RunFunc(logger log.Logger, name string, fn func() error) func() error {
+	return func() (err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				if logger != nil {
+					logger.Error(nil, "errgroup goroutine panic recovered", "name", name, "panic", fmt.Sprintf("%v", r), "stack", string(debug.Stack()))
+				}
+				// Do not try to "unwrap" r to an error, since it is a panic and we don't
+				// want to lose the fact that it was a panic.
+				err = fmt.Errorf("panic in %s: %v", name, r)
+			}
+		}()
+		return fn()
+	}
+}