Tools

.gitignore

@@ -7,3 +7,5 @@
 bitnsbot.cfg
 bitnsbot.conf
 .DS_Store
+/addrindex-scan
+/top-active

addrindex/addrindex.go

@@ -76,13 +76,6 @@ const rangeBlocks = 1000
 // offset within the range, and the transaction's index in that block.
 const entryLen = remainderLen + 2 + 2
 
-// maxLookup caps how many touches a single Lookup returns, so an exchange-hot
-// address can't produce an unbounded slice for a Telegram reply. Unlike the old
-// write-side cap this costs nothing on disk — the history is fully stored, it is
-// only the read that stops early, and the caller flags it the way the btcd path
-// flagged addrTxLimit. A package var for tests.
-var maxLookup = 10000
-
 // Touch is one appearance of an address in the chain: an output paying it or an
 // input spending from it, located by block height and the transaction's index in
 // that block.
@@ -160,11 +153,11 @@ func merge(touches map[string][]Touch, height int) error {
 }
 
 // Lookup returns an address's touches, oldest first, and whether the result hit
-// maxLookup (so the caller can flag partial history — the "10000+" case). It
-// seeks to the address's shard and walks that shard's ranges in order, keeping
-// only the entries whose stored remainder matches, since a shard holds every
-// address whose hash starts with the same two bytes.
-func Lookup(script []byte) (touches []Touch, capped bool) {
+// limit (so the caller can flag partial history). It seeks to the address's
+// shard and walks that shard's ranges in order, keeping only the entries whose
+// stored remainder matches, since a shard holds every address whose hash starts
+// with the same two bytes.
+func Lookup(script []byte, limit int) (touches []Touch, capped bool) {
     if db == nil { return nil, false }
     var prefix = Prefix(script)
     var remainder = prefix[shardLen:prefixLen]
@@ -174,7 +167,7 @@ func Lookup(script []byte) (touches []Touch, capped bool) {
             var base = binary.BigEndian.Uint32(k[shardLen:]) * rangeBlocks
             for i := 0; i+entryLen <= len(v); i += entryLen {
                 if !bytes.Equal(v[i:i+remainderLen], remainder) { continue }
-                if len(touches) >= maxLookup {
+                if len(touches) >= limit {
                     capped = true
                     return nil
                 }

addrindex/addrindex_test.go

@@ -27,14 +27,14 @@ func TestMergeAndLookup(t *testing.T) {
     if err := merge(map[string][]Touch{prefix: {{Height: 12, TxIndex: 3}}}, 12); err != nil {
         t.Fatalf("merge: %v", err)
     }
-    var got, capped = Lookup(script)
+    var got, capped = Lookup(script, 10000)
     var want = []Touch{{Height: 10, TxIndex: 0}, {Height: 12, TxIndex: 3}}
     if capped { t.Fatal("unexpectedly capped") }
     if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
         t.Fatalf("touches = %v, want %v", got, want)
     }
     // an address with no touches returns nothing, not an error
-    var empty, _ = Lookup([]byte("nevertouched"))
+    var empty, _ = Lookup([]byte("nevertouched"), 10000)
     if len(empty) != 0 {
         t.Fatalf("expected no touches, got %v", empty)
     }
@@ -46,29 +46,25 @@ func TestMergeAndLookup(t *testing.T) {
 // wrote, because appending rewrote the address's whole value every time.
 func TestLookupCaps(t *testing.T) {
     openTestDB(t)
-    var saved = maxLookup
-    t.Cleanup(func() { maxLookup = saved })
-    maxLookup = 3
     var script = []byte("hotaddress")
     var prefix = string(Prefix(script))
     for h := uint32(0); h < 5; h++ {
         if err := merge(map[string][]Touch{prefix: {{Height: h, TxIndex: 0}}}, int(h)); err != nil {
             t.Fatalf("merge at height %d: %v", h, err)
         }
     }
-    var got, capped = Lookup(script)
+    var got, capped = Lookup(script, 3)
     if !capped {
-        t.Fatal("expected capped once the read hit maxLookup")
+        t.Fatal("expected capped once the read hit limit 3")
     }
     if len(got) != 3 {
-        t.Fatalf("touches = %d, want exactly maxLookup (3)", len(got))
+        t.Fatalf("touches = %d, want exactly 3", len(got))
     }
     if got[0].Height != 0 || got[2].Height != 2 {
         t.Fatalf("expected the oldest 3 touches, got %v", got)
     }
-    // raising the cap must reveal the rest: nothing was ever dropped on disk
-    maxLookup = 100
-    var all, stillCapped = Lookup(script)
+    // raising the limit must reveal the rest: nothing was ever dropped on disk
+    var all, stillCapped = Lookup(script, 100)
     if stillCapped || len(all) != 5 {
         t.Fatalf("full history = %d touches (capped=%v), want all 5 stored", len(all), stillCapped)
     }
@@ -96,11 +92,11 @@ func TestSharedShardIsolation(t *testing.T) {
     t.Logf("shard %x shared by %q and %q", Prefix(a)[:shardLen], a, b)
     merge(map[string][]Touch{string(Prefix(a)): {{Height: 10, TxIndex: 1}}}, 10)
     merge(map[string][]Touch{string(Prefix(b)): {{Height: 20, TxIndex: 2}}}, 20)
-    var ta, _ = Lookup(a)
+    var ta, _ = Lookup(a, 10000)
     if len(ta) != 1 || ta[0].Height != 10 || ta[0].TxIndex != 1 {
         t.Fatalf("script A got %v, want only its own touch at height 10", ta)
     }
-    var tb, _ = Lookup(b)
+    var tb, _ = Lookup(b, 10000)
     if len(tb) != 1 || tb[0].Height != 20 || tb[0].TxIndex != 2 {
         t.Fatalf("script B got %v, want only its own touch at height 20", tb)
     }
@@ -116,7 +112,7 @@ func TestLookupSpansRanges(t *testing.T) {
     for _, h := range heights {
         merge(map[string][]Touch{prefix: {{Height: h, TxIndex: 0}}}, int(h))
     }
-    var got, _ = Lookup(script)
+    var got, _ = Lookup(script, 10000)
     if len(got) != len(heights) {
         t.Fatalf("touches = %d, want %d across %d ranges", len(got), len(heights), len(heights))
     }
@@ -155,8 +151,8 @@ func TestDistinctScriptsDistinctKeys(t *testing.T) {
     }
     merge(map[string][]Touch{string(Prefix(a)): {{Height: 1, TxIndex: 0}}}, 1)
     merge(map[string][]Touch{string(Prefix(b)): {{Height: 2, TxIndex: 0}}}, 2)
-    var ta, _ = Lookup(a)
-    var tb, _ = Lookup(b)
+    var ta, _ = Lookup(a, 10000)
+    var tb, _ = Lookup(b, 10000)
     if len(ta) != 1 || ta[0].Height != 1 {
         t.Fatalf("scriptA touches = %v", ta)
     }
@@ -233,8 +229,8 @@ func TestCatchUp(t *testing.T) {
     if err := catchUp(src); err != nil { t.Fatalf("catchUp: %v", err) }
     var rawA, _ = hex.DecodeString(scriptA)
     var rawB, _ = hex.DecodeString(scriptB)
-    var touchesA, _ = Lookup(rawA)
-    var touchesB, _ = Lookup(rawB)
+    var touchesA, _ = Lookup(rawA, 10000)
+    var touchesB, _ = Lookup(rawB, 10000)
     if len(touchesA) != 2 || touchesA[0].Height != 0 || touchesA[1].Height != 1 {
         t.Fatalf("scriptA touches = %v, want heights [0, 1]", touchesA)
     }
@@ -273,7 +269,7 @@ func TestCatchUpChunksAndRetries(t *testing.T) {
     }
     // heights 0-1 (one full chunk) must have been flushed before the failure at 3
     var raw, _ = hex.DecodeString(script)
-    var touches, _ = Lookup(raw)
+    var touches, _ = Lookup(raw, 10000)
     if len(touches) != 2 {
         t.Fatalf("touches after partial catch-up = %d, want 2 (the first chunk only)", len(touches))
     }
@@ -289,7 +285,7 @@ func TestCatchUpChunksAndRetries(t *testing.T) {
     if len(deepFetched) != 3 || deepFetched[0] != 2 {
         t.Fatalf("retry fetched %v, want [2 3 4]", deepFetched)
     }
-    touches, _ = Lookup(raw)
+    touches, _ = Lookup(raw, 10000)
     if len(touches) != 5 {
         t.Fatalf("touches after retry = %d, want 5", len(touches))
     }

go.mod

@@ -12,6 +12,8 @@ require (
 require (
 	github.com/go-zeromq/goczmq/v4 v4.2.2 // indirect
 	github.com/pin2t/flagex v1.0.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
 	golang.org/x/text v0.15.0 // indirect

go.sum

@@ -15,6 +15,10 @@ github.com/sourcegraph/jsonrpc2 v0.2.1 h1:2GtljixMQYUYCmIg7W9aF2dFmniq/mOr2T9tFR
 github.com/sourcegraph/jsonrpc2 v0.2.1/go.mod h1:ZafdZgk/axhT1cvZAPOhw+95nz2I/Ra5qMlU4gTRwIo=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
 go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=

info.go

@@ -301,7 +301,7 @@ var addrTxLimit = 10000
 // Both stages are concurrent and bounded, the same pattern the rest of the bot
 // uses. complete is false when the cap or the caller's deadline cut it short.
 func addressHistory(ctx context.Context, script []byte) (txs []*coreTransaction, complete bool) {
-    var touches, capped = addrindex.Lookup(script)
+    var touches, capped = addrindex.Lookup(script, 10000)
     if len(touches) == 0 { return nil, !capped }
     if len(touches) > addrTxLimit {
         touches, capped = touches[:addrTxLimit], true