From 2b62aa38cdec75c7aba7efd6269d51cb37375aca Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 11:18:53 +0500 Subject: [PATCH 01/11] add addrindex-scan tool to list transactions for an address from the index --- .gitignore | 1 + tools/addrindex-scan/main.go | 272 +++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 tools/addrindex-scan/main.go diff --git a/.gitignore b/.gitignore index 2789606..1228bf9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ bitnsbot.cfg bitnsbot.conf .DS_Store +addrindex-scan diff --git a/tools/addrindex-scan/main.go b/tools/addrindex-scan/main.go new file mode 100644 index 0000000..416f060 --- /dev/null +++ b/tools/addrindex-scan/main.go @@ -0,0 +1,272 @@ +// Command addrindex-scan scans the addrindex for a Bitcoin address and prints +// every transaction it was involved in, with amounts, inputs and outputs. +// +// Usage: +// +// addrindex-scan bc1q... -db=bitnsbot.db -core-url=http://127.0.0.1:8332 +package main + +import "bytes" +import "context" +import "encoding/hex" +import "encoding/json" +import "flag" +import "fmt" +import "net/http" +import "os" +import "strings" +import "time" + +import "go.etcd.io/bbolt" +import "bitnsbot/addrindex" +import "bitnsbot/logging" + +var dbPath = flag.String("db", "bitnsbot.db", "path to the bbolt database") +var coreURL = flag.String("core-url", "", "Bitcoin Core JSON-RPC URL") +var coreUser = flag.String("core-user", "", "Bitcoin Core RPC username") +var corePass = flag.String("core-pass", "", "Bitcoin Core RPC password") +var coreCookie = flag.String("core-cookie", "", "path to Bitcoin Core .cookie file") + +type rpcClient struct { + url string + client *http.Client + auth string +} + +func newRPCClient(url, user, pass, cookieFile string) (*rpcClient, error) { + var c = &rpcClient{url: url, client: &http.Client{}} + if cookieFile != "" { + var data, err = os.ReadFile(cookieFile) + if err != nil { return nil, fmt.Errorf("read cookie: %w", err) } + var parts = strings.SplitN(strings.TrimSpace(string(data)), ":", 2) + if len(parts) != 2 { return nil, fmt.Errorf("malformed cookie file") } + user, pass = parts[0], parts[1] + } + var req = &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + c.auth = req.Header.Get("Authorization") + return c, nil +} + +func (c *rpcClient) call(ctx context.Context, method string, params []interface{}, result interface{}) error { + if params == nil { params = []interface{}{} } + var body, err = json.Marshal(map[string]interface{}{ + "jsonrpc": "1.0", "id": "addrindex-scan", "method": method, "params": params, + }) + if err != nil { return err } + var req, reqErr = http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewReader(body)) + if reqErr != nil { return reqErr } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", c.auth) + var resp, doErr = c.client.Do(req) + if doErr != nil { return doErr } + defer resp.Body.Close() + var decoded struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + return fmt.Errorf("%s: %w", method, err) + } + if decoded.Error != nil { + return fmt.Errorf("%s: %s", method, decoded.Error.Message) + } + if result == nil { return nil } + return json.Unmarshal(decoded.Result, result) +} + +func (c *rpcClient) validateAddress(ctx context.Context, addr string) (string, error) { + var info struct { + IsValid bool `json:"isvalid"` + ScriptPubKey string `json:"scriptPubKey"` + } + if err := c.call(ctx, "validateaddress", []interface{}{addr}, &info); err != nil { + return "", err + } + if !info.IsValid { + return "", fmt.Errorf("%s is not a valid Bitcoin address", addr) + } + return info.ScriptPubKey, nil +} + +func (c *rpcClient) getBlockHash(ctx context.Context, height int64) (string, error) { + var hash string + var err = c.call(ctx, "getblockhash", []interface{}{height}, &hash) + return hash, err +} + +func (c *rpcClient) getBlockTxids(ctx context.Context, hash string) ([]string, error) { + var blk struct { + Tx []string `json:"tx"` + } + if err := c.call(ctx, "getblock", []interface{}{hash, 1}, &blk); err != nil { + return nil, err + } + return blk.Tx, nil +} + +type txDetail struct { + Txid string + Time int64 + Fee float64 + Vin []txIn + Vout []txOut +} + +type txIn struct { + Address string + Amount float64 +} + +type txOut struct { + Address string + Amount float64 +} + +func (c *rpcClient) getTransaction(ctx context.Context, txid string) (*txDetail, error) { + var raw struct { + Txid string `json:"txid"` + Time int64 `json:"time"` + Fee float64 `json:"fee"` + Vin []struct { + Coinbase string `json:"coinbase"` + PrevOut *struct { + Value float64 `json:"value"` + ScriptPubKey struct { + Address string `json:"address"` + } `json:"scriptPubKey"` + } `json:"prevout"` + } `json:"vin"` + Vout []struct { + Value float64 `json:"value"` + ScriptPubKey struct { + Address string `json:"address"` + } `json:"scriptPubKey"` + } `json:"vout"` + } + if err := c.call(ctx, "getrawtransaction", []interface{}{txid, 2}, &raw); err != nil { + return nil, err + } + var tx = &txDetail{Txid: raw.Txid, Time: raw.Time, Fee: raw.Fee} + for _, v := range raw.Vin { + if v.Coinbase != "" { + tx.Vin = append(tx.Vin, txIn{Address: "coinbase"}) + } else if v.PrevOut != nil { + tx.Vin = append(tx.Vin, txIn{Address: v.PrevOut.ScriptPubKey.Address, Amount: v.PrevOut.Value}) + } + } + for _, v := range raw.Vout { + tx.Vout = append(tx.Vout, txOut{Address: v.ScriptPubKey.Address, Amount: v.Value}) + } + return tx, nil +} + +func sats(v float64) string { + return fmt.Sprintf("%.0f sats", v*1e8) +} + +func short(s string) string { + if len(s) > 16 { return s[:8] + "..." + s[len(s)-8:] } + return s +} + +func main() { + flag.Parse() + if flag.NArg() == 0 { + fmt.Fprintln(os.Stderr, "usage: addrindex-scan
[-db=file] [-core-url=url]") + os.Exit(1) + } + var address = flag.Arg(0) + + var d, err = bbolt.Open(*dbPath, 0600, nil) + if err != nil { + logging.Fatal("open database: %v", err) + } + defer d.Close() + if err := addrindex.Init(d); err != nil { + logging.Fatal("init addrindex: %v", err) + } + + var rpc *rpcClient + if *coreURL != "" { + rpc, err = newRPCClient(*coreURL, *coreUser, *corePass, *coreCookie) + if err != nil { + logging.Fatal("RPC client: %v", err) + } + } + + var ctx = context.Background() + var scriptHex string + if rpc != nil { + scriptHex, err = rpc.validateAddress(ctx, address) + if err != nil { + logging.Fatal("validate address: %v", err) + } + } else { + logging.Fatal("Bitcoin Core RPC (-core-url) is required to resolve the address") + } + var script, _ = hex.DecodeString(scriptHex) + var touches, capped = addrindex.Lookup(script) + + if len(touches) == 0 { + fmt.Println("No transactions found for", address) + return + } + if capped { + fmt.Fprintf(os.Stderr, "warning: too many touches, showing the oldest %d\n", len(touches)) + } + + if _, ok := addrindex.LoadCursor(); !ok { + fmt.Println("Address index is still building — results may be partial.") + } + + for _, t := range touches { + var hash, err = rpc.getBlockHash(ctx, int64(t.Height)) + if err != nil { + fmt.Fprintf(os.Stderr, " block %d: %v\n", t.Height, err) + continue + } + var txids, txErr = rpc.getBlockTxids(ctx, hash) + if txErr != nil { + fmt.Fprintf(os.Stderr, " block %d txids: %v\n", t.Height, txErr) + continue + } + if int(t.TxIndex) >= len(txids) { + fmt.Fprintf(os.Stderr, " block %d: tx index %d out of range (max %d)\n", t.Height, t.TxIndex, len(txids)-1) + continue + } + var txid = txids[t.TxIndex] + var tx, txErr2 = rpc.getTransaction(ctx, txid) + if txErr2 != nil { + fmt.Fprintf(os.Stderr, " block %d tx %s: %v\n", t.Height, short(txid), txErr2) + continue + } + // build the output line + var tm = time.Unix(tx.Time, 0).UTC().Format("2 Jan 2006 15:04") + var total float64 + var inParts, outParts []string + for _, v := range tx.Vin { + total += v.Amount + inParts = append(inParts, fmt.Sprintf("%s (%s)", short(v.Address), sats(v.Amount))) + } + for _, v := range tx.Vout { + outParts = append(outParts, fmt.Sprintf("%s (%s)", short(v.Address), sats(v.Amount))) + } + var outTotal float64 + for _, v := range tx.Vout { + outTotal += v.Amount + } + fmt.Printf("%s: block #%d, pos %d, tx %s, amount %s", + tm, t.Height, t.TxIndex, short(txid), sats(outTotal)) + if len(inParts) > 0 { + fmt.Printf(", in: %s", strings.Join(inParts, ", ")) + } + if len(outParts) > 0 { + fmt.Printf(", out: %s", strings.Join(outParts, ", ")) + } + fmt.Println() + } +} From 284d7069f44d0aeef8d746250c44b631db115a1a Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 11:29:24 +0500 Subject: [PATCH 02/11] make addrindex.Lookup limit a parameter, remove maxLookup package var --- addrindex/addrindex.go | 19 ++++++------------- addrindex/addrindex_test.go | 36 ++++++++++++++++-------------------- info.go | 2 +- tools/addrindex-scan/main.go | 2 +- 4 files changed, 24 insertions(+), 35 deletions(-) diff --git a/addrindex/addrindex.go b/addrindex/addrindex.go index 79415f6..a85e864 100644 --- a/addrindex/addrindex.go +++ b/addrindex/addrindex.go @@ -76,13 +76,6 @@ const rangeBlocks = 1000 // offset within the range, and the transaction's index in that block. const entryLen = remainderLen + 2 + 2 -// maxLookup caps how many touches a single Lookup returns, so an exchange-hot -// address can't produce an unbounded slice for a Telegram reply. Unlike the old -// write-side cap this costs nothing on disk — the history is fully stored, it is -// only the read that stops early, and the caller flags it the way the btcd path -// flagged addrTxLimit. A package var for tests. -var maxLookup = 10000 - // Touch is one appearance of an address in the chain: an output paying it or an // input spending from it, located by block height and the transaction's index in // that block. @@ -160,11 +153,11 @@ func merge(touches map[string][]Touch, height int) error { } // Lookup returns an address's touches, oldest first, and whether the result hit -// maxLookup (so the caller can flag partial history — the "10000+" case). It -// seeks to the address's shard and walks that shard's ranges in order, keeping -// only the entries whose stored remainder matches, since a shard holds every -// address whose hash starts with the same two bytes. -func Lookup(script []byte) (touches []Touch, capped bool) { +// limit (so the caller can flag partial history). It seeks to the address's +// shard and walks that shard's ranges in order, keeping only the entries whose +// stored remainder matches, since a shard holds every address whose hash starts +// with the same two bytes. +func Lookup(script []byte, limit int) (touches []Touch, capped bool) { if db == nil { return nil, false } var prefix = Prefix(script) var remainder = prefix[shardLen:prefixLen] @@ -174,7 +167,7 @@ func Lookup(script []byte) (touches []Touch, capped bool) { var base = binary.BigEndian.Uint32(k[shardLen:]) * rangeBlocks for i := 0; i+entryLen <= len(v); i += entryLen { if !bytes.Equal(v[i:i+remainderLen], remainder) { continue } - if len(touches) >= maxLookup { + if len(touches) >= limit { capped = true return nil } diff --git a/addrindex/addrindex_test.go b/addrindex/addrindex_test.go index a398bb5..3b9cd1a 100644 --- a/addrindex/addrindex_test.go +++ b/addrindex/addrindex_test.go @@ -27,14 +27,14 @@ func TestMergeAndLookup(t *testing.T) { if err := merge(map[string][]Touch{prefix: {{Height: 12, TxIndex: 3}}}, 12); err != nil { t.Fatalf("merge: %v", err) } - var got, capped = Lookup(script) + var got, capped = Lookup(script, 10000) var want = []Touch{{Height: 10, TxIndex: 0}, {Height: 12, TxIndex: 3}} if capped { t.Fatal("unexpectedly capped") } if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { t.Fatalf("touches = %v, want %v", got, want) } // an address with no touches returns nothing, not an error - var empty, _ = Lookup([]byte("nevertouched")) + var empty, _ = Lookup([]byte("nevertouched"), 10000) if len(empty) != 0 { t.Fatalf("expected no touches, got %v", empty) } @@ -46,9 +46,6 @@ func TestMergeAndLookup(t *testing.T) { // wrote, because appending rewrote the address's whole value every time. func TestLookupCaps(t *testing.T) { openTestDB(t) - var saved = maxLookup - t.Cleanup(func() { maxLookup = saved }) - maxLookup = 3 var script = []byte("hotaddress") var prefix = string(Prefix(script)) for h := uint32(0); h < 5; h++ { @@ -56,19 +53,18 @@ func TestLookupCaps(t *testing.T) { t.Fatalf("merge at height %d: %v", h, err) } } - var got, capped = Lookup(script) + var got, capped = Lookup(script, 3) if !capped { - t.Fatal("expected capped once the read hit maxLookup") + t.Fatal("expected capped once the read hit limit 3") } if len(got) != 3 { - t.Fatalf("touches = %d, want exactly maxLookup (3)", len(got)) + t.Fatalf("touches = %d, want exactly 3", len(got)) } if got[0].Height != 0 || got[2].Height != 2 { t.Fatalf("expected the oldest 3 touches, got %v", got) } - // raising the cap must reveal the rest: nothing was ever dropped on disk - maxLookup = 100 - var all, stillCapped = Lookup(script) + // raising the limit must reveal the rest: nothing was ever dropped on disk + var all, stillCapped = Lookup(script, 100) if stillCapped || len(all) != 5 { t.Fatalf("full history = %d touches (capped=%v), want all 5 stored", len(all), stillCapped) } @@ -96,11 +92,11 @@ func TestSharedShardIsolation(t *testing.T) { t.Logf("shard %x shared by %q and %q", Prefix(a)[:shardLen], a, b) merge(map[string][]Touch{string(Prefix(a)): {{Height: 10, TxIndex: 1}}}, 10) merge(map[string][]Touch{string(Prefix(b)): {{Height: 20, TxIndex: 2}}}, 20) - var ta, _ = Lookup(a) + var ta, _ = Lookup(a, 10000) if len(ta) != 1 || ta[0].Height != 10 || ta[0].TxIndex != 1 { t.Fatalf("script A got %v, want only its own touch at height 10", ta) } - var tb, _ = Lookup(b) + var tb, _ = Lookup(b, 10000) if len(tb) != 1 || tb[0].Height != 20 || tb[0].TxIndex != 2 { t.Fatalf("script B got %v, want only its own touch at height 20", tb) } @@ -116,7 +112,7 @@ func TestLookupSpansRanges(t *testing.T) { for _, h := range heights { merge(map[string][]Touch{prefix: {{Height: h, TxIndex: 0}}}, int(h)) } - var got, _ = Lookup(script) + var got, _ = Lookup(script, 10000) if len(got) != len(heights) { t.Fatalf("touches = %d, want %d across %d ranges", len(got), len(heights), len(heights)) } @@ -155,8 +151,8 @@ func TestDistinctScriptsDistinctKeys(t *testing.T) { } merge(map[string][]Touch{string(Prefix(a)): {{Height: 1, TxIndex: 0}}}, 1) merge(map[string][]Touch{string(Prefix(b)): {{Height: 2, TxIndex: 0}}}, 2) - var ta, _ = Lookup(a) - var tb, _ = Lookup(b) + var ta, _ = Lookup(a, 10000) + var tb, _ = Lookup(b, 10000) if len(ta) != 1 || ta[0].Height != 1 { t.Fatalf("scriptA touches = %v", ta) } @@ -233,8 +229,8 @@ func TestCatchUp(t *testing.T) { if err := catchUp(src); err != nil { t.Fatalf("catchUp: %v", err) } var rawA, _ = hex.DecodeString(scriptA) var rawB, _ = hex.DecodeString(scriptB) - var touchesA, _ = Lookup(rawA) - var touchesB, _ = Lookup(rawB) + var touchesA, _ = Lookup(rawA, 10000) + var touchesB, _ = Lookup(rawB, 10000) if len(touchesA) != 2 || touchesA[0].Height != 0 || touchesA[1].Height != 1 { t.Fatalf("scriptA touches = %v, want heights [0, 1]", touchesA) } @@ -273,7 +269,7 @@ func TestCatchUpChunksAndRetries(t *testing.T) { } // heights 0-1 (one full chunk) must have been flushed before the failure at 3 var raw, _ = hex.DecodeString(script) - var touches, _ = Lookup(raw) + var touches, _ = Lookup(raw, 10000) if len(touches) != 2 { t.Fatalf("touches after partial catch-up = %d, want 2 (the first chunk only)", len(touches)) } @@ -289,7 +285,7 @@ func TestCatchUpChunksAndRetries(t *testing.T) { if len(deepFetched) != 3 || deepFetched[0] != 2 { t.Fatalf("retry fetched %v, want [2 3 4]", deepFetched) } - touches, _ = Lookup(raw) + touches, _ = Lookup(raw, 10000) if len(touches) != 5 { t.Fatalf("touches after retry = %d, want 5", len(touches)) } diff --git a/info.go b/info.go index 90518b5..f836aeb 100644 --- a/info.go +++ b/info.go @@ -301,7 +301,7 @@ var addrTxLimit = 10000 // Both stages are concurrent and bounded, the same pattern the rest of the bot // uses. complete is false when the cap or the caller's deadline cut it short. func addressHistory(ctx context.Context, script []byte) (txs []*coreTransaction, complete bool) { - var touches, capped = addrindex.Lookup(script) + var touches, capped = addrindex.Lookup(script, 10000) if len(touches) == 0 { return nil, !capped } if len(touches) > addrTxLimit { touches, capped = touches[:addrTxLimit], true diff --git a/tools/addrindex-scan/main.go b/tools/addrindex-scan/main.go index 416f060..ee819ce 100644 --- a/tools/addrindex-scan/main.go +++ b/tools/addrindex-scan/main.go @@ -209,7 +209,7 @@ func main() { logging.Fatal("Bitcoin Core RPC (-core-url) is required to resolve the address") } var script, _ = hex.DecodeString(scriptHex) - var touches, capped = addrindex.Lookup(script) + var touches, capped = addrindex.Lookup(script, 1000000000) if len(touches) == 0 { fmt.Println("No transactions found for", address) From e7c90cb6c7c7db4d9adba69db25add9bae97b40d Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 11:37:45 +0500 Subject: [PATCH 03/11] add README for addrindex-scan tool --- .gitignore | 2 +- tools/addrindex-scan/README.md | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tools/addrindex-scan/README.md diff --git a/.gitignore b/.gitignore index 1228bf9..9589ea9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ bitnsbot.cfg bitnsbot.conf .DS_Store -addrindex-scan +/addrindex-scan diff --git a/tools/addrindex-scan/README.md b/tools/addrindex-scan/README.md new file mode 100644 index 0000000..217ca90 --- /dev/null +++ b/tools/addrindex-scan/README.md @@ -0,0 +1,44 @@ +# addrindex-scan + +Scans the [bitnsbot](https://github.com/pin2t/bitnsbot) address index for a Bitcoin address and prints every transaction it was involved in. + +## Usage + +``` +addrindex-scan
-db= -core-url= [-core-cookie=] +``` + +### Flags + +| Flag | Default | Description | +|---|---|---| +| `-db` | `bitnsbot.db` | Path to the bbolt database | +| `-core-url` | | Bitcoin Core JSON-RPC URL (e.g. `http://127.0.0.1:8332`) | +| `-core-user` | | RPC username | +| `-core-pass` | | RPC password | +| `-core-cookie` | | Path to `.cookie` file (alternative to user/pass) | + +### Example + +``` +addrindex-scan bc1qeandws6k5jqxsjn7dw08pfkgnd64l4sw6uv49u \ + -db=bitnsbot.db \ + -core-url=http://127.0.0.1:8332 \ + -core-cookie=/path/to/.cookie +``` + +### Output + +One transaction per line: + +``` +2 Jan 2021 13:00: block #923456, pos 123, tx 1233456...987654, amount 123 sats, in: bc1qabc...lkjh (100 sats), out: bc1qghj...lllkj (50 sats) +``` + +Each line shows the transaction time, block height, position in block, shortened txid, total output amount, inputs (with their amounts), and outputs (with their amounts). + +## Build + +``` +go build ./tools/addrindex-scan +``` From 17016d43ab6f84022b9c2da342ae8175baef6860 Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 12:19:02 +0500 Subject: [PATCH 04/11] add totals and -totals flag to addrindex-scan --- tools/addrindex-scan/main.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tools/addrindex-scan/main.go b/tools/addrindex-scan/main.go index ee819ce..221ea21 100644 --- a/tools/addrindex-scan/main.go +++ b/tools/addrindex-scan/main.go @@ -26,6 +26,7 @@ var coreURL = flag.String("core-url", "", "Bitcoin Core JSON-RPC URL") var coreUser = flag.String("core-user", "", "Bitcoin Core RPC username") var corePass = flag.String("core-pass", "", "Bitcoin Core RPC password") var coreCookie = flag.String("core-cookie", "", "path to Bitcoin Core .cookie file") +var totalsOnly = flag.Bool("totals", false, "only print totals, not individual transactions") type rpcClient struct { url string @@ -173,6 +174,10 @@ func short(s string) string { return s } +func btc(v float64) string { + return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.8f", v), "0"), ".") +} + func main() { flag.Parse() if flag.NArg() == 0 { @@ -223,6 +228,8 @@ func main() { fmt.Println("Address index is still building — results may be partial.") } + var totalReceived, totalSent float64 + var txCount int for _, t := range touches { var hash, err = rpc.getBlockHash(ctx, int64(t.Height)) if err != nil { @@ -244,20 +251,31 @@ func main() { fmt.Fprintf(os.Stderr, " block %d tx %s: %v\n", t.Height, short(txid), txErr2) continue } + txCount++ + // track amounts for this address + for _, v := range tx.Vin { + if v.Address == address { + totalSent += v.Amount + } + } + for _, v := range tx.Vout { + if v.Address == address { + totalReceived += v.Amount + } + } + if *totalsOnly { + continue + } // build the output line var tm = time.Unix(tx.Time, 0).UTC().Format("2 Jan 2006 15:04") - var total float64 var inParts, outParts []string for _, v := range tx.Vin { - total += v.Amount inParts = append(inParts, fmt.Sprintf("%s (%s)", short(v.Address), sats(v.Amount))) } - for _, v := range tx.Vout { - outParts = append(outParts, fmt.Sprintf("%s (%s)", short(v.Address), sats(v.Amount))) - } var outTotal float64 for _, v := range tx.Vout { outTotal += v.Amount + outParts = append(outParts, fmt.Sprintf("%s (%s)", short(v.Address), sats(v.Amount))) } fmt.Printf("%s: block #%d, pos %d, tx %s, amount %s", tm, t.Height, t.TxIndex, short(txid), sats(outTotal)) @@ -269,4 +287,9 @@ func main() { } fmt.Println() } + // print totals + fmt.Printf("\n%d transactions\n", txCount) + fmt.Printf("received: %s BTC\n", btc(totalReceived)) + fmt.Printf("sent: %s BTC\n", btc(totalSent)) + fmt.Printf("balance: %s BTC\n", btc(totalReceived-totalSent)) } From 0de8ff230816369bf9cd3401d1030be825bda392 Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 12:47:59 +0500 Subject: [PATCH 05/11] add top-active tool to find most active addresses in addrindex --- .gitignore | 1 + tools/top-active/main.go | 188 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 tools/top-active/main.go diff --git a/.gitignore b/.gitignore index 9589ea9..1258b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ bitnsbot.cfg bitnsbot.conf .DS_Store /addrindex-scan +/top-active diff --git a/tools/top-active/main.go b/tools/top-active/main.go new file mode 100644 index 0000000..e8106ce --- /dev/null +++ b/tools/top-active/main.go @@ -0,0 +1,188 @@ +// Command top-active scans the addrindex and prints the addresses most often +// touched on chain, sorted by transaction count descending. +// +// Usage: +// +// top-active -db=bitnsbot.db -core-url=http://127.0.0.1:8332 -top=100 +package main + +import "bytes" +import "context" +import "encoding/binary" +import "encoding/hex" +import "encoding/json" +import "flag" +import "fmt" +import "net/http" +import "os" +import "sort" +import "strings" +import "time" + +import "go.etcd.io/bbolt" +import "bitnsbot/addrindex" +import "bitnsbot/logging" + +var dbPath = flag.String("db", "bitnsbot.db", "path to the bbolt database") +var coreURL = flag.String("core-url", "", "Bitcoin Core JSON-RPC URL") +var coreUser = flag.String("core-user", "", "Bitcoin Core RPC username") +var corePass = flag.String("core-pass", "", "Bitcoin Core RPC password") +var coreCookie = flag.String("core-cookie", "", "path to Bitcoin Core .cookie file") +var topN = flag.Int("top", 100, "number of top addresses to print") + +// addrindex layout constants (must match addrindex package) +const shardLen = 2 +const remainderLen = 6 +const entryLen = remainderLen + 2 + 2 // remainder + heightOffset + txIndex +const rangeBlocks = 1000 + +type rpcClient struct { + url string + client *http.Client + auth string +} + +func newRPCClient(url, user, pass, cookieFile string) (*rpcClient, error) { + var c = &rpcClient{url: url, client: &http.Client{}} + if cookieFile != "" { + var data, err = os.ReadFile(cookieFile) + if err != nil { return nil, fmt.Errorf("read cookie: %w", err) } + var parts = strings.SplitN(strings.TrimSpace(string(data)), ":", 2) + if len(parts) != 2 { return nil, fmt.Errorf("malformed cookie file") } + user, pass = parts[0], parts[1] + } + var req = &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + c.auth = req.Header.Get("Authorization") + return c, nil +} + +func (c *rpcClient) call(ctx context.Context, method string, params []interface{}, result interface{}) error { + if params == nil { params = []interface{}{} } + var body, err = json.Marshal(map[string]interface{}{ + "jsonrpc": "1.0", "id": "top-active", "method": method, "params": params, + }) + if err != nil { return err } + var req, reqErr = http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewReader(body)) + if reqErr != nil { return reqErr } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", c.auth) + var resp, doErr = c.client.Do(req) + if doErr != nil { return doErr } + defer resp.Body.Close() + var decoded struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + return fmt.Errorf("%s: %w", method, err) + } + if decoded.Error != nil { + return fmt.Errorf("%s: %s", method, decoded.Error.Message) + } + if result == nil { return nil } + return json.Unmarshal(decoded.Result, result) +} + +func (c *rpcClient) getBlockCount(ctx context.Context) (int64, error) { + var count int64 + var err = c.call(ctx, "getblockcount", nil, &count) + return count, err +} + +type addrCount struct { + addr string + count int +} + +func main() { + flag.Parse() + + var d, err = bbolt.Open(*dbPath, 0600, nil) + if err != nil { + logging.Fatal("open database: %v", err) + } + defer d.Close() + if err := addrindex.Init(d); err != nil { + logging.Fatal("init addrindex: %v", err) + } + + var rpc *rpcClient + var tip int64 + if *coreURL != "" { + rpc, err = newRPCClient(*coreURL, *coreUser, *corePass, *coreCookie) + if err != nil { + logging.Fatal("RPC client: %v", err) + } + var ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + tip, err = rpc.getBlockCount(ctx) + cancel() + if err != nil { + logging.Fatal("get tip: %v", err) + } + } else { + logging.Fatal("Bitcoin Core RPC (-core-url) is required to get the tip height") + } + var totalRanges = tip/rangeBlocks + 1 + + var counts = make(map[string]int) + var keysProcessed int64 + + var began = time.Now() + d.View(func(tx *bbolt.Tx) error { + var b = tx.Bucket([]byte("addrindex")) + if b == nil { + logging.Fatal("addrindex bucket not found — is the database empty?") + } + var c = b.Cursor() + var lastReport time.Time + for k, v := c.First(); k != nil; k, v = c.Next() { + if len(k) < shardLen+4 { + continue + } + var shard = k[:shardLen] + var rangeIdx = binary.BigEndian.Uint32(k[shardLen:]) + keysProcessed++ + + // progress report every 5 seconds + if time.Since(lastReport) > 5*time.Second { + var pct float64 + if totalRanges > 0 { + pct = float64(rangeIdx) / float64(totalRanges) * 100 + } + fmt.Printf("\rprocessed %d keys, range %d/%d (%.0f%%)", + keysProcessed, rangeIdx, totalRanges, pct) + lastReport = time.Now() + } + + for i := 0; i+entryLen <= len(v); i += entryLen { + var entry = v[i : i+entryLen] + var remainder = entry[:remainderLen] + var full = string(append(shard, remainder...)) + counts[full]++ + } + } + return nil + }) + fmt.Printf("\rprocessed %d keys in %s\n", keysProcessed, time.Since(began).Round(time.Second)) + + // sort by count descending, take top N + var list []addrCount + for addr, cnt := range counts { + list = append(list, addrCount{addr: addr, count: cnt}) + } + sort.Slice(list, func(i, j int) bool { + return list[i].count > list[j].count + }) + if len(list) > *topN { + list = list[:*topN] + } + + fmt.Printf("\nTop %d addresses by transaction count:\n\n", len(list)) + for i, a := range list { + fmt.Printf("%d. %s — %d transactions\n", i+1, hex.EncodeToString([]byte(a.addr)), a.count) + } +} From dfdbdb2872692f795ad61eef6b9771d2b9584a51 Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 12:56:29 +0500 Subject: [PATCH 06/11] top-active: scan blockchain to get full addresses, look up counts in addrindex --- tools/top-active/main.go | 166 +++++++++++++++++++++++++-------------- 1 file changed, 109 insertions(+), 57 deletions(-) diff --git a/tools/top-active/main.go b/tools/top-active/main.go index e8106ce..82377b8 100644 --- a/tools/top-active/main.go +++ b/tools/top-active/main.go @@ -1,5 +1,6 @@ -// Command top-active scans the addrindex and prints the addresses most often -// touched on chain, sorted by transaction count descending. +// Command top-active scans the blockchain from genesis to tip, collects every +// address, looks up its transaction count in the addrindex, and prints the top N +// addresses sorted by transaction count descending. // // Usage: // @@ -8,7 +9,6 @@ package main import "bytes" import "context" -import "encoding/binary" import "encoding/hex" import "encoding/json" import "flag" @@ -30,12 +30,6 @@ var corePass = flag.String("core-pass", "", "Bitcoin Core RPC password") var coreCookie = flag.String("core-cookie", "", "path to Bitcoin Core .cookie file") var topN = flag.Int("top", 100, "number of top addresses to print") -// addrindex layout constants (must match addrindex package) -const shardLen = 2 -const remainderLen = 6 -const entryLen = remainderLen + 2 + 2 // remainder + heightOffset + txIndex -const rangeBlocks = 1000 - type rpcClient struct { url string client *http.Client @@ -93,6 +87,48 @@ func (c *rpcClient) getBlockCount(ctx context.Context) (int64, error) { return count, err } +func (c *rpcClient) getBlockHash(ctx context.Context, height int64) (string, error) { + var hash string + var err = c.call(ctx, "getblockhash", []interface{}{height}, &hash) + return hash, err +} + +func (c *rpcClient) getBlockVerbose(ctx context.Context, hash string) (*blockData, error) { + var blk blockData + var err = c.call(ctx, "getblock", []interface{}{hash, 2}, &blk) + if err != nil { return nil, err } + return &blk, nil +} + +type blockData struct { + Height int64 `json:"height"` + Time int64 `json:"time"` + Tx []txData `json:"tx"` +} + +type txData struct { + Vin []vinData `json:"vin"` + Vout []voutData `json:"vout"` +} + +type vinData struct { + Coinbase string `json:"coinbase"` + PrevOut *prevOutData `json:"prevout"` +} + +type prevOutData struct { + ScriptPubKey spkData `json:"scriptPubKey"` +} + +type voutData struct { + ScriptPubKey spkData `json:"scriptPubKey"` +} + +type spkData struct { + Address string `json:"address"` + Hex string `json:"hex"` +} + type addrCount struct { addr string count int @@ -110,64 +146,80 @@ func main() { logging.Fatal("init addrindex: %v", err) } - var rpc *rpcClient - var tip int64 - if *coreURL != "" { - rpc, err = newRPCClient(*coreURL, *coreUser, *corePass, *coreCookie) - if err != nil { - logging.Fatal("RPC client: %v", err) - } - var ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) - tip, err = rpc.getBlockCount(ctx) - cancel() - if err != nil { - logging.Fatal("get tip: %v", err) - } - } else { - logging.Fatal("Bitcoin Core RPC (-core-url) is required to get the tip height") + if *coreURL == "" { + logging.Fatal("Bitcoin Core RPC (-core-url) is required") + } + var rpc, rpcErr = newRPCClient(*coreURL, *coreUser, *corePass, *coreCookie) + if rpcErr != nil { + logging.Fatal("RPC client: %v", rpcErr) } - var totalRanges = tip/rangeBlocks + 1 - var counts = make(map[string]int) - var keysProcessed int64 + var ctx = context.Background() + var tctx, tcancel = context.WithTimeout(ctx, 15*time.Second) + var tip, tipErr = rpc.getBlockCount(tctx) + tcancel() + if tipErr != nil { + logging.Fatal("get tip: %v", tipErr) + } + var counts = make(map[string]int) // address → tx count (cached) var began = time.Now() - d.View(func(tx *bbolt.Tx) error { - var b = tx.Bucket([]byte("addrindex")) - if b == nil { - logging.Fatal("addrindex bucket not found — is the database empty?") + var lastReport time.Time + + for h := int64(0); h <= tip; h++ { + // progress report every 5 seconds + if time.Since(lastReport) > 5*time.Second { + var pct float64 + if tip > 0 { + pct = float64(h) / float64(tip) * 100 + } + fmt.Printf("\rprocessed %d / %d (%.0f%%)", h, tip, pct) + lastReport = time.Now() } - var c = b.Cursor() - var lastReport time.Time - for k, v := c.First(); k != nil; k, v = c.Next() { - if len(k) < shardLen+4 { - continue + + var bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) + var hash, hashErr = rpc.getBlockHash(bctx, h) + bcancel() + if hashErr != nil { + fmt.Fprintf(os.Stderr, "\nblock %d hash: %v\n", h, hashErr) + continue + } + + bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) + var blk, blkErr = rpc.getBlockVerbose(bctx, hash) + bcancel() + if blkErr != nil { + fmt.Fprintf(os.Stderr, "\nblock %d: %v\n", h, blkErr) + continue + } + + // collect unique addresses from this block + var seen = make(map[string]string) // address → scriptHex + for _, tx := range blk.Tx { + for _, vin := range tx.Vin { + if vin.Coinbase != "" { continue } + if vin.PrevOut != nil && vin.PrevOut.ScriptPubKey.Address != "" { + seen[vin.PrevOut.ScriptPubKey.Address] = vin.PrevOut.ScriptPubKey.Hex + } } - var shard = k[:shardLen] - var rangeIdx = binary.BigEndian.Uint32(k[shardLen:]) - keysProcessed++ - - // progress report every 5 seconds - if time.Since(lastReport) > 5*time.Second { - var pct float64 - if totalRanges > 0 { - pct = float64(rangeIdx) / float64(totalRanges) * 100 + for _, vout := range tx.Vout { + if vout.ScriptPubKey.Address != "" { + seen[vout.ScriptPubKey.Address] = vout.ScriptPubKey.Hex } - fmt.Printf("\rprocessed %d keys, range %d/%d (%.0f%%)", - keysProcessed, rangeIdx, totalRanges, pct) - lastReport = time.Now() } + } - for i := 0; i+entryLen <= len(v); i += entryLen { - var entry = v[i : i+entryLen] - var remainder = entry[:remainderLen] - var full = string(append(shard, remainder...)) - counts[full]++ + // look up uncached addresses in the addrindex + for addr, scriptHex := range seen { + if _, ok := counts[addr]; ok { + continue } + var script, _ = hex.DecodeString(scriptHex) + var touches, _ = addrindex.Lookup(script, 1000000000) + counts[addr] = len(touches) } - return nil - }) - fmt.Printf("\rprocessed %d keys in %s\n", keysProcessed, time.Since(began).Round(time.Second)) + } + fmt.Printf("\rprocessed %d / %d (100%%) in %s\n", tip, tip, time.Since(began).Round(time.Second)) // sort by count descending, take top N var list []addrCount @@ -183,6 +235,6 @@ func main() { fmt.Printf("\nTop %d addresses by transaction count:\n\n", len(list)) for i, a := range list { - fmt.Printf("%d. %s — %d transactions\n", i+1, hex.EncodeToString([]byte(a.addr)), a.count) + fmt.Printf("%d. %s — %d transactions\n", i+1, a.addr, a.count) } } From a5cc52ecdba02a6ebaf367862cfc876e56e02bcf Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 13:03:33 +0500 Subject: [PATCH 07/11] top-active: keep only top N addresses in memory during scan --- tools/top-active/main.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/top-active/main.go b/tools/top-active/main.go index 82377b8..16ac408 100644 --- a/tools/top-active/main.go +++ b/tools/top-active/main.go @@ -209,14 +209,29 @@ func main() { } } - // look up uncached addresses in the addrindex + // look up uncached addresses in the addrindex, keep only top N for addr, scriptHex := range seen { if _, ok := counts[addr]; ok { continue } var script, _ = hex.DecodeString(scriptHex) var touches, _ = addrindex.Lookup(script, 1000000000) - counts[addr] = len(touches) + var cnt = len(touches) + if len(counts) < *topN { + counts[addr] = cnt + } else { + var minAddr string + var minCnt int + for a, c := range counts { + if minAddr == "" || c < minCnt { + minAddr, minCnt = a, c + } + } + if cnt > minCnt { + delete(counts, minAddr) + counts[addr] = cnt + } + } } } fmt.Printf("\rprocessed %d / %d (100%%) in %s\n", tip, tip, time.Since(began).Round(time.Second)) From 863d6b6bd0a277159848980f6cd33e06f7ce42c4 Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 14:45:03 +0500 Subject: [PATCH 08/11] parallel processing --- tools/top-active/main.go | 166 +++++++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 69 deletions(-) diff --git a/tools/top-active/main.go b/tools/top-active/main.go index 16ac408..d570ec2 100644 --- a/tools/top-active/main.go +++ b/tools/top-active/main.go @@ -17,6 +17,8 @@ import "net/http" import "os" import "sort" import "strings" +import "sync" +import "sync/atomic" import "time" import "go.etcd.io/bbolt" @@ -37,7 +39,14 @@ type rpcClient struct { } func newRPCClient(url, user, pass, cookieFile string) (*rpcClient, error) { - var c = &rpcClient{url: url, client: &http.Client{}} + var c = &rpcClient{url: url, client: &http.Client{ + Timeout: time.Second * 5, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 50, + IdleConnTimeout: 180 * time.Second, + }, + }} if cookieFile != "" { var data, err = os.ReadFile(cookieFile) if err != nil { return nil, fmt.Errorf("read cookie: %w", err) } @@ -136,7 +145,6 @@ type addrCount struct { func main() { flag.Parse() - var d, err = bbolt.Open(*dbPath, 0600, nil) if err != nil { logging.Fatal("open database: %v", err) @@ -145,7 +153,6 @@ func main() { if err := addrindex.Init(d); err != nil { logging.Fatal("init addrindex: %v", err) } - if *coreURL == "" { logging.Fatal("Bitcoin Core RPC (-core-url) is required") } @@ -153,7 +160,6 @@ func main() { if rpcErr != nil { logging.Fatal("RPC client: %v", rpcErr) } - var ctx = context.Background() var tctx, tcancel = context.WithTimeout(ctx, 15*time.Second) var tip, tipErr = rpc.getBlockCount(tctx) @@ -161,82 +167,105 @@ func main() { if tipErr != nil { logging.Fatal("get tip: %v", tipErr) } - var counts = make(map[string]int) // address → tx count (cached) + var countsMu sync.Mutex var began = time.Now() - var lastReport time.Time - - for h := int64(0); h <= tip; h++ { - // progress report every 5 seconds - if time.Since(lastReport) > 5*time.Second { - var pct float64 - if tip > 0 { - pct = float64(h) / float64(tip) * 100 + const numWorkers = 16 + var processed atomic.Int64 + // progress reporter: prints every 5 seconds until progressDone is closed + var progressDone = make(chan struct{}) + go func() { + var ticker = time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + var h = processed.Load() + var pct float64 + if tip > 0 { + pct = float64(h) / float64(tip) * 100 + } + fmt.Printf("\rprocessed %d / %d (%.0f%%)", h, tip, pct) + case <-progressDone: + return } - fmt.Printf("\rprocessed %d / %d (%.0f%%)", h, tip, pct) - lastReport = time.Now() } - - var bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) - var hash, hashErr = rpc.getBlockHash(bctx, h) - bcancel() - if hashErr != nil { - fmt.Fprintf(os.Stderr, "\nblock %d hash: %v\n", h, hashErr) - continue - } - - bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) - var blk, blkErr = rpc.getBlockVerbose(bctx, hash) - bcancel() - if blkErr != nil { - fmt.Fprintf(os.Stderr, "\nblock %d: %v\n", h, blkErr) - continue - } - - // collect unique addresses from this block - var seen = make(map[string]string) // address → scriptHex - for _, tx := range blk.Tx { - for _, vin := range tx.Vin { - if vin.Coinbase != "" { continue } - if vin.PrevOut != nil && vin.PrevOut.ScriptPubKey.Address != "" { - seen[vin.PrevOut.ScriptPubKey.Address] = vin.PrevOut.ScriptPubKey.Hex + }() + var heights = make(chan int64, numWorkers*2) + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for h := range heights { + var bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) + var hash, hashErr = rpc.getBlockHash(bctx, h) + bcancel() + if hashErr != nil { + fmt.Fprintf(os.Stderr, "\nblock %d hash: %v\n", h, hashErr) + processed.Add(1) + continue } - } - for _, vout := range tx.Vout { - if vout.ScriptPubKey.Address != "" { - seen[vout.ScriptPubKey.Address] = vout.ScriptPubKey.Hex + bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) + var blk, blkErr = rpc.getBlockVerbose(bctx, hash) + bcancel() + if blkErr != nil { + fmt.Fprintf(os.Stderr, "\nblock %d: %v\n", h, blkErr) + processed.Add(1) + continue } - } - } - - // look up uncached addresses in the addrindex, keep only top N - for addr, scriptHex := range seen { - if _, ok := counts[addr]; ok { - continue - } - var script, _ = hex.DecodeString(scriptHex) - var touches, _ = addrindex.Lookup(script, 1000000000) - var cnt = len(touches) - if len(counts) < *topN { - counts[addr] = cnt - } else { - var minAddr string - var minCnt int - for a, c := range counts { - if minAddr == "" || c < minCnt { - minAddr, minCnt = a, c + var seen = make(map[string]string) // address → scriptHex + for _, tx := range blk.Tx { + for _, vin := range tx.Vin { + if vin.Coinbase != "" { continue } + if vin.PrevOut != nil && vin.PrevOut.ScriptPubKey.Address != "" { + seen[vin.PrevOut.ScriptPubKey.Address] = vin.PrevOut.ScriptPubKey.Hex + } + } + for _, vout := range tx.Vout { + if vout.ScriptPubKey.Address != "" { + seen[vout.ScriptPubKey.Address] = vout.ScriptPubKey.Hex + } } } - if cnt > minCnt { - delete(counts, minAddr) - counts[addr] = cnt + for addr, scriptHex := range seen { + countsMu.Lock() + _, ok := counts[addr] + if ok { + countsMu.Unlock() + continue + } + countsMu.Unlock() // release lock during I/O-bound Lookup + var script, _ = hex.DecodeString(scriptHex) + var touches, _ = addrindex.Lookup(script, 1000000000) + var cnt = len(touches) + countsMu.Lock() + if len(counts) < *topN { + counts[addr] = cnt + } else { + var minAddr string + var minCnt int + for a, c := range counts { + if minAddr == "" || c < minCnt { + minAddr, minCnt = a, c + } + } + if cnt > minCnt { + delete(counts, minAddr) + counts[addr] = cnt + } + } + countsMu.Unlock() } + processed.Add(1) } - } + }() } + for h := int64(0); h <= tip; h++ { heights <- h } + close(heights) + wg.Wait() + close(progressDone) fmt.Printf("\rprocessed %d / %d (100%%) in %s\n", tip, tip, time.Since(began).Round(time.Second)) - - // sort by count descending, take top N var list []addrCount for addr, cnt := range counts { list = append(list, addrCount{addr: addr, count: cnt}) @@ -247,7 +276,6 @@ func main() { if len(list) > *topN { list = list[:*topN] } - fmt.Printf("\nTop %d addresses by transaction count:\n\n", len(list)) for i, a := range list { fmt.Printf("%d. %s — %d transactions\n", i+1, a.addr, a.count) From 7cff0b5e1b004fc20b796e4206e57921d4981887 Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 15:47:02 +0500 Subject: [PATCH 09/11] top-active: add -web-status flag to serve live progress and leaderboard page --- tools/top-active/main.go | 68 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tools/top-active/main.go b/tools/top-active/main.go index d570ec2..7723dbf 100644 --- a/tools/top-active/main.go +++ b/tools/top-active/main.go @@ -31,6 +31,7 @@ var coreUser = flag.String("core-user", "", "Bitcoin Core RPC username") var corePass = flag.String("core-pass", "", "Bitcoin Core RPC password") var coreCookie = flag.String("core-cookie", "", "path to Bitcoin Core .cookie file") var topN = flag.Int("top", 100, "number of top addresses to print") +var webStatus = flag.String("web-status", "", "address for live web status page (e.g. 127.0.0.1:8084)") type rpcClient struct { url string @@ -167,11 +168,78 @@ func main() { if tipErr != nil { logging.Fatal("get tip: %v", tipErr) } + var counts = make(map[string]int) // address → tx count (cached) var countsMu sync.Mutex var began = time.Now() const numWorkers = 16 var processed atomic.Int64 + + // web status page + if *webStatus != "" { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + var h = processed.Load() + var pct float64 + if tip > 0 { + pct = float64(h) / float64(tip) * 100 + } + var elapsed = time.Since(began).Round(time.Second) + + countsMu.Lock() + var list []addrCount + for addr, cnt := range counts { + list = append(list, addrCount{addr: addr, count: cnt}) + } + countsMu.Unlock() + + sort.Slice(list, func(i, j int) bool { + return list[i].count > list[j].count + }) + + fmt.Fprintf(w, ` + + + + + +top-active + + + +
+
%d / %d (%.0f%%)
+
elapsed %s
+ +`, h, tip, pct, elapsed) + for i, a := range list { + fmt.Fprintf(w, ``+"\n", i+1, a.addr, a.count) + } + fmt.Fprint(w, `
#AddressTransactions
%d%s%d
+
+ +`) + }) + go func() { + fmt.Fprintf(os.Stderr, "web status listening on http://%s\n", *webStatus) + if err := http.ListenAndServe(*webStatus, nil); err != nil { + fmt.Fprintf(os.Stderr, "web status: %v\n", err) + } + }() + } // progress reporter: prints every 5 seconds until progressDone is closed var progressDone = make(chan struct{}) go func() { From 620c6441d9f86d3da56584ea374813c74e6762dc Mon Sep 17 00:00:00 2001 From: pin2t Date: Wed, 29 Jul 2026 19:17:34 +0500 Subject: [PATCH 10/11] =?UTF-8?q?addresses:=20new=20tool=20to=20scan=20blo?= =?UTF-8?q?ckchain=20and=20store=20address=E2=86=92tx-count=20in=20bbolt?= =?UTF-8?q?=20with=20msgpack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 + go.sum | 4 + tools/addresses/main.go | 318 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 tools/addresses/main.go diff --git a/go.mod b/go.mod index 483a08e..e829f0f 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ require ( require ( github.com/go-zeromq/goczmq/v4 v4.2.2 // indirect github.com/pin2t/flagex v1.0.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.15.0 // indirect diff --git a/go.sum b/go.sum index 771b447..3d9c99a 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,10 @@ github.com/sourcegraph/jsonrpc2 v0.2.1 h1:2GtljixMQYUYCmIg7W9aF2dFmniq/mOr2T9tFR github.com/sourcegraph/jsonrpc2 v0.2.1/go.mod h1:ZafdZgk/axhT1cvZAPOhw+95nz2I/Ra5qMlU4gTRwIo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/tools/addresses/main.go b/tools/addresses/main.go new file mode 100644 index 0000000..c434c49 --- /dev/null +++ b/tools/addresses/main.go @@ -0,0 +1,318 @@ +// Command addresses scans the blockchain from genesis to tip, collects every +// address, looks up its transaction count in the addrindex, and stores the +// result in a new bbolt bucket "addresses" keyed by address string, with a +// msgpack-encoded value (extensible for future fields). +// +// Usage: +// +// addresses -db=bitnsbot.db -core-url=http://127.0.0.1:8332 +package main + +import "bytes" +import "context" +import "encoding/hex" +import "encoding/json" +import "flag" +import "fmt" +import "net/http" +import "os" +import "strings" +import "sync" +import "sync/atomic" +import "time" + +import "github.com/vmihailenco/msgpack/v5" +import "go.etcd.io/bbolt" +import "bitnsbot/addrindex" +import "bitnsbot/logging" + +var dbPath = flag.String("db", "bitnsbot.db", "path to the bbolt database") +var coreURL = flag.String("core-url", "", "Bitcoin Core JSON-RPC URL") +var coreUser = flag.String("core-user", "", "Bitcoin Core RPC username") +var corePass = flag.String("core-pass", "", "Bitcoin Core RPC password") +var coreCookie = flag.String("core-cookie", "", "path to Bitcoin Core .cookie file") + +var addressesBucket = []byte("addresses") + +type rpcClient struct { + url string + client *http.Client + auth string +} + +func newRPCClient(url, user, pass, cookieFile string) (*rpcClient, error) { + var c = &rpcClient{url: url, client: &http.Client{ + Timeout: time.Second * 5, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 50, + IdleConnTimeout: 180 * time.Second, + }, + }} + if cookieFile != "" { + var data, err = os.ReadFile(cookieFile) + if err != nil { return nil, fmt.Errorf("read cookie: %w", err) } + var parts = strings.SplitN(strings.TrimSpace(string(data)), ":", 2) + if len(parts) != 2 { return nil, fmt.Errorf("malformed cookie file") } + user, pass = parts[0], parts[1] + } + var req = &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + c.auth = req.Header.Get("Authorization") + return c, nil +} + +func (c *rpcClient) call(ctx context.Context, method string, params []interface{}, result interface{}) error { + if params == nil { params = []interface{}{} } + var body, err = json.Marshal(map[string]interface{}{ + "jsonrpc": "1.0", "id": "addresses", "method": method, "params": params, + }) + if err != nil { return err } + var req, reqErr = http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewReader(body)) + if reqErr != nil { return reqErr } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", c.auth) + var resp, doErr = c.client.Do(req) + if doErr != nil { return doErr } + defer resp.Body.Close() + var decoded struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + return fmt.Errorf("%s: %w", method, err) + } + if decoded.Error != nil { + return fmt.Errorf("%s: %s", method, decoded.Error.Message) + } + if result == nil { return nil } + return json.Unmarshal(decoded.Result, result) +} + +func (c *rpcClient) getBlockCount(ctx context.Context) (int64, error) { + var count int64 + var err = c.call(ctx, "getblockcount", nil, &count) + return count, err +} + +func (c *rpcClient) getBlockHash(ctx context.Context, height int64) (string, error) { + var hash string + var err = c.call(ctx, "getblockhash", []interface{}{height}, &hash) + return hash, err +} + +func (c *rpcClient) getBlockVerbose(ctx context.Context, hash string) (*blockData, error) { + var blk blockData + var err = c.call(ctx, "getblock", []interface{}{hash, 2}, &blk) + if err != nil { return nil, err } + return &blk, nil +} + +type blockData struct { + Height int64 `json:"height"` + Time int64 `json:"time"` + Tx []txData `json:"tx"` +} + +type txData struct { + Vin []vinData `json:"vin"` + Vout []voutData `json:"vout"` +} + +type vinData struct { + Coinbase string `json:"coinbase"` + PrevOut *prevOutData `json:"prevout"` +} + +type prevOutData struct { + ScriptPubKey spkData `json:"scriptPubKey"` +} + +type voutData struct { + ScriptPubKey spkData `json:"scriptPubKey"` +} + +type spkData struct { + Address string `json:"address"` + Hex string `json:"hex"` +} + +// AddressInfo is the msgpack-encoded value stored per address. New fields can +// be added at the end; old decoders will ignore unknown fields. +type AddressInfo struct { + TxCount int `msgpack:"tx_count"` +} + +type addrEntry struct { + addr string + txCount int +} + +func main() { + flag.Parse() + + var d, err = bbolt.Open(*dbPath, 0600, nil) + if err != nil { + logging.Fatal("open database: %v", err) + } + defer d.Close() + if err := addrindex.Init(d); err != nil { + logging.Fatal("init addrindex: %v", err) + } + // ensure the addresses bucket exists + if err := d.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(addressesBucket) + return err + }); err != nil { + logging.Fatal("create addresses bucket: %v", err) + } + + if *coreURL == "" { + logging.Fatal("Bitcoin Core RPC (-core-url) is required") + } + var rpc, rpcErr = newRPCClient(*coreURL, *coreUser, *corePass, *coreCookie) + if rpcErr != nil { + logging.Fatal("RPC client: %v", rpcErr) + } + + var ctx = context.Background() + var tctx, tcancel = context.WithTimeout(ctx, 15*time.Second) + var tip, tipErr = rpc.getBlockCount(tctx) + tcancel() + if tipErr != nil { + logging.Fatal("get tip: %v", tipErr) + } + + var began = time.Now() + const numWorkers = 16 + const batchSize = 1000 + + var processed atomic.Int64 + + // collector receives (addr, txCount) from workers, deduplicates, and + // flushes to bbolt in batches of batchSize. + var entries = make(chan addrEntry, 10000) + var collectorDone = make(chan struct{}) + go func() { + defer close(collectorDone) + var seen = make(map[string]bool) + var batch []addrEntry + var totalWritten int64 + + flush := func() { + if len(batch) == 0 { return } + if err := d.Update(func(tx *bbolt.Tx) error { + var b = tx.Bucket(addressesBucket) + for _, e := range batch { + if b.Get([]byte(e.addr)) != nil { continue } + var info = AddressInfo{TxCount: e.txCount} + var val, err = msgpack.Marshal(info) + if err != nil { return err } + if err := b.Put([]byte(e.addr), val); err != nil { return err } + } + return nil + }); err != nil { + fmt.Fprintf(os.Stderr, "\nflush error: %v\n", err) + } + totalWritten += int64(len(batch)) + batch = batch[:0] + seen = make(map[string]bool) + } + + for e := range entries { + if seen[e.addr] { continue } + seen[e.addr] = true + batch = append(batch, e) + if len(batch) >= batchSize { + flush() + } + } + flush() // final partial batch + fmt.Fprintf(os.Stderr, "collector finished: %d addresses written\n", totalWritten) + }() + + // progress reporter + var progressDone = make(chan struct{}) + go func() { + var ticker = time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + var h = processed.Load() + var pct float64 + if tip > 0 { + pct = float64(h) / float64(tip) * 100 + } + fmt.Printf("\rprocessed %d / %d (%.0f%%)", h, tip, pct) + case <-progressDone: + return + } + } + }() + + // worker pool + var heights = make(chan int64, numWorkers*2) + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for h := range heights { + var bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) + var hash, hashErr = rpc.getBlockHash(bctx, h) + bcancel() + if hashErr != nil { + fmt.Fprintf(os.Stderr, "\nblock %d hash: %v\n", h, hashErr) + processed.Add(1) + continue + } + + bctx, bcancel = context.WithTimeout(ctx, 60*time.Second) + var blk, blkErr = rpc.getBlockVerbose(bctx, hash) + bcancel() + if blkErr != nil { + fmt.Fprintf(os.Stderr, "\nblock %d: %v\n", h, blkErr) + processed.Add(1) + continue + } + + // collect unique addresses from this block + var seen = make(map[string]string) // address → scriptHex + for _, tx := range blk.Tx { + for _, vin := range tx.Vin { + if vin.Coinbase != "" { continue } + if vin.PrevOut != nil && vin.PrevOut.ScriptPubKey.Address != "" { + seen[vin.PrevOut.ScriptPubKey.Address] = vin.PrevOut.ScriptPubKey.Hex + } + } + for _, vout := range tx.Vout { + if vout.ScriptPubKey.Address != "" { + seen[vout.ScriptPubKey.Address] = vout.ScriptPubKey.Hex + } + } + } + + for addr, scriptHex := range seen { + var script, _ = hex.DecodeString(scriptHex) + var touches, _ = addrindex.Lookup(script, 1000000000) + var cnt = len(touches) + entries <- addrEntry{addr: addr, txCount: cnt} + } + processed.Add(1) + } + }() + } + + for h := int64(0); h <= tip; h++ { heights <- h } + close(heights) + wg.Wait() + close(progressDone) + close(entries) // signal collector to flush remaining and exit + <-collectorDone + + fmt.Printf("\rprocessed %d / %d (100%%) in %s\n", tip, tip, time.Since(began).Round(time.Second)) +} From f081a0eec14ae560bcc09acc4c4a38782478f883 Mon Sep 17 00:00:00 2001 From: pin2t Date: Thu, 30 Jul 2026 09:58:29 +0500 Subject: [PATCH 11/11] tools/dbui: add standalone database web UI tool --- tools/dbui/main.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tools/dbui/main.go diff --git a/tools/dbui/main.go b/tools/dbui/main.go new file mode 100644 index 0000000..eb6fdcf --- /dev/null +++ b/tools/dbui/main.go @@ -0,0 +1,51 @@ +// Command dbui starts the database admin web UI as a standalone process, +// serving the same interface the main bot exposes under -dbui-listen. Use it +// when you need to inspect or edit the bbolt database without running the bot. +package main + +import "context" +import "flag" +import "fmt" +import "os" +import "os/signal" +import "syscall" +import "time" + +import "go.etcd.io/bbolt" +import "bitnsbot/dbui" +import "bitnsbot/logging" + +var dbPath = flag.String("db", "", "path to the bbolt watches database (required)") +var listenAddr = flag.String("listen", "", "listen address, e.g. 127.0.0.1:8090 (required; bind to localhost only — the UI can write any bucket)") + +func main() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + if *dbPath == "" { + logging.Fatal("-db is required") + } + if *listenAddr == "" { + logging.Fatal("-listen is required") + } + db, err := bbolt.Open(*dbPath, 0600, nil) + if err != nil { + logging.Fatal("open %s: %v", *dbPath, err) + } + var srv = dbui.Start(db, *listenAddr) + var ctx, stop = signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + <-ctx.Done() + stop() + logging.Status("shutting down") + var sdCtx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := srv.Shutdown(sdCtx); err != nil { + logging.Err("database UI shutdown: %v", err) + } + if err := db.Close(); err != nil { + logging.Err("close database: %v", err) + } +}