From 0e40d13bc85dcfe93224b44517eb022011d9f82c Mon Sep 17 00:00:00 2001 From: lani_karrot Date: Fri, 7 Aug 2026 15:31:03 +0900 Subject: [PATCH] feat: add go bloom filter characterization --- README.md | 2 +- go/bloom_filter_accuracy_profile.go | 126 ++++++++++++++++++++++++ go/bloom_filter_space_profile.go | 125 +++++++++++++++++++++++ go/bloom_filter_update_speed_profile.go | 120 ++++++++++++++++++++++ go/filter_utils.go | 106 ++++++++++++++++++++ go/go.mod | 7 +- go/go.sum | 14 ++- go/main.go | 53 ++++++++++ go/main_test.go | 12 +++ 9 files changed, 553 insertions(+), 12 deletions(-) create mode 100644 go/bloom_filter_accuracy_profile.go create mode 100644 go/bloom_filter_space_profile.go create mode 100644 go/bloom_filter_update_speed_profile.go create mode 100644 go/filter_utils.go diff --git a/README.md b/README.md index 4b1b587..ddff550 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ This will compile and link the C++ characterization executable. ## Build Instructions (Go) ### Dependencies -* Go 1.24.9 or later is required to compile the Go code. +* Go 1.25.0 or later is required to compile the Go code. ### Build * The project uses Go modules, so you can build the project by running the following command: diff --git a/go/bloom_filter_accuracy_profile.go b/go/bloom_filter_accuracy_profile.go new file mode 100644 index 0000000..324f63a --- /dev/null +++ b/go/bloom_filter_accuracy_profile.go @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "math" + "strings" + + "github.com/apache/datasketches-go/filters" +) + +type BloomFilterAccuracyProfile struct { + config filterJobConfig + + sketch filters.BloomFilter + filterLengthBits uint64 + numItemsInserted uint64 + vIn uint64 +} + +func MustNewBloomFilterAccuracyProfile(cfg filterJobConfig) *BloomFilterAccuracyProfile { + return &BloomFilterAccuracyProfile{ + config: cfg, + numItemsInserted: cfg.numItemsInserted(), + vIn: 1, + } +} + +func (p *BloomFilterAccuracyProfile) run() { + fmt.Println(p.getHeader()) + + numQueries := uint64(1) << (p.config.minNumHashes + 1) + + sb := &strings.Builder{} + for nh := p.config.minNumHashes; nh <= p.config.maxNumHashes; nh++ { + fpr := 0.0 + filterNumBits := uint64(0) + + numTrials := p.config.getNumTrials(nh) + for t := 0; t < numTrials; t++ { + fpr += p.doTrial(nh, numQueries) + filterNumBits += p.getFilterLengthBits() + } + fpr /= float64(numTrials) + filterNumBits /= uint64(numTrials) + + p.process(nh, fpr, filterNumBits, numQueries, numTrials, sb) + fmt.Println(sb.String()) + + numQueries = pwr2SeriesNext(p.config.tppo, uint64(1)<<(nh+1)) + } +} + +func (p *BloomFilterAccuracyProfile) doTrial(numHashes int, numQueries uint64) float64 { + p.filterLengthBits = uint64(float64(uint64(numHashes)*p.numItemsInserted) / math.Ln2) + + sketch, err := filters.NewBloomFilterBySize(p.filterLengthBits, uint16(numHashes)) + if err != nil { + panic(err) + } + p.sketch = sketch + + for i := uint64(0); i < p.numItemsInserted; i++ { + p.vIn++ + if err := p.sketch.UpdateUInt64(p.vIn); err != nil { + panic(err) + } + } + + numFalsePositive := uint64(0) + for i := uint64(0); i < numQueries; i++ { + p.vIn++ + if p.sketch.QueryUInt64(p.vIn) { + numFalsePositive++ + } + } + return float64(numFalsePositive) / float64(numQueries) +} + +func (p *BloomFilterAccuracyProfile) getFilterLengthBits() uint64 { + return p.sketch.Capacity() +} + +func (p *BloomFilterAccuracyProfile) getBitsPerEntry(numHashes int) int { + return int(float64(numHashes) / math.Ln2) +} + +func (p *BloomFilterAccuracyProfile) getHeader() string { + return strings.Join([]string{ + "numHashes", + "FPR", + "filterSizeBits", + "numQueryPoints", + "numTrials", + }, "\t") +} + +func (p *BloomFilterAccuracyProfile) process(numHashes int, falsePositiveRate float64, + filterSizeBits, numQueryPoints uint64, numTrials int, sb *strings.Builder) { + sb.Reset() + sb.WriteString(fmt.Sprintf("%d", numHashes)) + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%.5e", falsePositiveRate)) + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%d", filterSizeBits)) + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%d", numQueryPoints)) + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%d", numTrials)) +} diff --git a/go/bloom_filter_space_profile.go b/go/bloom_filter_space_profile.go new file mode 100644 index 0000000..876f58a --- /dev/null +++ b/go/bloom_filter_space_profile.go @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/datasketches-go/filters" +) + +type BloomFilterSpaceProfile struct { + config filterSpaceJobConfig + + vIn uint64 +} + +func MustNewBloomFilterSpaceProfile(cfg filterSpaceJobConfig) *BloomFilterSpaceProfile { + return &BloomFilterSpaceProfile{config: cfg} +} + +type spaceTrialResults struct { + filterSizeBits uint64 + measuredFPR float64 + numHashes uint16 +} + +func (p *BloomFilterSpaceProfile) run() { + fmt.Println(p.getHeader()) + + maxU := uint64(1) << p.config.lgMaxU + inputCardinality := pwr2SeriesNext(p.config.uppo, uint64(1)<= 1", p.config.numHashesDelta, numHashes, inputCardinality)) + } + + sketch, err := filters.NewBloomFilterBySize(numBits, uint16(numHashes), + filters.WithSeed(p.config.seed)) + if err != nil { + panic(err) + } + + numQueries := pwr2SeriesNext(p.config.tppo, uint64(1)< 0; u-- { + p.vIn++ + p.sketch.UpdateUInt64(p.vIn) + } + elapsed := time.Since(start) + + return float64(elapsed.Nanoseconds()) / float64(uPerTrial) +} + +func (p *BloomFilterUpdateSpeedProfile) getHeader() string { + cols := []string{"InU", "Trials", "nS/Set"} + if p.config.numSketches > 1 { + cols = append(cols, "nS/Sketch") + } + return strings.Join(cols, "\t") +} + +func (p *BloomFilterUpdateSpeedProfile) process(meanUpdateTimePerSetNanoSec float64, + trials int, uPerTrial uint64, sb *strings.Builder) { + sb.Reset() + sb.WriteString(fmt.Sprintf("%d", uPerTrial)) + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%d", trials)) + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%e", meanUpdateTimePerSetNanoSec)) + if p.config.numSketches > 1 { + sb.WriteString("\t") + sb.WriteString(fmt.Sprintf("%e", meanUpdateTimePerSetNanoSec/float64(p.config.numSketches))) + } +} diff --git a/go/filter_utils.go b/go/filter_utils.go new file mode 100644 index 0000000..94485f7 --- /dev/null +++ b/go/filter_utils.go @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import "math" + +type filterJobConfig struct { + lgU int + capacity float64 + + lgMinT int + lgMaxT int + tppo int + + lgMinBpU int + lgMaxBpU int + + minNumHashes int + maxNumHashes int +} + +func (c filterJobConfig) numItemsInserted() uint64 { + return uint64(math.Round(c.capacity * float64(uint64(1)<= maxBpU { + return minT + } + // Negative slope: trials decay as the work per trial grows. + slope := float64(lgMaxT-lgMinT) / float64(lgMinBpU-lgMaxBpU) + lgX := math.Log(x) / math.Ln2 + lgTrials := slope*(lgX-float64(lgMinBpU)) + float64(lgMaxT) + return int(math.Pow(2.0, lgTrials)) +} diff --git a/go/go.mod b/go/go.mod index dbd7701..4d41e23 100644 --- a/go/go.mod +++ b/go/go.mod @@ -16,11 +16,12 @@ // module github.com/apache/datasketches-characterization/datasketches-characterization-go -go 1.24.11 +go 1.25.0 -require github.com/apache/datasketches-go v0.0.0-20260117014825-fabb7290e16c +require github.com/apache/datasketches-go v0.2.0 require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 // indirect ) diff --git a/go/go.sum b/go/go.sum index f8e516e..9fcf4a8 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,11 +1,7 @@ -github.com/apache/datasketches-go v0.0.0-20251119134622-22517a622447 h1:9B5BDC0HxnZiunOdZVSqxN2yTtM3agAmrZVeBD9DwoA= -github.com/apache/datasketches-go v0.0.0-20251119134622-22517a622447/go.mod h1:4FkC6sbeiSlLSW/OwrtiTfwj01JYf9AK7DlENi9IIzg= -github.com/apache/datasketches-go v0.0.0-20260112141520-e1cb959c71df h1:aNlsKI1eBiWAiUzJ/496C0dergqspcnBPBy68oO4K9s= -github.com/apache/datasketches-go v0.0.0-20260112141520-e1cb959c71df/go.mod h1:s+dd951Fa5Xk8BV/jy2+hm38Ab4bJ5vN1DNB1eV7kPU= -github.com/apache/datasketches-go v0.0.0-20260117014825-fabb7290e16c h1:vTZp0e8BAIpG+81agb9khH7+FdSybSrxIkYEXC9gy9U= -github.com/apache/datasketches-go v0.0.0-20260117014825-fabb7290e16c/go.mod h1:s+dd951Fa5Xk8BV/jy2+hm38Ab4bJ5vN1DNB1eV7kPU= -github.com/apache/datasketches-go v0.1.0-RC1 h1:4M/7NdXhh4TgefHPzEmikwTnsmXJ0NCsvKvZLgybf0Q= -github.com/apache/datasketches-go v0.1.0-RC1/go.mod h1:s+dd951Fa5Xk8BV/jy2+hm38Ab4bJ5vN1DNB1eV7kPU= +github.com/apache/datasketches-go v0.2.0 h1:whPblZiipxvN1+tiU3t4B8wnrIZRMmTN6oJyzhNxApA= +github.com/apache/datasketches-go v0.2.0/go.mod h1:2uFqFMsu21griPfN2rrK47rYO6XnJs4x+GvkrDwRbt8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -16,5 +12,7 @@ github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= +golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/main.go b/go/main.go index 73301f2..6bc1d2b 100644 --- a/go/main.go +++ b/go/main.go @@ -266,6 +266,59 @@ var ( numSketches: 32, }, ), + "bloom_filter_accuracy_profile": MustNewBloomFilterAccuracyProfile( + filterJobConfig{ + lgU: 20, + capacity: 0.8, + + lgMinT: 0, + lgMaxT: 0, + tppo: 1, + + lgMinBpU: 1, + lgMaxBpU: 5, + + minNumHashes: 4, + maxNumHashes: 24, + }, + ), + "bloom_filter_update_speed_profile": MustNewBloomFilterUpdateSpeedProfile( + filterSpeedJobConfig{ + lgMinU: 0, + lgMaxU: 20, + uppo: 16, + + lgMinT: 6, + lgMaxT: 12, + + lgMinBpU: 4, + lgMaxBpU: 20, + + numSketches: 1, + + numBits: 16777216, + numHashes: 16, + }, + ), + "bloom_filter_space_profile": MustNewBloomFilterSpaceProfile( + filterSpaceJobConfig{ + targetFpp: 1e-3, + + lgMinU: 0, + lgMaxU: 20, + uppo: 10, + + lgMinT: 10, + lgMaxT: 14, + tppo: 1, + + lgMinBpU: 1, + lgMaxBpU: 5, + + numHashesDelta: -4, + seed: 348675132, + }, + ), } ) diff --git a/go/main_test.go b/go/main_test.go index 588b608..c9ded31 100644 --- a/go/main_test.go +++ b/go/main_test.go @@ -84,3 +84,15 @@ func TestTDigestDoubleUpdateSpeedRunner(t *testing.T) { func TestTDigestDoubleMergeSpeedRunner(t *testing.T) { jobs["tdigest_double_merge_speed_profile"].run() } + +func TestBloomFilterAccuracyRunner(t *testing.T) { + jobs["bloom_filter_accuracy_profile"].run() +} + +func TestBloomFilterUpdateSpeedRunner(t *testing.T) { + jobs["bloom_filter_update_speed_profile"].run() +} + +func TestBloomFilterSpaceRunner(t *testing.T) { + jobs["bloom_filter_space_profile"].run() +}