diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 9c18b52e..035b14ed 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -206,6 +206,20 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int { bitset = bitmap[base] continue } + if decodeWithTrailingZeros { + if len(buf)-n >= 64 { + for bitset != 0 { + buf[n] = uint32((base*64)+bits.TrailingZeros64(bitset)) | hs + n++ + bitset &= bitset - 1 + } + continue + } + buf[n] = uint32((base*64)+bits.TrailingZeros64(bitset)) | hs + n++ + bitset &= bitset - 1 + continue + } t := bitset & -bitset buf[n] = uint32(((base * 64) + bits.OnesCount64(t-1))) | hs n = n + 1 @@ -234,6 +248,20 @@ func (bcmi *bitmapContainerManyIterator) nextMany64(hs uint64, buf []uint64) int bitset = bcmi.ptr.bitmap[base] continue } + if decodeWithTrailingZeros { + if len(buf)-n >= 64 { + for bitset != 0 { + buf[n] = uint64((base*64)+bits.TrailingZeros64(bitset)) | hs + n++ + bitset &= bitset - 1 + } + continue + } + buf[n] = uint64((base*64)+bits.TrailingZeros64(bitset)) | hs + n++ + bitset &= bitset - 1 + continue + } t := bitset & -bitset buf[n] = uint64(((base * 64) + bits.OnesCount64(t-1))) | hs n = n + 1 diff --git a/decodewords_arm64.go b/decodewords_arm64.go new file mode 100644 index 00000000..abd9f1a3 --- /dev/null +++ b/decodewords_arm64.go @@ -0,0 +1,8 @@ +package roaring + +// decodeWithTrailingZeros selects the bitmapContainerManyIterator decode loop +// built on bits.TrailingZeros64 and whole-word draining. arm64 has no scalar +// popcount, so bits.OnesCount64 goes through the vector unit for every set +// bit. The other targets keep the original loop: on amd64 the result depends +// on the microarchitecture and on GOAMD64, and did not consistently improve. +const decodeWithTrailingZeros = true diff --git a/decodewords_generic.go b/decodewords_generic.go new file mode 100644 index 00000000..6d0679c6 --- /dev/null +++ b/decodewords_generic.go @@ -0,0 +1,6 @@ +//go:build !arm64 + +package roaring + +// decodeWithTrailingZeros is documented in decodewords_arm64.go. +const decodeWithTrailingZeros = false diff --git a/manyiterator_bitmap_test.go b/manyiterator_bitmap_test.go index da3ae938..05b096b7 100644 --- a/manyiterator_bitmap_test.go +++ b/manyiterator_bitmap_test.go @@ -51,6 +51,62 @@ func TestBitmapContainerNextManyBufferSizes(t *testing.T) { } } +func TestBitmapContainerNextMany64BufferSizes(t *testing.T) { + const high = uint64(0x1234500000000) + r := rand.New(rand.NewSource(13)) + for _, cardinality := range []int{1, 63, 64, 65, 4096, 30000, 65535, 65536} { + bc := newBitmapContainer() + for bc.cardinality < cardinality { + bc.iadd(uint16(r.Intn(65536))) + } + + want := make([]uint64, 0, cardinality) + for k, w := range bc.bitmap { + for w != 0 { + want = append(want, high|uint64(k*64+bits.TrailingZeros64(w))) + w &= w - 1 + } + } + + for _, bufSize := range []int{1, 2, 63, 64, 65, 100, 127, 128, 129, 1000, 4096, 65536} { + it := bc.getManyIterator() + buf := make([]uint64, bufSize) + got := make([]uint64, 0, cardinality) + for { + n := it.nextMany64(high, buf) + if n == 0 { + break + } + if n > bufSize { + t.Fatalf("card=%d buf=%d: returned %d, more than the buffer holds", cardinality, bufSize, n) + } + got = append(got, buf[:n]...) + } + if len(got) != len(want) { + t.Fatalf("card=%d buf=%d: got %d values, want %d", cardinality, bufSize, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("card=%d buf=%d: at %d got %d want %d", cardinality, bufSize, i, got[i], want[i]) + } + } + } + } +} + +func BenchmarkBitmapContainerNextMany64(b *testing.B) { + bc := fillArrayTestBitmap(32768, 32769) + buf := make([]uint64, 4096) + var sink int + for b.Loop() { + it := bc.getManyIterator() + for n := it.nextMany64(0x50000, buf); n != 0; n = it.nextMany64(0x50000, buf) { + sink += n + } + } + _ = sink +} + func benchBitmapNextMany(b *testing.B, cardinality, bufSize int) { r := rand.New(rand.NewSource(12)) bc := newBitmapContainer()