diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 3cefb8d9..e1959d33 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -346,6 +346,11 @@ func bitmapEquals(a, b []uint64) bool { // is the array-to-bitmap conversion point and leaves a comfortable margin. const bitmapContainerVectorFillMinCardinality = 4096 +// arrayVectorFillMinCardinality is where decoding a whole key with the vector +// fill beats decoding only the words that hold the values. Measured crossover +// is near 1000 values. +const arrayVectorFillMinCardinality = 1024 + func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { if useVectorFill && bc.cardinality >= bitmapContainerVectorFillMinCardinality { return fillLeastSignificant16bitsVector(bc.bitmap, x, i, mask) @@ -837,7 +842,7 @@ func (bc *bitmapContainer) iand(a container) container { return bc.iandBitmap(x) case *runContainer16: if x.isFull() { - return bc.clone() + return bc } return bc.iandRun16(x) } @@ -845,8 +850,20 @@ func (bc *bitmapContainer) iand(a container) container { } func (bc *bitmapContainer) iandRun16(rc *runContainer16) container { - rcb := newBitmapContainerFromRun(rc) - return bc.iandBitmap(rcb) + if rc.andInScratch() { + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(scratch[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return bc.iandBitmap(&bitmapContainer{bitmap: scratch[:]}) + } + card := rc.andBitmapContainerCardinality(bc) + if card <= arrayDefaultMaxSize { + return rc.andBitmapArray(bc, card) + } + clearBitmapGaps(bc.bitmap, rc.iv, 0, maxCapacity) + bc.cardinality = card + return bc } func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { @@ -856,15 +873,13 @@ func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { answer := newArrayContainerCapacity(len(value2.content)) - answer.content = answer.content[:cap(answer.content)] - c := value2.getCardinality() + out, bitmap := answer.content[:cap(answer.content)], bc.bitmap pos := 0 - for k := 0; k < c; k++ { - v := value2.content[k] - answer.content[pos] = v - pos += int(bc.bitValue(v)) + for _, v := range value2.content { + out[pos] = v + pos += int((bitmap[v>>6] >> (v & 63)) & 1) } - answer.content = answer.content[:pos] + answer.content = out[:pos] return answer } @@ -878,6 +893,26 @@ func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int { return pos } +func (bc *bitmapContainer) intersectsRange(start, end uint) bool { + if start >= end { + return false + } + firstword, endword := start/64, (end-1)/64 + lo, hi := ^uint64(0)<<(start%64), ^uint64(0)>>((64-end)&63) + if firstword == endword { + return bc.bitmap[firstword]&lo&hi != 0 + } + if bc.bitmap[firstword]&lo != 0 || bc.bitmap[endword]&hi != 0 { + return true + } + for _, w := range bc.bitmap[firstword+1 : endword] { + if w != 0 { + return true + } + } + return false +} + func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int { if start >= end { return 0 diff --git a/fastaggregation.go b/fastaggregation.go index 3881b88a..f4d511f5 100644 --- a/fastaggregation.go +++ b/fastaggregation.go @@ -125,26 +125,342 @@ func (x1 *Bitmap) repairAfterLazy() { } } -// FastAnd computes the intersection between many bitmaps quickly -// Compared to the And function, it can take many bitmaps as input, thus saving the trouble -// of manually calling "And" many times. +// The walk covers every input, up to fastAndWide, when the driver has one +// key, and fastAndNarrow of them otherwise. +const ( + fastAndWide = 8 + fastAndNarrow = 4 +) + +// FastAnd computes the intersection between many bitmaps quickly. +// Compared to the And function, it can take many bitmaps as input. // -// Performance hints: if you have very large and tiny bitmaps, -// it may be beneficial performance-wise to put a tiny bitmap -// in first position. +// FastAnd walks the keys of the input with the fewest keys and looks each +// one up in the other inputs before allocating anything for it; inputs past +// the walk are intersected into the result one at a time. func FastAnd(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { + switch len(bitmaps) { + case 0: return NewBitmap() - } else if len(bitmaps) == 1 { + case 1: return bitmaps[0].Clone() } - answer := And(bitmaps[0], bitmaps[1]) - for _, bm := range bitmaps[2:] { - answer.And(bm) + driver, fewest := 0, bitmaps[0].highlowcontainer.size() + for i := 1; i < len(bitmaps); i++ { + if n := bitmaps[i].highlowcontainer.size(); n < fewest { + driver, fewest = i, n + } + } + dra := &bitmaps[driver].highlowcontainer + if len(bitmaps) == 2 { + // A pair over many keys is And's job; a pair on one key is walked. + if dra.size() > 1 { + return And(bitmaps[0], bitmaps[1]) + } + if dra.size() == 0 || bitmaps[1-driver].highlowcontainer.getIndex(dra.getKeyAtIndex(0)) < 0 { + return NewBitmap() + } + } + n := len(bitmaps) + if n > fastAndWide || dra.size() > 1 { + n = min(n, fastAndNarrow) + } + var pos [fastAndWide]int + var cs [fastAndWide]container + answer := NewBitmap() + survived := false // the last key passed every input, so checks are skipped +keys: + for j := 0; j < dra.size(); j++ { + key := dra.getKeyAtIndex(j) + dc := dra.getContainerAtIndex(j) + dx, isArray := dc.(*arrayContainer) + // An array driver is checked for a value it shares with each input. + check := isArray && !survived + // Full containers change nothing; the smallest array goes first. + m, small, smallLen := 0, -1, 0 + if isArray { + cs[0], m, small, smallLen = dc, 1, 0, len(dx.content) + } else if !dc.isFull() { + cs[0], m = dc, 1 + } + for i, bm := range bitmaps[:n] { + if i == driver { + continue + } + ra := &bm.highlowcontainer + p := pos[i] + if p < ra.size() && ra.getKeyAtIndex(p) < key { + p = ra.advanceUntil(key, p) + } + pos[i] = p + if p == ra.size() { + break keys + } + if ra.getKeyAtIndex(p) != key { + continue keys + } + pos[i]++ + c := ra.getContainerAtIndex(p) + if x, ok := c.(*arrayContainer); ok { + if small < 0 || len(x.content) < smallLen { + small, smallLen = m, len(x.content) + } + } else if c.isFull() { + continue + } + if check && !dc.intersects(c) { + survived = false + continue keys + } + cs[m] = c + m++ + } + c := andK(cs[:m], small) + survived = c != nil + if c != nil { + if answer.IsEmpty() { + // Room for up to 64 of the driver's remaining keys at once. + k := min(dra.size()-j, 64) + answer.highlowcontainer = roaringArray{ + keys: make([]uint16, 0, k), + containers: make([]container, 0, k), + needCopyOnWrite: make([]bool, 0, k), + } + } + answer.highlowcontainer.appendContainer(key, c, false) + } + } + for i := n; i < len(bitmaps) && !answer.IsEmpty(); i++ { + if i != driver { + answer.And(bitmaps[i]) + } } return answer } +// andK intersects the containers of one key, none of them full, and returns +// nil when the result is empty. small is the index of the smallest array in +// cs, or -1 if there is none. +func andK(cs []container, small int) container { + switch len(cs) { + case 0: + return newRunContainer16Range(0, maxCapacity-1) + case 1: + return cs[0].clone() + } + if small >= 0 { + a := cs[small].(*arrayContainer) + copy(cs[1:small+1], cs[:small]) // the rest keep their order after it + return andKArray(a, cs[1:]) + } + n := 0 + for i, c := range cs { + if _, ok := c.(*runContainer16); ok { + cs[n], cs[i] = cs[i], cs[n] + n++ + } + } + if n == len(cs) { + return andKRuns(cs) + } + if n == 1 && len(cs) == 2 { + // A single interval against one bitmap is counted in place. + if rc := cs[0].(*runContainer16); len(rc.iv) == 1 { + return rc.andBitmapCounted(cs[1].(*bitmapContainer)) + } + } + return andKBitmaps(cs[n:], cs[:n]) +} + +// andKArray narrows the array through rest in a buffer on the stack, sized +// to the array, and allocates only once a value survives every container. +func andKArray(ac *arrayContainer, rest []container) container { + if len(rest) == 1 { + return andArrayLast(ac.content, rest[0]) + } + switch n := len(ac.content); { + case n <= 64: + var buf [64]uint16 + return andKArrayInto(ac.content, rest, buf[:]) + case n <= 512: + var buf [512]uint16 + return andKArrayInto(ac.content, rest, buf[:]) + default: + var buf [arrayDefaultMaxSize]uint16 + return andKArrayInto(ac.content, rest, buf[:]) + } +} + +func andKArrayInto(vals []uint16, rest []container, buf []uint16) container { + last := len(rest) - 1 + for _, c := range rest[:last] { + n := 0 + switch c := c.(type) { + case *arrayContainer: + n = intersection2by2(vals, c.content, buf) + case *bitmapContainer: + for _, v := range vals { + buf[n] = v + n += int(c.bitValue(v)) + } + case *runContainer16: + n = andArrayInto(c.iv, vals, buf) + } + if n == 0 { + return nil + } + vals = buf[:n] + } + return andArrayLast(vals, rest[last]) +} + +// andArrayLast intersects vals with c from the first value they share, so +// nothing is allocated when there is none. +func andArrayLast(vals []uint16, c container) container { + switch c := c.(type) { + case *arrayContainer: + i, j := firstShared2by2(vals, c.content) + if i < 0 { + return nil + } + ac := newArrayContainerCapacity(min(len(vals)-i, len(c.content)-j)) + ac.content = ac.content[:intersection2by2(vals[i:], c.content[j:], ac.content[:cap(ac.content)])] + return ac + case *bitmapContainer: + for i, v := range vals { + if c.bitValue(v) != 0 { + return c.andArray(&arrayContainer{content: vals[i:]}) + } + } + case *runContainer16: + if i := c.firstInArray(vals); i >= 0 { + return c.andArray(&arrayContainer{content: vals[i:]}) + } + } + return nil +} + +// andKRuns intersects runs interval by interval, allocating only where they +// all overlap. +func andKRuns(cs []container) container { + a := cs[0].(*runContainer16) + if !a.intersects(cs[1]) { + return nil + } + if len(cs) == 2 { + return a.intersect(cs[1].(*runContainer16)).toEfficientContainer() + } + var pos [fastAndWide]int + var iv []interval16 + for { + lo, hi, end := 0, maxCapacity-1, maxCapacity-1 + for i, c := range cs { + rc := c.(*runContainer16) + if pos[i] == len(rc.iv) { + if len(iv) == 0 { + return nil + } + return (&runContainer16{iv: iv}).toEfficientContainer() + } + r := rc.iv[pos[i]] + lo, hi, end = max(lo, int(r.start)), min(hi, int(r.last())), min(end, int(r.last())) + } + if lo <= hi { + iv = append(iv, newInterval16Range(uint16(lo), uint16(hi))) + } + for i, c := range cs { // step past every interval that ends first + if int(c.(*runContainer16).iv[pos[i]].last()) == end { + pos[i]++ + } + } + } +} + +// andKChain intersects cs in order: the first two into a new container, then +// the rest into it in place, so nothing bigger than the first result is built. +func andKChain(cs []container) container { + c := cs[0].and(cs[1]) + for i := 2; i < len(cs) && !c.isEmpty(); i++ { + c = c.iand(cs[i]) + } + if c.isEmpty() { + return nil + } + return c +} + +// andBitmapsMeetWords is how many leading words are scanned for a shared bit +// before dense bitmaps are ANDed straight into the result, skipping scratch. +const andBitmapsMeetWords = 8 + +// andKBitmaps ANDs the bitmaps, masked by the runs, allocating only for a +// non-empty result. +func andKBitmaps(bms, runs []container) container { + if len(runs) == 0 && andWordsMeet(bms, andBitmapsMeetWords) { + answer := newBitmapContainer() + src := bms[0].(*bitmapContainer).bitmap + card := uint64(0) + for _, c := range bms[1:] { + card = andCardSlice(answer.bitmap, src, c.(*bitmapContainer).bitmap) + src = answer.bitmap + } + answer.cardinality = int(card) + if card <= arrayDefaultMaxSize { + return answer.toArrayContainer() + } + return answer + } + lo, hi := 0, maxCapacity-1 + gaps := false + for _, c := range runs { + rc := c.(*runContainer16) + lo, hi = max(lo, int(rc.minimum())), min(hi, int(rc.maximum())) + gaps = gaps || len(rc.iv) > 1 + } + if lo > hi { + return nil + } + first, last := lo/64, hi/64 + var scratch [bitmapContainerSize]uint64 + w := scratch[first : last+1] + src := bms[0].(*bitmapContainer).bitmap[first : last+1] + card := uint64(0) + if len(bms) == 1 { + copy(w, src) + } + for _, c := range bms[1:] { + if card = andCardSlice(w, src, c.(*bitmapContainer).bitmap[first:last+1]); card == 0 { + return nil + } + src = w + } + if len(bms) == 1 || gaps || lo%64 != 0 || (hi+1)%64 != 0 { + resetBitmapRange(scratch[:], first*64, lo) + resetBitmapRange(scratch[:], hi+1, (last+1)*64) + for _, c := range runs { + clearBitmapGaps(scratch[:], c.(*runContainer16).iv, lo, hi+1) + } + if card = popcntSlice(w); card == 0 { + return nil + } + } + return containerFromWords(scratch[:], first, last, int(card)) +} + +// andWordsMeet reports whether the bitmaps share a bit in their first n words. +func andWordsMeet(bms []container, n int) bool { + for i := 0; i < n; i++ { + w := bms[0].(*bitmapContainer).bitmap[i] + for _, c := range bms[1:] { + w &= c.(*bitmapContainer).bitmap[i] + } + if w != 0 { + return true + } + } + return false +} + // FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. // It might also be faster than calling Or repeatedly. func FastOr(bitmaps ...*Bitmap) *Bitmap { diff --git a/fastand_kway_test.go b/fastand_kway_test.go new file mode 100644 index 00000000..56fbf054 --- /dev/null +++ b/fastand_kway_test.go @@ -0,0 +1,324 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// kwayKey picks the container key for the k-th key of an input: inputs share +// their keys, but each leaves some out, the way real inputs do. +func kwayKey(rng *rand.Rand, k int) (uint64, bool) { + return uint64(k) << 16, rng.Intn(8) != 0 +} + +// kwayBitmap builds a bitmap over up to keys keys holding random values at +// the given density: arrays when sparse, bitmaps when dense. +func kwayBitmap(rng *rand.Rand, keys int, density float64) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for i := 0; i < int(density*65536); i++ { + bm.Add(uint32(base) + uint32(rng.Intn(65536))) + } + } + return bm +} + +// kwayRuns builds a run-optimized bitmap of random ranges over up to keys keys. +func kwayRuns(rng *rand.Rand, keys int) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for start := 0; start < 65536; start += 1000 + rng.Intn(3000) { + bm.AddRange(base+uint64(start), base+uint64(min(start+1+rng.Intn(800), 65536))) + } + } + bm.RunOptimize() + return bm +} + +// intervalBitmap holds one run container at key 0 made of the given +// half-open intervals. +func intervalBitmap(t *testing.T, intervals [][2]uint64) *Bitmap { + t.Helper() + bm := New() + for _, iv := range intervals { + bm.AddRange(iv[0], iv[1]) + } + bm.RunOptimize() + require.Equal(t, uint64(1), bm.Stats().RunContainers, "%v", intervals) + return bm +} + +func randomIntervals(rng *rand.Rand) [][2]uint64 { + var out [][2]uint64 + pos := uint64(rng.Intn(64)) + for n := 1 + rng.Intn(40); n > 0 && pos < 65536; n-- { + length := min(uint64(4+rng.Intn(3000)), 65536-pos) + out = append(out, [2]uint64{pos, pos + length}) + pos += length + uint64(rng.Intn(200)) + } + return out +} + +// requireFastAnd checks FastAnd against the pairwise definition, that the +// result is valid, and that it shares no storage with its inputs, and +// reports whether the result is empty. +func requireFastAnd(t *testing.T, inputs ...*Bitmap) bool { + t.Helper() + before := make([]*Bitmap, len(inputs)) + for i, in := range inputs { + before[i] = in.Clone() + } + want := pairwiseAnd(inputs) + got := FastAnd(inputs...) + require.NoError(t, got.Validate()) + require.True(t, got.Equals(want), "%d values, want %d", got.GetCardinality(), want.GetCardinality()) + empty := got.IsEmpty() + got.Flip(0, 1<<21) // rewrites every container the inputs could share + for i, in := range inputs { + require.True(t, in.Equals(before[i]), "input %d was modified through the result", i) + } + return empty +} + +func TestFastAndMatchesPairwise(t *testing.T) { + rng := rand.New(rand.NewSource(3)) + densities := []float64{0.0005, 0.01, 0.05, 0.3, 0.8, 0.98} + whole := New() + whole.AddRange(0, 4<<16) + nonEmpty := 0 + for trial := 0; trial < 400; trial++ { + inputs := make([]*Bitmap, 3+rng.Intn(10)) + for i := range inputs { + switch rng.Intn(6) { + case 0: + inputs[i] = kwayRuns(rng, 1+rng.Intn(4)) + case 1: + inputs[i] = intervalBitmap(t, randomIntervals(rng)) + case 2: + inputs[i] = whole + default: + inputs[i] = kwayBitmap(rng, 1+rng.Intn(4), densities[rng.Intn(len(densities))]) + } + } + if !requireFastAnd(t, inputs...) { + nonEmpty++ + } + } + require.Greater(t, nonEmpty, 100, "most trials must reach the intersection, not stop at a missing key") +} + +func TestFastAndEdges(t *testing.T) { + rng := rand.New(rand.NewSource(12)) + dense := func() *Bitmap { + bm := New() + for range 32768 { + bm.Add(uint32(rng.Intn(65536))) + } + bm.AddRange(0, 256) // so a mask's edges always have values to drop + return bm + } + full := intervalBitmap(t, [][2]uint64{{0, 65536}}) + // A range ending on a word boundary on one side only is masked on the other. + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{64, 100}}), dense()) + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{70, 128}}), dense()) + // Two ranges spanning whole words still mask the gap between them. + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{0, 64}, {128, 192}}), dense()) + // A range narrower than a many-interval run's span clips it. + var manyShort [][2]uint64 + for p := uint64(0); p < 65536; p += 64 { + manyShort = append(manyShort, [2]uint64{p, p + 8}) + } + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{100, 40000}}), intervalBitmap(t, manyShort), dense(), dense()) + // Two ranges that each meet the bitmap but overlap only where it is + // empty, so the masked result is empty. + gap := New() + gap.Add(0) + gap.AddRange(128, 5300) + requireFastAnd(t, gap, intervalBitmap(t, [][2]uint64{{0, 128}}), intervalBitmap(t, [][2]uint64{{64, 192}})) + // Ranges that each meet the first one but have no common value end the + // key before a bitmap is read, as do disjoint runs. + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 40000}}), intervalBitmap(t, [][2]uint64{{30000, 65536}}), intervalBitmap(t, [][2]uint64{{0, 20000}}), dense()) + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 1000}}), intervalBitmap(t, [][2]uint64{{2000, 3000}}), dense()) + // Whole-key runs change nothing. + y := New() + y.AddMany([]uint32{1, 5, 9, 70000}) + require.True(t, FastAnd(y, full, y).Equals(And(y, full))) + require.True(t, FastAnd(full, full, full).Equals(full)) +} + +// singletons returns n inputs holding the values 0 to n-1, one each: the +// second input already fails the check. +func singletons(n int) []*Bitmap { + inputs := make([]*Bitmap, n) + for i := range inputs { + inputs[i] = New() + inputs[i].Add(uint32(i)) + } + return inputs +} + +// kwayTriple returns three dense inputs whose pairs all overlap but whose +// intersection is empty. +func kwayTriple() (a, b, c *Bitmap) { + a, b, c = New(), New(), New() + for id := uint32(0); id < 4<<16; id++ { + switch id % 3 { + case 0: + a.Add(id) + b.Add(id) + case 1: + b.Add(id) + c.Add(id) + case 2: + a.Add(id) + c.Add(id) + } + } + return a, b, c +} + +func TestFastAndEmptyAllocatesNoContainer(t *testing.T) { + a, b, c := kwayTriple() + require.True(t, a.Intersects(b) && b.Intersects(c) && a.Intersects(c)) + require.True(t, FastAnd(a, b, c).IsEmpty()) + // Only the answer bitmap; never a container. + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(a, b, c) }), 1.0) + + mask := intervalBitmap(t, [][2]uint64{{0, 65536}}) + evens, odds := New(), New() + for v := uint32(0); v < 65536; v += 2 { + evens.Add(v) + odds.Add(v + 1) + } + require.True(t, FastAnd(mask, evens, odds).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(mask, evens, odds) }), 1.0) + + // Nothing is allocated for the inputs past an empty result. + inputs := singletons(1000) + require.True(t, FastAnd(inputs...).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(inputs...) }), 1.0) + + // Inputs that pairwise intersect but are jointly empty, including dense + // ones that only the fourth or fifth input empties, allocate nothing. + partial := intervalBitmap(t, [][2]uint64{{100, 40000}}) + cycle := func(a, b uint32) *Bitmap { + bm := New() + for v := uint32(0); v < 65536; v += 32 { + bm.AddMany([]uint32{v + a, v + b}) + } + return bm + } + band := func(lo, hi uint32) *Bitmap { + bm := New() + bm.AddRange(uint64(lo), uint64(hi)) + return bm + } + a1 := band(0, 50000) + a2 := band(10000, 60000) + a3 := band(20000, 65536) // a1,a2,a3 overlap in [20000,50000) + a4 := band(0, 20000) // meets each of a1..a3 but empties the four together + a5 := band(55000, 65536) // for a five-input version, empties at the fifth + for _, inputs := range [][]*Bitmap{ + {mask, cycle(0, 1), cycle(1, 2), cycle(2, 0)}, + {partial, cycle(0, 1), evens, cycle(1, 2)}, + {mask, a1, a2, a3, a4}, + {mask, a1, a2, a3, band(0, 60000), a5}, + {partial, a1, a2, a3, a4}, + {partial, band(50000, 65536)}, // a pair whose term lies past the range + } { + require.True(t, FastAnd(inputs...).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(inputs...) }), 1.0) + } +} + +// randomKey holds card random values on key. +func randomKey(rng *rand.Rand, card int, key uint32) *Bitmap { + bm := New() + for bm.GetCardinality() < uint64(card) { + bm.Add(key<<16 | uint32(rng.Intn(65536))) + } + return bm +} + +func pairwiseAnd(bms []*Bitmap) *Bitmap { + r := And(bms[0], bms[1]) + for _, bm := range bms[2:] { + r.And(bm) + } + return r +} + +// BenchmarkFastAndShapes runs FastAnd against pairwise And on the same +// inputs, one shape per container pairing plus many-input shapes; the first +// seven add a whole-key range like a query window. +func BenchmarkFastAndShapes(b *testing.B) { + rng := rand.New(rand.NewSource(9)) + window := New() + window.AddRange(0, 65536) + wide := New() + wide.AddRange(0, 8<<16) + spread := func(card int) *Bitmap { + bm := New() + for k := uint32(0); k < 8; k++ { + bm.Or(randomKey(rng, card, k)) + } + return bm + } + x, y, z := kwayTriple() + same := make([]*Bitmap, 128) + base := spread(30) + for i := range same { + same[i] = base.Clone() + } + // 1,024 inputs of 256 keys that all survive: the widest walk. + keys256 := New() + for k := uint32(0); k < 256; k++ { + for v := uint32(0); v < 64; v += 2 { + keys256.Add(k<<16 | v) + } + } + surviving := make([]*Bitmap, 1024) + for i := range surviving { + surviving[i] = keys256.Clone() + } + for _, tc := range []struct { + name string + inputs []*Bitmap + }{ + {"array30-bitmap", []*Bitmap{randomKey(rng, 30, 0), randomKey(rng, 33000, 0), window}}, + {"array4000-bitmap", []*Bitmap{randomKey(rng, 4000, 0), randomKey(rng, 33000, 0), window}}, + {"array30-runs", []*Bitmap{randomKey(rng, 30, 0), runBitmap(250, 16, 32), window}}, + {"array4000-runs", []*Bitmap{randomKey(rng, 4000, 0), runBitmap(250, 16, 32), window}}, + {"bitmap-bitmap", []*Bitmap{randomKey(rng, 33000, 0), randomKey(rng, 33000, 0), window}}, + {"runs-bitmap", []*Bitmap{runBitmap(250, 16, 32), randomKey(rng, 33000, 0), window}}, + {"runs-runs", []*Bitmap{runBitmap(250, 16, 32), runBitmap(1875, 16, 32), window}}, + {"jointly-empty", []*Bitmap{x, y, z}}, + {"eight-keys", []*Bitmap{spread(30), spread(33000), wide}}, + {"singletons-10000", singletons(10000)}, + {"equal-x128", same}, + {"surviving-1024x256", surviving}, + } { + b.Run(tc.name+"/fastand", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = FastAnd(tc.inputs...) + } + }) + b.Run(tc.name+"/pairwise", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = pairwiseAnd(tc.inputs) + } + }) + } +} diff --git a/parallel.go b/parallel.go index 01ba856f..9703912f 100644 --- a/parallel.go +++ b/parallel.go @@ -283,18 +283,8 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { andFunc := func() { // Assumes only structs with >=2 containers are passed for input := range inputChan { - c := input.containers[0].and(input.containers[1]) - for _, next := range input.containers[2:] { - if c.isEmpty() { - break - } - c = c.iand(next) - } - - // Send a nil explicitly if the result of the intersection is an empty container - if c.isEmpty() { - c = nil - } + // A nil result tells the receiver the intersection is empty. + c := andKChain(input.containers) kx := keyedContainer{ input.key, diff --git a/real_data_benchmark_test.go b/real_data_benchmark_test.go index 8b02198b..e3f67198 100644 --- a/real_data_benchmark_test.go +++ b/real_data_benchmark_test.go @@ -187,6 +187,12 @@ func BenchmarkRealDataFastOr(b *testing.B) { }) } +func BenchmarkRealDataFastAnd(b *testing.B) { + benchmarkRealDataAggregate(b, func(bitmaps []*Bitmap) uint64 { + return FastAnd(bitmaps...).GetCardinality() + }) +} + func BenchmarkRealDataOrInPlace(b *testing.B) { benchmarkRealDataAggregate(b, func(bitmaps []*Bitmap) uint64 { tot := uint64(0) diff --git a/runbitmap_and_test.go b/runbitmap_and_test.go new file mode 100644 index 00000000..bb3f5abf --- /dev/null +++ b/runbitmap_and_test.go @@ -0,0 +1,274 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// randomRun builds a canonical run container: up to n sorted intervals of +// at most maxLen values separated by gaps of at most maxGap, optionally +// touching 0 and 65535. +func randomRun(rng *rand.Rand, n, maxLen, maxGap int, touchEdges bool) *runContainer16 { + rc := &runContainer16{} + pos := 0 + if !touchEdges { + pos = rng.Intn(100) + } + for i := 0; i < n && pos < 65536; i++ { + length := min(1+rng.Intn(1+rng.Intn(maxLen)), 65536-pos) + rc.iv = append(rc.iv, interval16{start: uint16(pos), length: uint16(length - 1)}) + pos += length + 1 + rng.Intn(maxGap) + } + if last := int(rc.maximum()); touchEdges && last != 65535 { + if start := 65535 - rng.Intn(500); last+2 <= start { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(65535 - start)}) + } + } + return rc +} + +// randomBitmapContainer ANDs ands random words per position, so the density +// is 1/2^ands. +func randomBitmapContainer(rng *rand.Rand, ands int) *bitmapContainer { + bc := newBitmapContainer() + for i := range bc.bitmap { + w := rng.Uint64() + for k := 1; k < ands; k++ { + w &= rng.Uint64() + } + bc.bitmap[i] = w + } + bc.computeCardinality() + return bc +} + +func requireAndResult(t *testing.T, want, got container) { + t.Helper() + require.True(t, want.equals(got), "want %d values, got %d", want.getCardinality(), got.getCardinality()) + _, isArray := got.(*arrayContainer) + require.Equal(t, got.getCardinality() <= arrayDefaultMaxSize, isArray, "container kind must follow its cardinality") +} + +// The reference is the old path: materialize the run, then the unchanged +// bitmap-bitmap intersection. Runs of one to hundreds of intervals check +// both the per-interval and the scratch paths, against every density. +func TestRunAndBitmapContainer(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + regular := func(stride, length int) *runContainer16 { + rc := &runContainer16{} + for start := 0; start+length <= 65536; start += stride { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(length - 1)}) + } + return rc + } + runs := []*runContainer16{regular(32, 8), regular(64, 1), regular(3, 2), regular(200, 100)} + for i := 0; i < 120; i++ { + runs = append(runs, randomRun(rng, 1+rng.Intn(40), 3000, 2000, i%5 == 0), randomRun(rng, 65+rng.Intn(400), 60, 120, false)) + } + // Only values in the gaps of regular(32, 8), so that run's result is empty. + gaps := newBitmapContainer() + for v := 0; v < 65536; v++ { + if v%32 >= 8 { + gaps.iadd(uint16(v)) + } + } + for i, rc := range runs { + for _, bc := range []*bitmapContainer{randomBitmapContainer(rng, 6), randomBitmapContainer(rng, 2), randomBitmapContainer(rng, 1), newBitmapContainerwithRange(0, 65535), gaps} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.and(rc)) + requireAndResult(t, want, bc.clone().iand(rc)) + require.Equal(t, !want.isEmpty(), rc.intersects(bc), "run %d", i) + } + } +} + +func TestRunAndBitmapContainerEdges(t *testing.T) { + empty := newBitmapContainer() + full := newBitmapContainerwithRange(0, 65535) + single := &runContainer16{iv: []interval16{{start: 100, length: 9}}} + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + atThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize - 1}}} + pastThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize}}} + for _, rc := range []*runContainer16{single, edges, atThreshold, pastThreshold} { + for _, bc := range []*bitmapContainer{empty, full} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.clone().iand(rc)) + } + } + // Exactly 4096 values through the scratch path: every even value below + // 16384 against the values below 8192. + evens := &runContainer16{} + for v := 0; v < 4*arrayDefaultMaxSize; v += 2 { + evens.iv = append(evens.iv, interval16{start: uint16(v)}) + } + half := newBitmapContainerwithRange(0, 2*arrayDefaultMaxSize-1) + requireAndResult(t, newBitmapContainerFromRun(evens).andBitmap(half), evens.and(half)) +} + +// An empty intersection allocates only its empty array container, and an +// in-place And with a full run and Intersects allocate nothing; the old paths +// allocated an 8 KiB bitmap first. +func TestRunAndBitmapContainerAllocations(t *testing.T) { + rc := &runContainer16{iv: []interval16{{start: 0, length: 999}, {start: 30000, length: 999}}} + bc := newBitmapContainerwithRange(2000, 20000) + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + if !rc.and(bc).isEmpty() { + t.Fatal("expected empty") + } + }), 1.0) + // The clone costs two allocations, the empty result one. + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + _ = bc.clone().iand(rc) + }), 3.0) + full := newRunContainer16Range(0, MaxUint16) + require.Zero(t, testing.AllocsPerRun(100, func() { _ = bc.iand(full) })) + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.iand(full) })) + for _, c := range []container{bc, newArrayContainerRange(2000, 3000), &runContainer16{iv: []interval16{{start: 5000, length: 100}}}} { + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.intersects(c) }), "%T", c) + } +} + +// The reference is the old definition, !rc.and(c).isEmpty(). +func TestRunContainerIntersects(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + for trial := 0; trial < 400; trial++ { + rc := randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%7 == 0) + ac := newArrayContainer() + for range rng.Intn(300) { + ac.iadd(uint16(rng.Intn(65536))) + } + others := []container{randomBitmapContainer(rng, 1+trial%3*3), ac, randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%5 == 0)} + for _, c := range others { + want := !rc.and(c).isEmpty() + require.Equal(t, want, rc.intersects(c), "trial %d run vs %T", trial, c) + require.Equal(t, want, c.intersects(rc), "trial %d %T vs run", trial, c) + } + } + // A few values against many intervals take the search branch; the + // reference is the same definition. + many := randomRun(rng, 300, 60, 120, false) + require.Greater(t, len(many.iv), 200) + for trial := 0; trial < 200; trial++ { + few := newArrayContainer() + for range 1 + rng.Intn(6) { + few.iadd(uint16(rng.Intn(65536))) + } + want := !many.and(few).isEmpty() + require.Equal(t, want, many.intersects(few), "trial %d few vs many", trial) + require.Equal(t, want, few.intersects(many), "trial %d many vs few", trial) + } + // Edges: empty array, touching intervals, single values at both ends. + empty := newArrayContainer() + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + require.False(t, edges.intersects(empty)) + one := newArrayContainer() + one.iadd(65535) + require.True(t, edges.intersects(one)) + touching := &runContainer16{iv: []interval16{{start: 1, length: 65533}}} + require.False(t, edges.intersects(touching)) + require.True(t, edges.intersects(newBitmapContainerwithRange(0, 0))) +} + +// runBitmap holds one run container at key 0 covering n intervals of the +// given length, spaced stride apart. +func runBitmap(n, length, stride int) *Bitmap { + bm := New() + for i := 0; i < n; i++ { + bm.AddRange(uint64(i*stride), uint64(i*stride+length)) + } + bm.RunOptimize() + return bm +} + +// BenchmarkRunAndBitmap intersects a run-optimized bitmap with a dense one +// over a single key through the public API, one case per result kind, plus +// the in-place form, alone and with the copy it needs, and the Intersects test. +func BenchmarkRunAndBitmap(b *testing.B) { + rng := rand.New(rand.NewSource(7)) + short := runBitmap(20, 300, 3000) // 6000 values + long := runBitmap(4, 10000, 16000) // 40000 values + many := runBitmap(1024, 8, 64) // 8192 values in 1024 intervals + gaps, quarter, half, dense := New(), New(), New(), New() + for v := uint32(0); v < 65536; v++ { + if !short.Contains(v) && rng.Intn(2) == 0 { + gaps.Add(v) + } + if rng.Intn(4) == 0 { + quarter.Add(v) + } + if rng.Intn(2) == 0 { + half.Add(v) + } + if rng.Intn(10) != 0 { + dense.Add(v) + } + } + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"and/empty", short, gaps}, + {"and/array", short, half}, + {"and/bitmap", long, dense}, + {"and/many-intervals/array", many, quarter}, + {"and/many-intervals/bitmap", many, dense}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = And(tc.x, tc.y) + } + }) + } + for _, tc := range []struct { + name string + x *Bitmap + }{ + {"iand/bitmap", long}, + {"iand/many-intervals", many}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var xs []*Bitmap + for b.Loop() { + if len(xs) == 0 { + // Fresh copies, made outside the timer: one And turns a copy into its result. + b.StopTimer() + for range 64 { + xs = append(xs, dense.Clone()) + } + b.StartTimer() + } + xs[len(xs)-1].And(tc.x) + xs = xs[:len(xs)-1] + } + }) + b.Run("clone+"+tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + dense.Clone().And(tc.x) + } + }) + } + few := New() + few.AddMany([]uint32{100, 30000, 60000}) + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"intersects/disjoint", short, gaps}, + {"intersects/overlapping", short, half}, + {"intersects/few-values-many-intervals", many, few}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = tc.x.Intersects(tc.y) + } + }) + } +} diff --git a/runcontainer.go b/runcontainer.go index bcb762bb..456bd4e9 100644 --- a/runcontainer.go +++ b/runcontainer.go @@ -1885,10 +1885,75 @@ func (rc *runContainer16) andCardinality(a container) int { panic("unsupported container type") } -// andBitmapContainer finds the intersection of rc and b. +const runAndScratchIntervals = 64 + +// andInScratch reports whether ANDing rc with a bitmap is cheaper in a +// scratch bitmap than interval by interval: past runAndScratchIntervals +// intervals, the measured crossover, if the run covers more than 4096 values +// or the vector fill makes reading the scratch back cheap. +func (rc *runContainer16) andInScratch() bool { + return len(rc.iv) > runAndScratchIntervals && (useVectorFill || rc.getCardinality() > arrayDefaultMaxSize) +} + +// andBitmapContainer intersects rc with bc, counting before allocating. func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.andBitmap(bc) + if c := rc.andBitmapCounted(bc); c != nil { + return c + } + return newArrayContainerCapacity(0) +} + +// andBitmapCounted counts the intersection before building it, and returns +// nil, allocating nothing, when it is empty. +func (rc *runContainer16) andBitmapCounted(bc *bitmapContainer) container { + if rc.andInScratch() { + return rc.andBitmapContainerScratch(bc) + } + card := rc.andBitmapContainerCardinality(bc) + if card == 0 { + return nil + } + if card > arrayDefaultMaxSize { + answer := newBitmapContainer() + for i := range rc.iv { + copyBitmapRange(answer.bitmap, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + answer.cardinality = card + return answer + } + return rc.andBitmapArray(bc, card) +} + +// andBitmapArray returns the card values of bc inside rc's intervals as an +// array container. +func (rc *runContainer16) andBitmapArray(bc *bitmapContainer, card int) *arrayContainer { + if useVectorFill && card >= arrayVectorFillMinCardinality { + var runBits [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(runBits[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + ac := newArrayContainerSize(card) + fillArrayAND(ac.content, runBits[:], bc.bitmap) + return ac + } + ac := newArrayContainerCapacity(card) + for i := range rc.iv { + ac.content = appendBitmapRange(ac.content, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return ac +} + +func (rc *runContainer16) andBitmapContainerScratch(bc *bitmapContainer) container { + first, last := int(rc.iv[0].start)/64, int(rc.maximum())/64 + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + copyBitmapRange(scratch[:], bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + card := int(popcntSlice(scratch[first : last+1])) + if card == 0 { + return nil + } + return containerFromWords(scratch[:], first, last, card) } func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { @@ -1924,6 +1989,9 @@ func (rc *runContainer16) iand(a container) container { if rc.isFull() { return a.clone() } + if a.isFull() { + return rc + } switch c := a.(type) { case *runContainer16: // Important: there is no reason to believe that the @@ -1950,33 +2018,43 @@ func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container { } func (rc *runContainer16) andArray(ac *arrayContainer) container { - if len(rc.iv) == 0 { - return newArrayContainer() - } - - acCardinality := ac.getCardinality() - c := newArrayContainerCapacity(acCardinality) + c := newArrayContainerCapacity(len(ac.content)) + c.content = c.content[:andArrayInto(rc.iv, ac.content, c.content[:cap(c.content)])] + return c +} - for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; { - iv := rc.iv[rlePos] - arrayVal := ac.content[arrayPos] +// andArrayInto writes the values of vals inside the intervals to buf, which +// may be vals itself, and returns how many there are. +func andArrayInto(intervals []interval16, vals, buf []uint16) int { + if len(intervals) == 0 { + return 0 + } + if len(intervals) == 1 { + lo, hi := sliceInInterval(vals, intervals[0]) + return copy(buf, vals[lo:hi]) + } + n := 0 + for rlePos, arrayPos := 0, 0; arrayPos < len(vals); { + iv := intervals[rlePos] + arrayVal := vals[arrayPos] for iv.last() < arrayVal { rlePos++ - if rlePos == len(rc.iv) { - return c + if rlePos == len(intervals) { + return n } - iv = rc.iv[rlePos] + iv = intervals[rlePos] } if iv.start > arrayVal { - arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start) + arrayPos = advanceUntil(vals, arrayPos, len(vals), iv.start) } else { - c.content = append(c.content, arrayVal) + buf[n] = arrayVal + n++ arrayPos++ } } - return c + return n } func (rc *runContainer16) andNot(a container) container { @@ -2484,10 +2562,77 @@ func (rc *runContainer16) lazyOR(a container) container { return rc.or(a) } +// sliceInInterval returns the bounds of the sorted vals inside iv, galloping +// from the front (advanceUntil searches after -1), where the slice starts. +func sliceInInterval(vals []uint16, iv interval16) (int, int) { + lo := advanceUntil(vals, -1, len(vals), iv.start) + if iv.last() == MaxUint16 { + return lo, len(vals) + } + return lo, advanceUntil(vals, lo-1, len(vals), iv.last()+1) +} + +// firstInArray returns the position of the first of the sorted vals inside +// rc, or -1. +func (rc *runContainer16) firstInArray(vals []uint16) int { + if len(rc.iv) == 1 { + if lo, hi := sliceInInterval(vals, rc.iv[0]); lo < hi { + return lo + } + return -1 + } + if 2*len(vals)*bits.Len(uint(len(rc.iv))) < len(rc.iv) { + // Few values against many intervals: a search each beats walking + // every interval. + for i, v := range vals { + if rc.contains(v) { + return i + } + } + return -1 + } + for i, pos := 0, 0; i < len(rc.iv) && pos < len(vals); { + v := vals[pos] + for rc.iv[i].last() < v { + if i++; i == len(rc.iv) { + return -1 + } + } + if rc.iv[i].start <= v { + return pos + } + pos = advanceUntil(vals, pos, len(vals), rc.iv[i].start) + } + return -1 +} + func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation - isect := rc.and(a) - return !isect.isEmpty() + switch c := a.(type) { + case *bitmapContainer: + for i := range rc.iv { + if c.intersectsRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) { + return true + } + } + return false + case *arrayContainer: + return rc.firstInArray(c.content) >= 0 + case *runContainer16: + i, j := 0, 0 + for i < len(rc.iv) && j < len(c.iv) { + x, y := rc.iv[i], c.iv[j] + if haveOverlap16(x, y) { + return true + } + if x.last() < y.last() { + i++ + } else { + j++ + } + } + return false + } + panic("unsupported container type") } func (rc *runContainer16) xor(a container) container { diff --git a/setutil.go b/setutil.go index d3a80be6..3d9404ab 100644 --- a/setutil.go +++ b/setutil.go @@ -197,95 +197,66 @@ func intersection2by2Cardinality( } } -// intersects2by2 computes whether the two sets intersect -func intersects2by2( - set1 []uint16, - set2 []uint16, -) bool { - if (len(set1) == 0) || (len(set2) == 0) { - return false - } +// firstShared2by2 returns the positions of the first value the two sorted +// sets share, or -1, -1. +func firstShared2by2(set1, set2 []uint16) (int, int) { if len(set1)*64 < len(set2) { - return onesidedgallopingintersect2by2Bool(set1, set2) + return firstSharedGalloping(set1, set2) } else if len(set2)*64 < len(set1) { - return onesidedgallopingintersect2by2Bool(set2, set1) - } else { - return intersects2by2Bool(set1, set2) + j, i := firstSharedGalloping(set2, set1) + return i, j } -} - -func intersects2by2Bool( - set1 []uint16, - set2 []uint16, -) bool { - index1 := 0 - index2 := 0 - value1 := set1[index1] - value2 := set2[index2] -mainwhile: + if len(set1) == 0 || len(set2) == 0 { + return -1, -1 + } + n1, n2 := len(set1), len(set2) + s1, s2 := set1[0], set2[0] for { - - if value2 < value1 { + if s1 < s2 { for { - index2++ - if index2 == len(set2) { - break mainwhile + if set1 = set1[1:]; len(set1) == 0 { + return -1, -1 } - value2 = set2[index2] - if value2 >= value1 { + if s1 = set1[0]; s1 >= s2 { break } } } - if value1 < value2 { + if s2 < s1 { for { - index1++ - if index1 == len(set1) { - break mainwhile + if set2 = set2[1:]; len(set2) == 0 { + return -1, -1 } - value1 = set1[index1] - if value1 >= value2 { + if s2 = set2[0]; s2 >= s1 { break } } - } else { - // (set2[k2] == set1[k1]) - return true + } + if s1 == s2 { + return n1 - len(set1), n2 - len(set2) } } - return false } -func onesidedgallopingintersect2by2Bool( - smallset []uint16, - largeset []uint16, -) bool { - k1 := 0 - k2 := 0 - s1 := largeset[k1] - s2 := smallset[k2] -mainwhile: - for { - if s1 < s2 { - k1 = advanceUntil(largeset, k1, len(largeset), s2) - if k1 == len(largeset) { - break mainwhile - } - s1 = largeset[k1] +// firstSharedGalloping is firstShared2by2 for a small set against a far +// larger one. +func firstSharedGalloping(small, large []uint16) (int, int) { + k := 0 + for i, v := range small { + if k = advanceUntil(large, k-1, len(large), v); k == len(large) { + return -1, -1 } - if s2 < s1 { - k2++ - if k2 == len(smallset) { - break mainwhile - } - s2 = smallset[k2] - } else { - // (set2[k2] == set1[k1]) - return true + if large[k] == v { + return i, k } - } - return false + return -1, -1 +} + +// intersects2by2 computes whether the two sets intersect +func intersects2by2(set1, set2 []uint16) bool { + i, _ := firstShared2by2(set1, set2) + return i >= 0 } func localintersect2by2( diff --git a/util.go b/util.go index e49ad29c..ac133947 100644 --- a/util.go +++ b/util.go @@ -152,6 +152,76 @@ func flipBitmapRange(bitmap []uint64, start int, end int) { bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) } +// copyBitmapRange copies the bits of src within [start, end) into dst, which +// must be zero there. +func copyBitmapRange(dst, src []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + return + } + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) + copy(dst[firstword+1:endword], src[firstword+1:endword]) + dst[endword] |= src[endword] & (^uint64(0) >> (uint(-end) % 64)) +} + +// appendBitmapRange appends the positions of the bits set in [start, end). +func appendBitmapRange(out []uint16, bitmap []uint64, start int, end int) []uint16 { + if start >= end { + return out + } + firstword := start / 64 + endword := (end - 1) / 64 + for i := firstword; i <= endword; i++ { + w := bitmap[i] + if i == firstword { + w &= ^uint64(0) << uint(start%64) + } + if i == endword { + w &= ^uint64(0) >> (uint(-end) % 64) + } + for w != 0 { + out = append(out, uint16(i*64+bits.TrailingZeros64(w))) + w &= w - 1 + } + } + return out +} + +// clearBitmapGaps clears the bits in [from, to) that no interval of iv covers. +func clearBitmapGaps(bitmap []uint64, iv []interval16, from, to int) { + for _, x := range iv { + if int(x.start) >= to { + break + } + resetBitmapRange(bitmap, from, int(x.start)) + from = max(from, int(x.last())+1) + } + resetBitmapRange(bitmap, from, to) +} + +// containerFromWords builds an array or bitmap container from the card bits +// set in words [first, last] of w, which spans a whole key and is zero +// outside them. +func containerFromWords(w []uint64, first, last, card int) container { + if card <= arrayDefaultMaxSize { + if useVectorFill && card >= arrayVectorFillMinCardinality { + return newArrayContainerFromBitmap(&bitmapContainer{cardinality: card, bitmap: w}) + } + ac := newArrayContainerCapacity(card) + ac.content = appendBitmapRange(ac.content, w, first*64, (last+1)*64) + return ac + } + bc := newBitmapContainer() + copy(bc.bitmap[first:last+1], w[first:last+1]) + bc.cardinality = card + return bc +} + func resetBitmapRange(bitmap []uint64, start int, end int) { if start >= end { return @@ -163,9 +233,7 @@ func resetBitmapRange(bitmap []uint64, start int, end int) { return } bitmap[firstword] &= ^(^uint64(0) << uint(start%64)) - for i := firstword + 1; i < endword; i++ { - bitmap[i] = 0 - } + clear(bitmap[firstword+1 : endword]) bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64)) }