lib/uint64set: Add roaring64 bitmap to vendors and use it in benchmarks

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

README.md

@@ -1,6 +1,6 @@
 # VictoriaMetrics
 
-[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?logo=github&labelColor=gray&color=gray&label=Release)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)

app/victoria-metrics/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -22,7 +23,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer/push"
 )
 
 var (
@@ -35,9 +35,10 @@ var (
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate.")
-	vmselectMaxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
+	vmselectMaxConcurrentRequests = flagutil.NewIntWithDynamicDefault("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "vmselect.getDefaultMaxConcurrentRequests()",
+		"The maximum number of concurrent search requests. "+
+			"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
+			"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	vmselectMaxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
 )
@@ -91,7 +92,9 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
+
 	vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
+	appmetrics.MustCreateUncleanShutdownMarker(vmstorage.DataPath())
 	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()
@@ -104,10 +107,8 @@ func main() {
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
-	push.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	push.Stop()
 	pushmetrics.Stop()
 
 	stopSelfScraper()
@@ -123,6 +124,7 @@ func main() {
 
 	vmstorage.Stop()
 	vmselect.Stop()
+	appmetrics.MustRemoveUncleanShutdownMarker(vmstorage.DataPath())
 
 	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }

app/vmagent/remotewrite/remotewrite.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -62,9 +63,10 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+	queues = flagutil.NewArrayIntWithDynamicDefault("remoteWrite.queues", cgroup.AvailableCPUs()*2, "2*cgroup.AvailableCPUs()",
+		"The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+			"isn't enough for sending high volume of collected data to remote storage. "+
+			"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	inmemoryQueues = flagutil.NewArrayInt("remoteWrite.inmemoryQueues", 0, "The number of additional workers per each -remoteWrite.url, which send only recently ingested data from the in-memory queue, "+
 		"while the file-based queue at -remoteWrite.tmpDataPath is drained by workers configured via -remoteWrite.queues. "+
 		"This reduces delivery lag for fresh samples when the file-based queue contains a backlog accumulated during remote storage outages.")
@@ -233,6 +235,7 @@ func Init() {
 	initStreamAggrConfigGlobal()
 
 	initRemoteWriteCtxs(*remoteWriteURLs)
+	appmetrics.MustCreateUncleanShutdownMarker(*tmpDataPath)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -391,6 +394,8 @@ func Stop() {
 	if sl := dailySeriesLimiter; sl != nil {
 		sl.MustStop()
 	}
+
+	appmetrics.MustRemoveUncleanShutdownMarker(*tmpDataPath)
 }
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url

app/vmalert/remotewrite/init.go

@@ -36,9 +36,10 @@ var (
 	idleConnectionTimeout = flag.Duration("remoteWrite.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider settings this value less to the value of "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 	maxIdleConnections    = flag.Int("remoteWrite.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to -remoteWrite.url for the vmalert-tool debug writer, which sends every series in a separate request. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 
-	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
-	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
-	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
+	maxQueueSize = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
+	maxBatchSize = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
+	concurrency  = flagutil.NewIntWithDynamicDefault("remoteWrite.concurrency", defaultConcurrency, "2*cgroup.AvailableCPUs()",
+		"Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
 	flushInterval = flag.Duration("remoteWrite.flushInterval", defaultFlushInterval, "Defines interval of flushes to remote write endpoint")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")

app/vmselect/main.go

@@ -26,7 +26,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer/push"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmalertproxy"
 )
@@ -118,15 +117,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	// Handle non-trivial dynamic requests, which may take big amounts of time and resources.
 	startTime := time.Now()
 	defer requestDuration.UpdateDuration(startTime)
-	tracerEnabled := httputil.GetBool(r, "trace") || push.IsEnabled()
+	tracerEnabled := httputil.GetBool(r, "trace")
 	qt := querytracer.New(tracerEnabled, "%s", r.URL.Path)
-	if push.IsEnabled() {
-		defer func() {
-			if time.Since(startTime) >= push.MinTraceDuration() {
-				push.Push(qt)
-			}
-		}()
-	}
 
 	// Limit the number of concurrent queries.
 	select {

app/vmselect/netstorage/netstorage.go

@@ -20,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
@@ -30,11 +31,12 @@ var (
 	maxSamplesPerSeries = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage")
 	maxSamplesPerQuery  = flag.Int("search.maxSamplesPerQuery", 1e9, "The maximum number of raw samples a single query can process across all time series. "+
 		"This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries")
-	maxWorkersPerQuery = flag.Int("search.maxWorkersPerQuery", defaultMaxWorkersPerQuery, "The maximum number of CPU cores a single query can use. "+
-		"The default value should work good for most cases. "+
-		"The flag can be set to lower values for improving performance of big number of concurrently executed queries. "+
-		"The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). "+
-		"There is no sense in setting this flag to values bigger than the number of CPU cores available on the system")
+	maxWorkersPerQuery = flagutil.NewIntWithDynamicDefault("search.maxWorkersPerQuery", defaultMaxWorkersPerQuery, "netstorage.defaultMaxWorkersPerQuery()",
+		"The maximum number of CPU cores a single query can use. "+
+			"The default value should work good for most cases. "+
+			"The flag can be set to lower values for improving performance of big number of concurrently executed queries. "+
+			"The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). "+
+			"There is no sense in setting this flag to values bigger than the number of CPU cores available on the system")
 )
 
 // Result is a single timeseries result.

deployment/docker/rules/alerts-health.yml

@@ -16,6 +16,19 @@ groups:
             Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
             It might be crashlooping.
 
+      - alert: UncleanShutdown
+        expr: vm_app_prev_shutdown_unclean == 1 and time() - vm_app_start_timestamp < 600
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }} on instance {{ $labels.instance }} started after an unclean shutdown"
+          description: |
+            The previous process run didn't shut down cleanly. Check the logs for OOM, SIGKILL,
+            a host failure, or another unexpected termination. In Kubernetes, a pod may be forcefully
+            killed with SIGKILL if the shutdown takes longer than terminationGracePeriodSeconds.
+            This alert stops firing 10 minutes after startup.
+            See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443 for more details.
+
       - alert: ServiceDown
         expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"} == 0
         for: 2m

docs/Makefile

@@ -90,12 +90,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/victoria_metrics_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/victoria_metrics_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent insert requests/ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of concurrent search requests\./ s/(default [0-9]\+)/(default vmselect.getDefaultMaxConcurrentRequests())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of CPU cores a single query can use\./ s/(default [0-9]\+)/(default netstorage.defaultMaxWorkersPerQuery())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/victoria_metrics_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/victoria_metrics_common_flags.md
 
 docs-update-vmauth-flags:
 ifndef TAG
@@ -119,9 +116,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmauth_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmauth_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmauth_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmauth_common_flags.md
 
 docs-update-vmagent-flags:
 ifndef TAG
@@ -145,11 +142,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmagent_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmagent_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent insert requests/ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmagent_common_flags.md
-	sed -i '/The number of concurrent queues to each -remoteWrite.url./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmagent_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmagent_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmagent_common_flags.md
 
 docs-update-vmalert-flags:
 ifndef TAG
@@ -173,10 +168,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmalert_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmalert_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/Defines number of writers for concurrent writing into remote write endpoint./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmalert_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmalert_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmalert_common_flags.md
 
 docs-update-vmselect-flags:
 ifndef TAG

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,7 +26,9 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose the `vm_app_prev_shutdown_unclean` gauge. It is set to `1` when the previous process run didn't shut down cleanly. Added the `UncleanShutdown` [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml), which fires for 10 minutes after an unclean shutdown is detected. See [#8443](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show the selected time zone UTC offset next to the date/time controls and allow opening time zone settings from it. See [#11332](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11332).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/), and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): show how the default value is calculated for command-line flags which derive it from the number of available CPU cores. For example, `-maxConcurrentInserts` now prints `(default 16 = 2*cgroup.AvailableCPUs())` in `-help` output instead of `(default 16)`. Updated flags: `-search.maxConcurrentRequests`, `-search.maxWorkersPerQuery`, `-fs.maxConcurrency`, `-remoteWrite.concurrency`, `-remoteWrite.queues`. See [#9680](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680). Thanks to @Vandit1604 for contribution.
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix infinite loop in the OpenTelemetry Firehose ingestion endpoint (`/opentelemetry/api/v1/push`) when receiving a malformed record with an incomplete varint in the `data` field. Previously this caused the goroutine to spin forever, permanently consuming CPU until the process was restarted.
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): reuse connections to `-remoteWrite.url` when writing the results of recording rules and alerts. Previously every series was sent over a new connection, which left a lot of sockets in `TIME_WAIT` state and could exhaust the ephemeral port range. The number of idle connections can be tuned via the new `-remoteWrite.maxIdleConnections` command-line flag. Thanks @evkuzin for contribution.

docs/victoriametrics/integrations/grafana/_index.md

@@ -50,7 +50,7 @@ If you don't see an option to create a data source - try contacting system admin
 Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure/)
 in Grafana. Follow the same connection instructions as for [VictoriaMetrics datasource](#VictoriaMetrics-datasource).
 
-In the "Type and version" section set the type to "Prometheus" and the version to at least "2.24.x".
+In the "Performance" section set the Prometheus type to "Prometheus" and the Prometheus version to at least "2.24.x".
 This allows Grafana to use a more efficient API to get label values:
 
 ![Datasource](datasource-prometheus.webp)

docs/victoriametrics/vmalert.md

@@ -1127,13 +1127,18 @@ Or for all rules within the [group](#groups) {{% available_from "v1.117.0" %}}.
 Just set `debug: true` in configuration and vmalert will start printing additional log messages:
 
 ```sh
-2022-09-15T13:35:41.155Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 series (elapsed: 5.896041ms, isPartial: false)
-2022-09-15T13:35:56.149Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29+by%28instance%29+%3E+0&step=15s&time=1663248945"
-2022-09-15T13:35:56.178Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: query returned 1 series (elapsed: 28.368208ms, isPartial: false)
-2022-09-15T13:35:56.178Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29&step=15s&time=1663248945"
-2022-09-15T13:35:56.179Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} created in state PENDING
+2026-08-20T08:21:29.464Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A20%3A00Z"
+2026-08-20T08:21:29.465Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:20:00Z: query returned 0 series (series_fetched: 0, elapsed: 1.075166ms, isPartial: false)
 ...
-2022-09-15T13:36:56.153Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:36:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} PENDING => FIRING: 1m0s since becoming active at 2022-09-15 15:35:56.126006 +0200 CEST m=+39.384575417
+2026-08-20T08:22:29.466Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A21%3A00Z"
+2026-08-20T08:22:29.468Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: query returned 2 series (series_fetched: 2, elapsed: 2.055916ms, isPartial: false)
+2026-08-20T08:22:29.469Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: alert 4671711516378822929 {alertgroup="vm-health",alertname="ServiceDown",instance="victoriametrics:8428",job="victoriametrics",severity="critical"} created in state PENDING
+2026-08-20T08:22:29.469Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: alert 6230585559362831632 {alertgroup="vm-health",alertname="ServiceDown",instance="vmagent:8429",job="vmagent",severity="critical"} created in state PENDING
+...
+2026-08-20T08:23:29.463Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A22%3A00Z"
+2026-08-20T08:23:29.465Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: query returned 2 series (series_fetched: 2, elapsed: 1.391416ms, isPartial: false)
+2026-08-20T08:23:29.466Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: alert 4671711516378822929 {alertgroup="vm-health",alertname="ServiceDown",instance="victoriametrics:8428",job="victoriametrics",severity="critical"} PENDING => FIRING: 1m0s since becoming active at 2026-08-20 08:21:00 +0000 UTC
+2026-08-20T08:23:29.466Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: alert 6230585559362831632 {alertgroup="vm-health",alertname="ServiceDown",instance="vmagent:8429",job="vmagent",severity="critical"} PENDING => FIRING: 1m0s since becoming active at 2026-08-20 08:21:00 +0000 UTC
 ```
 
 Sensitive info is stripped from the `curl` examples - see [security](#security) section for more details.

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
+	github.com/RoaringBitmap/roaring/v2 v2.25.0
 	github.com/VictoriaMetrics/VictoriaLogs v1.51.1-0.20260624061259-dc94972a8708
 	github.com/VictoriaMetrics/easyproto v1.2.0
 	github.com/VictoriaMetrics/fastcache v1.13.3
@@ -73,6 +74,7 @@ require (
 	github.com/aws/smithy-go v1.27.4 // indirect
 	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.24.4 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
 	github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
@@ -108,6 +110,7 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.157.0 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/RoaringBitmap/roaring/v2 v2.25.0 h1:HjcMG0PfmgO1rJcp2VHMarvQiulkB51qA31UH4I+j/U=
+github.com/RoaringBitmap/roaring/v2 v2.25.0/go.mod h1:SfT3of9nYh3vis1dIbCj4Yw6KQGujTN+f345nrN/0JA=
 github.com/VictoriaMetrics/VictoriaLogs v1.51.1-0.20260624061259-dc94972a8708 h1:D9/Jzlm3B8PBnrWxg4ft8KYZdG607dV3lpBfPCoiJD8=
 github.com/VictoriaMetrics/VictoriaLogs v1.51.1-0.20260624061259-dc94972a8708/go.mod h1:H4sDxcvk6OmC6zOt++IlDyrwfbn4F1eSLwMpR+kpRt8=
 github.com/VictoriaMetrics/easyproto v1.2.0 h1:FJT9uNXA2isppFuJErbLqD306KoFlehl7Wn2dg/6oIE=
@@ -126,6 +128,8 @@ github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoGXLwLQOZE=
+github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
 github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
@@ -342,6 +346,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=

lib/appmetrics/appmetrics.go

@@ -65,6 +65,9 @@ func writePrometheusMetrics(w io.Writer) {
 	// Export start time and uptime in seconds
 	metrics.WriteGaugeUint64(w, "vm_app_start_timestamp", uint64(startTime.Unix()))
 	metrics.WriteGaugeUint64(w, "vm_app_uptime_seconds", uint64(time.Since(startTime).Seconds()))
+	if uncleanShutdownEnabled.Load() {
+		metrics.WriteGaugeUint64(w, "vm_app_prev_shutdown_unclean", uncleanShutdown)
+	}
 
 	// Export flags as metrics.
 	isSetMap := make(map[string]bool)

lib/appmetrics/osmetrics.go

@@ -13,13 +13,13 @@ type osInfo struct {
 	release string
 }
 
-var os osInfo
+var hostOS osInfo
 var initOSOnce sync.Once
 
 func writeOSMetrics(w io.Writer) {
 	initOSOnce.Do(initOS)
 
-	if os.name != "" {
-		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_os_info{os=%q, release=%q}`, os.name, os.release), 1)
+	if hostOS.name != "" {
+		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_os_info{os=%q, release=%q}`, hostOS.name, hostOS.release), 1)
 	}
 }

lib/appmetrics/osmetrics_darwin.go

@@ -8,7 +8,7 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "darwin"}
+	hostOS = osInfo{name: "darwin"}
 
 	out, err := exec.Command("sysctl", "-n", "kern.osrelease").Output()
 	if err != nil {
@@ -16,5 +16,5 @@ func initOS() {
 		return
 	}
 
-	os.release = strings.TrimSpace(string(out))
+	hostOS.release = strings.TrimSpace(string(out))
 }

lib/appmetrics/osmetrics_linux.go

@@ -7,7 +7,7 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "linux"}
+	hostOS = osInfo{name: "linux"}
 
 	var uname syscall.Utsname
 	if err := syscall.Uname(&uname); err != nil {
@@ -22,5 +22,5 @@ func initOS() {
 		}
 		ur = append(ur, byte(v))
 	}
-	os.release = string(ur)
+	hostOS.release = string(ur)
 }

lib/appmetrics/osmetrics_windows.go

@@ -8,12 +8,12 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "windows"}
+	hostOS = osInfo{name: "windows"}
 
 	ver := windows.RtlGetVersion()
 	if ver == nil {
 		logger.Warnf("vm_os_info metric will miss release info since windows.RtlGetVersion returned nil version")
 		return
 	}
-	os.release = fmt.Sprintf("%d.%d.%d", ver.MajorVersion, ver.MinorVersion, ver.BuildNumber)
+	hostOS.release = fmt.Sprintf("%d.%d.%d", ver.MajorVersion, ver.MinorVersion, ver.BuildNumber)
 }

lib/appmetrics/unclean_shutdown.go

@@ -0,0 +1,53 @@
+package appmetrics
+
+import (
+	"os"
+	"path/filepath"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// UncleanShutdownMarkerFilename is the marker file used to detect a previous unclean shutdown.
+const UncleanShutdownMarkerFilename = ".vm_app_running"
+
+var (
+	uncleanShutdownEnabled atomic.Bool
+	uncleanShutdown        uint64
+)
+
+// MustCreateUncleanShutdownMarker creates an UncleanShutdownMarkerFilename marker file in the given dirPath.
+// Must be called once on program startup and paired with a single MustRemoveUncleanShutdownMarker call on exit.
+//
+// If the marker file already exists on startup, it indicates a previous unclean shutdown and uncleanShutdown is set to 1.
+func MustCreateUncleanShutdownMarker(dirPath string) {
+	if !uncleanShutdownEnabled.CompareAndSwap(false, true) {
+		logger.Fatalf("BUG: unclean shutdown marker was already initialized. It could only be called once")
+	}
+	marker := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+	f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+	if err == nil {
+		fs.MustClose(f)
+		return
+	}
+	if os.IsExist(err) {
+		uncleanShutdown = 1
+		logger.Warnf("Previous shutdown was unclean since file %q exists. Please check logs and investigate the reason of unclean shutdown", marker)
+		return
+	}
+	logger.Panicf("FATAL: cannot create unclean shutdown marker %q: %s", marker, err)
+}
+
+// MustRemoveUncleanShutdownMarker removes the UncleanShutdownMarkerFilename marker file created by MustCreateUncleanShutdownMarker.
+// Must be called once, as late as possible before program exit.
+func MustRemoveUncleanShutdownMarker(dirPath string) {
+	if !uncleanShutdownEnabled.Load() {
+		logger.Fatalf("BUG: unclean shutdown marker was not initialized with MustCreateUncleanShutdownMarker call")
+	}
+	marker := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+	if err := os.Remove(marker); err != nil {
+		logger.Fatalf("FATAL: cannot remove unclean shutdown marker %q: %s", marker, err)
+	}
+	fs.MustSyncPath(dirPath)
+}

lib/appmetrics/unclean_shutdown_test.go

@@ -0,0 +1,60 @@
+package appmetrics
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestUncleanShutdownLifecycle(t *testing.T) {
+
+	t.Cleanup(func() {
+		uncleanShutdownEnabled.Store(false)
+	})
+	dirPath := t.TempDir()
+	markerPath := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+
+	// unclean logic is disabled. the unclean shutdown metric should not be exposed
+	var bb bytes.Buffer
+	writePrometheusMetrics(&bb)
+	if strings.Contains(bb.String(), "vm_app_prev_shutdown_unclean") {
+		t.Fatalf("unexpected unclean shutdown metric before starting the marker")
+	}
+
+	// clean start, the metric must report 0
+	MustCreateUncleanShutdownMarker(dirPath)
+	mustContainUncleanShutdownMetric(t, 0)
+	if _, err := os.Stat(markerPath); err != nil {
+		t.Fatalf("cannot stat the running marker after the first start: %s", err)
+	}
+	MustRemoveUncleanShutdownMarker(dirPath)
+	if _, err := os.Stat(markerPath); !os.IsNotExist(err) {
+		t.Fatalf("unexpected running marker after a clean shutdown; got error %v; want os.ErrNotExist", err)
+	}
+	uncleanShutdownEnabled.Store(false)
+
+	// simulate prev unclean shutdown, the metric must report 1
+	if err := os.WriteFile(markerPath, nil, 0600); err != nil {
+		t.Fatalf("cannot create test marker: %s", err)
+	}
+	MustCreateUncleanShutdownMarker(dirPath)
+	mustContainUncleanShutdownMetric(t, 1)
+	MustRemoveUncleanShutdownMarker(dirPath)
+	if _, err := os.Stat(markerPath); !os.IsNotExist(err) {
+		t.Fatalf("unexpected running marker after a clean shutdown; got error %v; want os.ErrNotExist", err)
+	}
+}
+
+func mustContainUncleanShutdownMetric(t *testing.T, value uint64) {
+	t.Helper()
+
+	var bb bytes.Buffer
+	writePrometheusMetrics(&bb)
+	want := "vm_app_prev_shutdown_unclean " + strconv.FormatUint(value, 10) + "\n"
+	if !strings.Contains(bb.String(), want) {
+		t.Fatalf("missing %q in the exported app metrics", want)
+	}
+}

lib/backup/fscommon/fscommon.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/backupnames"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
@@ -107,7 +108,7 @@ func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
 }
 
 func isSpecialFile(name string) bool {
-	return name == "flock.lock" || name == backupnames.RestoreInProgressFilename || name == backupnames.RestoreMarkFileName || strings.HasSuffix(name, ".tmp")
+	return name == "flock.lock" || name == appmetrics.UncleanShutdownMarkerFilename || name == backupnames.RestoreInProgressFilename || name == backupnames.RestoreMarkFileName || strings.HasSuffix(name, ".tmp")
 }
 
 // RemoveEmptyDirs recursively removes empty directories under the given dir.

lib/flagutil/array.go

@@ -39,8 +39,29 @@ func NewArrayBool(name, description string) *ArrayBool {
 }
 
 // NewArrayInt returns new ArrayInt with the given name, defaultValue and description.
+//
+// -help shows defaultValue as a plain number. Use NewArrayIntWithDynamicDefault when
+// defaultValue is calculated at runtime.
 func NewArrayInt(name string, defaultValue int, description string) *ArrayInt {
-	description += fmt.Sprintf(" (default %d)", defaultValue)
+	return newArrayInt(name, defaultValue, strconv.Itoa(defaultValue), description)
+}
+
+// NewArrayIntWithDynamicDefault returns new ArrayInt with the given name, defaultValue and description.
+//
+// Use it instead of NewArrayInt when defaultValue is calculated at runtime.
+// See NewIntWithDynamicDefault for why such a value needs a hint.
+func NewArrayIntWithDynamicDefault(name string, defaultValue int, defaultValueHint, description string) *ArrayInt {
+	if defaultValueHint == "" {
+		panic(fmt.Sprintf("BUG: missing defaultValueHint for -%s", name))
+	}
+	return newArrayInt(name, defaultValue, fmt.Sprintf("%d = %s", defaultValue, defaultValueHint), description)
+}
+
+// newArrayInt registers an int array flag, which shows defaultValueText as its default in -help.
+//
+// Array flags keep the default in the description, since flag.Var hides an empty DefValue.
+func newArrayInt(name string, defaultValue int, defaultValueText, description string) *ArrayInt {
+	description += fmt.Sprintf(" (default %s)", defaultValueText)
 	description += "\nSupports `array` of values separated by comma or specified via multiple flags."
 	description += "\nEmpty values are set to default value."
 	a := &ArrayInt{

lib/flagutil/flag.go

@@ -7,6 +7,25 @@ import (
 	"strings"
 )
 
+// NewIntWithDynamicDefault returns a new int flag with the given name, defaultValue and description.
+//
+// Use it instead of flag.Int when defaultValue is calculated at runtime, for example
+// from the number of CPU cores. Such a value differs per machine, so -help shows both
+// the value and defaultValueHint, for example "16 = 2 * availableCPUs".
+//
+// Only -help output changes. The flag value stays defaultValue.
+func NewIntWithDynamicDefault(name string, defaultValue int, defaultValueHint, description string) *int {
+	if defaultValueHint == "" {
+		panic(fmt.Sprintf("BUG: missing defaultValueHint for -%s", name))
+	}
+	p := flag.Int(name, defaultValue, description)
+
+	// DefValue is only the text shown by -help: "default value (as text); for usage message".
+	flag.Lookup(name).DefValue = fmt.Sprintf("%d = %s", defaultValue, defaultValueHint)
+
+	return p
+}
+
 // WriteFlags writes all the explicitly set flags to w.
 func WriteFlags(w io.Writer) {
 	flag.Visit(func(f *flag.Flag) {

lib/flagutil/flag_test.go

@@ -0,0 +1,51 @@
+package flagutil
+
+import (
+	"flag"
+	"strings"
+	"testing"
+)
+
+// The flags are registered at package level, since flag registration panics when it repeats.
+var (
+	fooFlagIntDynamicDefault      = NewIntWithDynamicDefault("fooFlagIntDynamicDefault", 42, "2 * availableCPUs", "test")
+	fooFlagArrayIntDynamicDefault = NewArrayIntWithDynamicDefault("fooFlagArrayIntDynamicDefault", 42, "2 * availableCPUs", "test")
+	fooFlagArrayIntPlainDefault   = NewArrayInt("fooFlagArrayIntPlainDefault", 42, "test")
+)
+
+func TestNewIntWithDynamicDefaultSuccess(t *testing.T) {
+	// -help must show the value together with the hint.
+	f := flag.Lookup("fooFlagIntDynamicDefault")
+	if f.DefValue != "42 = 2 * availableCPUs" {
+		t.Fatalf("unexpected DefValue; got %q; want %q", f.DefValue, "42 = 2 * availableCPUs")
+	}
+
+	// the flag value must stay the calculated one.
+	if *fooFlagIntDynamicDefault != 42 {
+		t.Fatalf("unexpected flag value; got %d; want %d", *fooFlagIntDynamicDefault, 42)
+	}
+}
+
+func TestNewArrayIntWithDynamicDefaultSuccess(t *testing.T) {
+	// array flags keep the default in the description, so the hint must go there.
+	f := flag.Lookup("fooFlagArrayIntDynamicDefault")
+	if !strings.Contains(f.Usage, "(default 42 = 2 * availableCPUs)") {
+		t.Fatalf("missing the hint in the flag description; got %q", f.Usage)
+	}
+
+	// the default value must stay the calculated one.
+	if n := fooFlagArrayIntDynamicDefault.GetOptionalArg(0); n != 42 {
+		t.Fatalf("unexpected default value; got %d; want %d", n, 42)
+	}
+}
+
+func TestNewArrayIntKeepsPlainDefault(t *testing.T) {
+	// NewArrayInt must keep showing a plain number, since it shares the body with the dynamic one.
+	f := flag.Lookup("fooFlagArrayIntPlainDefault")
+	if !strings.Contains(f.Usage, "(default 42)") {
+		t.Fatalf("unexpected flag description; got %q", f.Usage)
+	}
+	if n := fooFlagArrayIntPlainDefault.GetOptionalArg(0); n != 42 {
+		t.Fatalf("unexpected default value; got %d; want %d", n, 42)
+	}
+}

lib/fs/fsutil/concurrency.go

@@ -1,14 +1,15 @@
 package fsutil
 
 import (
-	"flag"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
-var maxConcurrency = flag.Int("fs.maxConcurrency", getDefaultConcurrency(), "The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency "+
-	"on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph")
+var maxConcurrency = flagutil.NewIntWithDynamicDefault("fs.maxConcurrency", getDefaultConcurrency(), "fsutil.getDefaultConcurrency()",
+	"The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency "+
+		"on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph")
 
 func getDefaultConcurrency() int {
 	n := min(16*cgroup.AvailableCPUs(), 256)

lib/querytracer/otlp.go

@@ -1,122 +0,0 @@
-package querytracer
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"time"
-)
-
-// OTLPTrace is an intermediate representation of a finished tracer tree
-// suitable for conversion to OTLP protobuf spans.
-type OTLPTrace struct {
-	TraceID string
-	Spans   []OTLPSpan
-}
-
-// OTLPSpan represents a single span ready for OTLP export.
-type OTLPSpan struct {
-	TraceID      string
-	SpanID       string
-	ParentSpanID string // empty for root
-	Name         string
-	StartNano    uint64
-	EndNano      uint64
-	Events       []OTLPEvent
-}
-
-// OTLPEvent is a timestamped annotation within a span (from Printf leaf nodes).
-type OTLPEvent struct {
-	TimeNano uint64
-	Name     string
-}
-
-// ToOTLPTrace converts the finished tracer tree into a flat list of OTLP spans.
-// It must be called after Done/Donef.
-func (t *Tracer) ToOTLPTrace() *OTLPTrace {
-	if t == nil {
-		return nil
-	}
-	traceID := newTraceID()
-	tr := &OTLPTrace{TraceID: traceID}
-	collectSpans(tr, t, traceID, "", t.startTime)
-	return tr
-}
-
-// collectSpans recursively walks the tracer tree and populates tr.Spans.
-// parentSpanID is empty for the root.
-// prevTime is used only when approximating timestamps for JSON-embedded spans.
-func collectSpans(tr *OTLPTrace, t *Tracer, traceID, parentSpanID string, prevTime time.Time) {
-	if t.span != nil {
-		collectJSONSpans(tr, t.span, traceID, parentSpanID, prevTime)
-		return
-	}
-
-	isLeaf := t.doneTime.Equal(t.startTime)
-	if isLeaf {
-		// Printf leaf: no span to emit here; the caller handles attaching it.
-		return
-	}
-
-	// Regular span.
-	spanID := newSpanID()
-	spanIdx := len(tr.Spans)
-	tr.Spans = append(tr.Spans, OTLPSpan{
-		TraceID:      traceID,
-		SpanID:       spanID,
-		ParentSpanID: parentSpanID,
-		Name:         t.message,
-		StartNano:    uint64(t.startTime.UnixNano()),
-		EndNano:      uint64(t.doneTime.UnixNano()),
-	})
-
-	// Process children. Leaf children become events on this span; non-leaf
-	// children recurse and produce their own spans.
-	childPrev := t.startTime
-	for _, child := range t.children {
-		if child.span == nil && child.doneTime.Equal(child.startTime) {
-			// Printf leaf: attach as span event.
-			tr.Spans[spanIdx].Events = append(tr.Spans[spanIdx].Events, OTLPEvent{
-				TimeNano: uint64(child.startTime.UnixNano()),
-				Name:     child.message,
-			})
-			continue
-		}
-		collectSpans(tr, child, traceID, spanID, childPrev)
-		if !child.doneTime.IsZero() && !child.doneTime.Equal(child.startTime) {
-			childPrev = child.doneTime
-		}
-	}
-}
-
-// collectJSONSpans converts a span tree that was deserialized via AddJSON.
-// Timestamps are approximated from prevTime since JSON spans only carry duration_msec.
-func collectJSONSpans(tr *OTLPTrace, s *span, traceID, parentSpanID string, prevTime time.Time) {
-	startNano := uint64(prevTime.UnixNano())
-	durationNano := uint64(s.DurationMsec * float64(time.Millisecond))
-	spanID := newSpanID()
-	tr.Spans = append(tr.Spans, OTLPSpan{
-		TraceID:      traceID,
-		SpanID:       spanID,
-		ParentSpanID: parentSpanID,
-		Name:         s.Message,
-		StartNano:    startNano,
-		EndNano:      startNano + durationNano,
-	})
-	childPrev := prevTime
-	for _, child := range s.Children {
-		collectJSONSpans(tr, child, traceID, spanID, childPrev)
-		childPrev = childPrev.Add(time.Duration(child.DurationMsec * float64(time.Millisecond)))
-	}
-}
-
-func newTraceID() string {
-	var b [16]byte
-	_, _ = rand.Read(b[:])
-	return hex.EncodeToString(b[:])
-}
-
-func newSpanID() string {
-	var b [8]byte
-	_, _ = rand.Read(b[:])
-	return hex.EncodeToString(b[:])
-}

lib/querytracer/push/pb.go

@@ -1,156 +0,0 @@
-package push
-
-// This file contains marshal-only protobuf types for OTLP trace export.
-// Copied and trimmed from VictoriaTraces/lib/protoparser/opentelemetry/pb/
-// Only marshaling is needed since this package only pushes traces outward.
-
-import (
-	"encoding/hex"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// exportTraceServiceRequest is the top-level OTLP protobuf message.
-type exportTraceServiceRequest struct {
-	ResourceSpans []*resourceSpans
-}
-
-func (r *exportTraceServiceRequest) marshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	mm := m.MessageMarshaler()
-	for _, rs := range r.ResourceSpans {
-		rs.marshalProtobuf(mm.AppendMessage(1))
-	}
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-// resourceSpans groups spans from a single resource (e.g. service instance).
-type resourceSpans struct {
-	Resource   resource
-	ScopeSpans []*scopeSpans
-}
-
-func (rs *resourceSpans) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	rs.Resource.marshalProtobuf(mm.AppendMessage(1))
-	for _, ss := range rs.ScopeSpans {
-		ss.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-// resource holds resource-level attributes (service.name, etc.).
-type resource struct {
-	Attributes []*keyValue
-}
-
-func (r *resource) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, a := range r.Attributes {
-		a.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-// scopeSpans groups spans from a single instrumentation scope.
-type scopeSpans struct {
-	Scope instrumentationScope
-	Spans []*span
-}
-
-func (ss *scopeSpans) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	ss.Scope.marshalProtobuf(mm.AppendMessage(1))
-	for _, s := range ss.Spans {
-		s.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-// instrumentationScope identifies the library that produced the spans.
-type instrumentationScope struct {
-	Name    string
-	Version string
-}
-
-func (is *instrumentationScope) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, is.Name)
-	mm.AppendString(2, is.Version)
-}
-
-// span represents a single operation within a trace.
-type span struct {
-	// TraceID is a 32-char lowercase hex string (16 bytes).
-	TraceID string
-	// SpanID is a 16-char lowercase hex string (8 bytes).
-	SpanID string
-	// ParentSpanID is a 16-char lowercase hex string; empty for root spans.
-	ParentSpanID      string
-	Name              string
-	StartTimeUnixNano uint64
-	EndTimeUnixNano   uint64
-	Attributes        []*keyValue
-	Events            []*spanEvent
-}
-
-func (s *span) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	traceID, err := hex.DecodeString(s.TraceID)
-	if err != nil {
-		traceID = []byte(s.TraceID)
-	}
-	mm.AppendBytes(1, traceID)
-
-	spanID, err := hex.DecodeString(s.SpanID)
-	if err != nil {
-		spanID = []byte(s.SpanID)
-	}
-	mm.AppendBytes(2, spanID)
-
-	// field 3: trace_state — omitted
-
-	parentSpanID, err := hex.DecodeString(s.ParentSpanID)
-	if err != nil {
-		parentSpanID = []byte(s.ParentSpanID)
-	}
-	mm.AppendBytes(4, parentSpanID)
-
-	mm.AppendString(5, s.Name)
-	// field 6: kind — omitted (INTERNAL=1 is default)
-	mm.AppendFixed64(7, s.StartTimeUnixNano)
-	mm.AppendFixed64(8, s.EndTimeUnixNano)
-	for _, a := range s.Attributes {
-		a.marshalProtobuf(mm.AppendMessage(9))
-	}
-	for _, e := range s.Events {
-		e.marshalProtobuf(mm.AppendMessage(11))
-	}
-}
-
-// spanEvent is a time-stamped annotation within a span.
-type spanEvent struct {
-	TimeUnixNano uint64
-	Name         string
-}
-
-func (se *spanEvent) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendFixed64(1, se.TimeUnixNano)
-	mm.AppendString(2, se.Name)
-}
-
-// keyValue is an OTLP attribute key-value pair.
-type keyValue struct {
-	Key   string
-	Value anyValue
-}
-
-func (kv *keyValue) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, kv.Key)
-	kv.Value.marshalProtobuf(mm.AppendMessage(2))
-}
-
-// anyValue holds a single string attribute value (sufficient for our use case).
-type anyValue struct {
-	StringValue string
-}
-
-func (av *anyValue) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, av.StringValue)
-}

lib/querytracer/push/push.go

@@ -1,233 +0,0 @@
-// Package push implements background exporting of VictoriaMetrics query traces
-// to VictoriaTraces in OTLP protobuf format over HTTP.
-package push
-
-import (
-	"bytes"
-	"compress/gzip"
-	"flag"
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	pushURL = flag.String("search.traceExportURL", "", "If set, query traces are exported to this URL in OTLP protobuf format. "+
-		"For example, -search.traceExportURL=http://victoria-traces:4318/insert/opentelemetry/v1/traces . "+
-		"See https://docs.victoriametrics.com/victoriametrics/query-tracing/")
-	minTraceDuration = flag.Duration("search.traceExportMinDuration", 0, "Minimum query duration for exporting traces via -search.traceExportURL. "+
-		"Traces for faster queries are dropped. 0 means all traces are exported.")
-)
-
-var (
-	pushesTotal  = metrics.NewCounter(`vm_trace_export_pushes_total`)
-	errorsTotal  = metrics.NewCounter(`vm_trace_export_errors_total`)
-	droppedTotal = metrics.NewCounter(`vm_trace_export_dropped_total`)
-)
-
-// queueCap is the maximum number of pending traces waiting to be exported.
-const queueCap = 1000
-
-var (
-	queue chan *querytracer.OTLPTrace
-
-	stopCh chan struct{}
-	wg     sync.WaitGroup
-)
-
-// IsEnabled returns true when trace export is configured.
-func IsEnabled() bool {
-	return *pushURL != ""
-}
-
-// MinTraceDuration returns the configured minimum trace duration threshold.
-func MinTraceDuration() time.Duration {
-	return *minTraceDuration
-}
-
-// Init starts the background export goroutine. Must be called after flag.Parse and logger.Init.
-func Init() {
-	if !IsEnabled() {
-		return
-	}
-	queue = make(chan *querytracer.OTLPTrace, queueCap)
-	stopCh = make(chan struct{})
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		runExporter()
-	}()
-	logger.Infof("started query trace exporter to %s", *pushURL)
-}
-
-// Stop drains pending traces and stops the background goroutine.
-// Must be called during graceful shutdown before the process exits.
-func Stop() {
-	if !IsEnabled() {
-		return
-	}
-	close(stopCh)
-	wg.Wait()
-}
-
-// Push enqueues t for async export. t must be a finished tracer.
-// Traces are silently dropped if the queue is full.
-func Push(t *querytracer.Tracer) {
-	if !t.Enabled() {
-		return
-	}
-	tr := t.ToOTLPTrace()
-	if tr == nil {
-		return
-	}
-	select {
-	case queue <- tr:
-	default:
-		droppedTotal.Inc()
-	}
-}
-
-// runExporter reads from queue and sends batches to the configured endpoint.
-func runExporter() {
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-
-	var batch []*querytracer.OTLPTrace
-	for {
-		select {
-		case tr := <-queue:
-			batch = append(batch, tr)
-			if len(batch) >= 100 {
-				flushBatch(batch)
-				batch = batch[:0]
-			}
-		case <-ticker.C:
-			if len(batch) > 0 {
-				flushBatch(batch)
-				batch = batch[:0]
-			}
-		case <-stopCh:
-			// Drain remaining items.
-			for {
-				select {
-				case tr := <-queue:
-					batch = append(batch, tr)
-				default:
-					if len(batch) > 0 {
-						flushBatch(batch)
-					}
-					return
-				}
-			}
-		}
-	}
-}
-
-// flushBatch serializes and POSTs a batch of traces.
-func flushBatch(batch []*querytracer.OTLPTrace) {
-	data, err := marshalBatch(batch)
-	if err != nil {
-		logger.Errorf("cannot marshal trace batch: %s", err)
-		errorsTotal.Inc()
-		return
-	}
-	if err := postData(data); err != nil {
-		logger.Warnf("cannot export query traces to %s: %s", *pushURL, err)
-		errorsTotal.Inc()
-		return
-	}
-	pushesTotal.Add(len(batch))
-}
-
-// marshalBatch packs all traces into a single ExportTraceServiceRequest and gzip-compresses it.
-func marshalBatch(batch []*querytracer.OTLPTrace) ([]byte, error) {
-	svcVersion := buildinfo.Version
-
-	// Build resource attributes once (same for all spans in this process).
-	resAttrs := []*keyValue{
-		{Key: "service.name", Value: anyValue{StringValue: "victoriametrics"}},
-		{Key: "service.version", Value: anyValue{StringValue: svcVersion}},
-	}
-
-	var pbSpans []*span
-	for _, tr := range batch {
-		for i := range tr.Spans {
-			s := &tr.Spans[i]
-			pbSpan := &span{
-				TraceID:           s.TraceID,
-				SpanID:            s.SpanID,
-				ParentSpanID:      s.ParentSpanID,
-				Name:              s.Name,
-				StartTimeUnixNano: s.StartNano,
-				EndTimeUnixNano:   s.EndNano,
-			}
-			for _, ev := range s.Events {
-				pbSpan.Events = append(pbSpan.Events, &spanEvent{
-					TimeUnixNano: ev.TimeNano,
-					Name:         ev.Name,
-				})
-			}
-			pbSpans = append(pbSpans, pbSpan)
-		}
-	}
-
-	req := &exportTraceServiceRequest{
-		ResourceSpans: []*resourceSpans{
-			{
-				Resource: resource{Attributes: resAttrs},
-				ScopeSpans: []*scopeSpans{
-					{
-						Scope: instrumentationScope{
-							Name:    "victoriametrics/querytracer",
-							Version: svcVersion,
-						},
-						Spans: pbSpans,
-					},
-				},
-			},
-		},
-	}
-
-	raw := req.marshalProtobuf(nil)
-
-	var buf bytes.Buffer
-	gz := gzip.NewWriter(&buf)
-	if _, err := gz.Write(raw); err != nil {
-		return nil, fmt.Errorf("cannot gzip trace data: %w", err)
-	}
-	if err := gz.Close(); err != nil {
-		return nil, fmt.Errorf("cannot close gzip writer: %w", err)
-	}
-	return buf.Bytes(), nil
-}
-
-var httpClient = &http.Client{
-	Timeout: 10 * time.Second,
-}
-
-// postData sends gzip-compressed protobuf data to the configured endpoint.
-func postData(data []byte) error {
-	req, err := http.NewRequest(http.MethodPost, *pushURL, bytes.NewReader(data))
-	if err != nil {
-		return fmt.Errorf("cannot create HTTP request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/x-protobuf")
-	req.Header.Set("Content-Encoding", "gzip")
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("HTTP request failed: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 != 2 {
-		return fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
-	}
-	return nil
-}

lib/uint64set/uint64set_timing_test.go

@@ -5,14 +5,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/RoaringBitmap/roaring/v2/roaring64"
 	"github.com/valyala/fastrand"
 )
 
 func BenchmarkAddMulti(b *testing.B) {
 	for _, itemsCount := range []int{1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7} {
 		start := uint64(time.Now().UnixNano())
-		sa := createRangeSet(start, itemsCount)
-		a := sa.AppendTo(nil)
+		a := createRangeSet(start, itemsCount).ToArray()
 		b.Run(fmt.Sprintf("items_%d", itemsCount), func(b *testing.B) {
 			benchmarkAddMulti(b, a)
 		})
@@ -22,8 +22,7 @@ func BenchmarkAddMulti(b *testing.B) {
 func BenchmarkAdd(b *testing.B) {
 	for _, itemsCount := range []int{1e3, 1e4, 1e5, 1e6, 1e7} {
 		start := uint64(time.Now().UnixNano())
-		sa := createRangeSet(start, itemsCount)
-		a := sa.AppendTo(nil)
+		a := createRangeSet(start, itemsCount).ToArray()
 		b.Run(fmt.Sprintf("items_%d", itemsCount), func(b *testing.B) {
 			benchmarkAdd(b, a)
 		})
@@ -68,7 +67,7 @@ func benchmarkAdd(b *testing.B, a []uint64) {
 	b.SetBytes(int64(len(a)))
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			var s Set
+			s := roaring64.New()
 			for _, x := range a {
 				s.Add(x)
 			}
@@ -81,26 +80,26 @@ func benchmarkAddMulti(b *testing.B, a []uint64) {
 	b.SetBytes(int64(len(a)))
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			var s Set
+			s := roaring64.New()
 			n := 0
 			for n < len(a) {
 				m := min(n+64, len(a))
-				s.AddMulti(a[n:m])
+				s.AddMany(a[n:m])
 				n = m
 			}
 		}
 	})
 }
 
-func benchmarkUnion(b *testing.B, sa, sb *Set) {
+func benchmarkUnion(b *testing.B, sa, sb *roaring64.Bitmap) {
 	b.ReportAllocs()
-	b.SetBytes(int64(sa.Len() + sb.Len()))
+	b.SetBytes(int64(sa.Stats().Cardinality + sb.Stats().Cardinality))
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			saCopy := sa.Clone()
 			sbCopy := sb.Clone()
-			saCopy.Union(sb)
-			sbCopy.Union(sa)
+			saCopy.Or(sb)
+			sbCopy.Or(sa)
 		}
 	})
 }
@@ -138,15 +137,15 @@ func BenchmarkIntersectFullOverlap(b *testing.B) {
 	}
 }
 
-func benchmarkIntersect(b *testing.B, sa, sb *Set) {
+func benchmarkIntersect(b *testing.B, sa, sb *roaring64.Bitmap) {
 	b.ReportAllocs()
-	b.SetBytes(int64(sa.Len() + sb.Len()))
+	b.SetBytes(int64(sa.Stats().Cardinality + sb.Stats().Cardinality))
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			saCopy := sa.Clone()
 			sbCopy := sb.Clone()
-			saCopy.Intersect(sb)
-			sbCopy.Intersect(sa)
+			saCopy.And(sb)
+			sbCopy.And(sa)
 		}
 	})
 }
@@ -156,10 +155,10 @@ func BenchmarkSubtract(b *testing.B) {
 		sa := createRangeSet(startA, int(itemsCountA))
 		sb := createRangeSet(startB, int(itemsCountB))
 		b.ReportAllocs()
-		b.SetBytes(int64(sa.Len() + sb.Len()))
+		b.SetBytes(int64(sa.Stats().Cardinality + sb.Stats().Cardinality))
 		for b.Loop() {
 			saCopy := sa.Clone()
-			saCopy.Subtract(sb)
+			saCopy.AndNot(sb)
 		}
 	}
 
@@ -211,13 +210,13 @@ func BenchmarkSubtract(b *testing.B) {
 	}
 }
 
-func createRangeSet(start uint64, itemsCount int) *Set {
-	var s Set
+func createRangeSet(start uint64, itemsCount int) *roaring64.Bitmap {
+	s := roaring64.New()
 	for i := range itemsCount {
 		n := start + uint64(i)
 		s.Add(n)
 	}
-	return &s
+	return s
 }
 
 func BenchmarkSetAddRandomLastBits(b *testing.B) {
@@ -231,7 +230,7 @@ func BenchmarkSetAddRandomLastBits(b *testing.B) {
 				var rng fastrand.RNG
 				for pb.Next() {
 					start := uint64(time.Now().UnixNano())
-					var s Set
+					s := roaring64.New()
 					for range int(itemsCount) {
 						n := start | (uint64(rng.Uint32()) & mask)
 						s.Add(n)
@@ -273,7 +272,7 @@ func BenchmarkSetAddWithAllocs(b *testing.B) {
 				for pb.Next() {
 					start := uint64(time.Now().UnixNano())
 					end := start + itemsCount
-					var s Set
+					s := roaring64.New()
 					n := start
 					for n < end {
 						s.Add(n)
@@ -357,13 +356,13 @@ func BenchmarkSetHasHitRandomLastBits(b *testing.B) {
 		mask := (uint64(1) << lastBits) - 1
 		b.Run(fmt.Sprintf("lastBits_%d", lastBits), func(b *testing.B) {
 			start := uint64(time.Now().UnixNano())
-			var s Set
+			s := roaring64.New()
 			var rng fastrand.RNG
 			for range int(itemsCount) {
 				n := start | (uint64(rng.Uint32()) & mask)
 				s.Add(n)
 			}
-			a := s.AppendTo(nil)
+			a := s.ToArray()
 
 			b.ResetTimer()
 			b.ReportAllocs()
@@ -371,7 +370,7 @@ func BenchmarkSetHasHitRandomLastBits(b *testing.B) {
 			b.RunParallel(func(pb *testing.PB) {
 				for pb.Next() {
 					for _, n := range a {
-						if !s.Has(n) {
+						if !s.Contains(n) {
 							panic("unexpected miss")
 						}
 					}
@@ -415,7 +414,7 @@ func BenchmarkSetHasHit(b *testing.B) {
 		b.Run(fmt.Sprintf("items_%d", itemsCount), func(b *testing.B) {
 			start := uint64(time.Now().UnixNano())
 			end := start + itemsCount
-			var s Set
+			s := roaring64.New()
 			n := start
 			for n < end {
 				s.Add(n)
@@ -429,7 +428,7 @@ func BenchmarkSetHasHit(b *testing.B) {
 				for pb.Next() {
 					n := start
 					for n < end {
-						if !s.Has(n) {
+						if !s.Contains(n) {
 							panic("unexpected miss")
 						}
 						n++
@@ -475,7 +474,7 @@ func BenchmarkSetHasMiss(b *testing.B) {
 		b.Run(fmt.Sprintf("items_%d", itemsCount), func(b *testing.B) {
 			start := uint64(time.Now().UnixNano())
 			end := start + itemsCount
-			var s Set
+			s := roaring64.New()
 			n := start
 			for n < end {
 				s.Add(n)
@@ -490,7 +489,7 @@ func BenchmarkSetHasMiss(b *testing.B) {
 					n := end
 					nEnd := end + itemsCount
 					for n < nEnd {
-						if s.Has(n) {
+						if s.Contains(n) {
 							panic("unexpected hit")
 						}
 						n++
@@ -531,3 +530,62 @@ func BenchmarkMapHasMiss(b *testing.B) {
 		})
 	}
 }
+
+func BenchmarkSizeBytes_uint64slice(b *testing.B) {
+	benchmarkSizeBytes(b, func(start, n, step uint64) uint64 {
+		s := []uint64{}
+		for i := range n {
+			v := start + i*step
+			s = append(s, v)
+		}
+		return uint64(len(s) * 8)
+	})
+}
+
+func BenchmarkSizeBytes_uint64set(b *testing.B) {
+	benchmarkSizeBytes(b, func(start, n, step uint64) uint64 {
+		s := &Set{}
+		for i := range n {
+			v := start + i*step
+			s.Add(v)
+		}
+		return s.SizeBytes()
+	})
+}
+
+func BenchmarkSizeBytes_roaring(b *testing.B) {
+	benchmarkSizeBytes(b, func(start, n, step uint64) uint64 {
+		s := roaring64.New()
+		for i := range n {
+			v := start + i*step
+			s.Add(v)
+		}
+		stats := s.Stats()
+		sizeBytes := stats.ArrayContainerBytes
+		sizeBytes += stats.BitmapContainerBytes
+		sizeBytes += stats.RunContainerBytes
+		return sizeBytes
+	})
+}
+
+func benchmarkSizeBytes(b *testing.B, sizeBytesFunc func(start, n, step uint64) uint64) {
+	f := func(b *testing.B, n, step uint64) {
+		start := uint64(time.Now().UnixNano())
+		var sizeBytes uint64
+		for b.Loop() {
+			sizeBytes = sizeBytesFunc(start, n, step)
+		}
+
+		b.ReportAllocs()
+		b.ReportMetric(float64(sizeBytes), "bytes")
+	}
+
+	for _, n := range []uint64{15_000_000} {
+		for _, step := range []uint64{1, 10, 100, 1e3, 1e4, 1e5, 1e6} {
+			name := fmt.Sprintf("%d/%d", n, step)
+			b.Run(name, func(b *testing.B) {
+				f(b, n, step)
+			})
+		}
+	}
+}

lib/writeconcurrencylimiter/concurrencylimiter.go

@@ -10,16 +10,18 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	maxConcurrentInserts = flag.Int("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent insert requests. "+
-		"Set higher value when clients send data over slow networks. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. "+
-		"See also -insert.maxQueueDuration")
+	maxConcurrentInserts = flagutil.NewIntWithDynamicDefault("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "2*cgroup.AvailableCPUs()",
+		"The maximum number of concurrent insert requests. "+
+			"Set higher value when clients send data over slow networks. "+
+			"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. "+
+			"See also -insert.maxQueueDuration")
 	maxQueueDuration = flag.Duration("insert.maxQueueDuration", time.Minute, "The maximum duration to wait in the queue when -maxConcurrentInserts "+
 		"concurrent insert requests are executed")
 )

vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml

@@ -0,0 +1,19 @@
+kind: pipeline
+name: default
+
+workspace:
+  base: /go
+  path: src/github.com/RoaringBitmap/roaring
+
+steps:
+- name: test
+  image: golang
+  commands:
+  - go get -t 
+  - go test 
+  - go build -tags appengine 
+  - go test -tags appengine 
+  - GOARCH=386 go build
+  - GOARCH=386 go test
+  - GOARCH=arm go build
+  - GOARCH=arm64 go build

vendor/github.com/RoaringBitmap/roaring/v2/.gitignore

@@ -0,0 +1,6 @@
+*~
+roaring-fuzz.zip
+workdir
+coverage.out
+testdata/all3.classic
+/vendor

vendor/github.com/RoaringBitmap/roaring/v2/AGENTS.md

@@ -0,0 +1,36 @@
+# Agent Guidance for roaring (Go)
+
+## AI-Generated Bug Reports About Deserialization
+
+Many AI-generated bug reports claim that deserialization functions (e.g., `ReadFrom`, `FromBuffer`, `FromUnsafeBytes`, `UnmarshalBinary`, and their `roaring64` equivalents) "trigger bugs", "cause crashes", or "panic" when given malformed or untrusted input.
+
+**These reports are bogus.**
+
+The library's documented contract is explicit and consistent across the public API and README:
+
+- The deserialization functions are memory-safe in the ordinary Go sense: they will not read out of bounds of the slices they are given.
+- However, if the input does not conform to the format specification (i.e., it is garbage or from an untrusted source), the resulting bitmap may be in an invalid internal state. Using such a bitmap without validation can lead to panics, incorrect results, or excessive memory consumption.
+- **If the source is untrusted, you MUST call `Validate()` on the result and only proceed if it returns `nil`. Only after a successful validation is the bitmap considered safe for use.** Equivalently, `MustReadFrom` calls `ReadFrom` followed by `Validate` and panics on an invalid bitmap.
+
+Relevant documentation locations:
+
+- roaring.go: doc comments on `ReadFrom`, `MustReadFrom`, `FromBuffer`, `FromUnsafeBytes`, `UnmarshalBinary`, and `Validate`.
+- roaring64/roaring64.go: equivalent `ReadFrom`, `UnmarshalBinary`, and `Validate` for 64-bit bitmaps.
+- README.md (the serialization example, around lines 250-272): sample code showing the required validate-after-deserialize pattern for untrusted input, with the explicit comment: "if buf is an untrusted source, you should validate the result".
+
+The format specification is documented at https://github.com/RoaringBitmap/RoaringFormatSpec. The Go, Java, C and C++ implementations are binary compatible.
+
+A special note on `FromBuffer` and `FromUnsafeBytes`: these are zero-copy entry points (for advanced users only). The resulting bitmap holds references into the caller-provided byte slice and uses copy-on-write. It is the caller's responsibility to ensure that the input slice is not modified and remains valid for the lifetime of the bitmap (and of any bitmap derived from it). Reports that mutate the backing slice after the fact, or that free/reuse it while the bitmap is still alive, are describing documented misuse, not a library bug.
+
+The fuzzing harnesses (`FuzzSmat`, `FuzzSerializationBuffer`, `FuzzSerializationStream`, and the corpus under `testdata/fuzz/`) and the property tests exist precisely to ensure the deserializers and the validator behave correctly under adversarial input. A report that treats "deserializing attacker-controlled bytes and then using the result without calling `Validate()`" as a bug in the deserializer is a misunderstanding of the stated API contract.
+
+When triaging such reports, point to the validation requirement in the function documentation and the README example, and close as "not a bug / user error / documented behavior."
+
+## Building, Testing, and Conventions
+
+- This is a pure-Go library; there is no cgo. Use the standard Go toolchain.
+- Run the test suite with `go test ./...` (the root package and the `roaring64` subpackage both have extensive tests).
+- The `Makefile` exposes `make unconvert`, which runs `go tool unconvert -apply ./...` to remove unnecessary type conversions. Run it before proposing changes that touch type conversions.
+- The root package implements 32-bit bitmaps; `roaring64/` implements 64-bit bitmaps and should be kept behaviorally consistent with the root package.
+- Architecture-specific files exist for performance (e.g., `popcnt_amd64.s`, `setutil_arm64.s`, with `_generic.go` and `_compat.go` fallbacks). Any change to one path must be mirrored in the generic fallback so all build targets stay correct.
+- Keep new code consistent with the surrounding style: match existing naming, error handling, and comment density. Public API changes must update doc comments and, where relevant, the README.

vendor/github.com/RoaringBitmap/roaring/v2/AUTHORS

@@ -0,0 +1,11 @@
+# This is the official list of roaring authors for copyright purposes. 
+
+Todd Gruben (@tgruben),
+Daniel Lemire (@lemire),
+Elliot Murphy (@statik),
+Bob Potter (@bpot),
+Tyson Maly (@tvmaly),
+Will Glynn (@willglynn),
+Brent Pedersen (@brentp)
+Maciej Biłas (@maciej),
+Joe Nall (@joenall)

vendor/github.com/RoaringBitmap/roaring/v2/CONTRIBUTORS

@@ -0,0 +1,18 @@
+# This is the official list of roaring contributors 
+
+Todd Gruben (@tgruben),
+Daniel Lemire (@lemire),
+Elliot Murphy (@statik),
+Bob Potter (@bpot),
+Tyson Maly (@tvmaly),
+Will Glynn (@willglynn),
+Brent Pedersen (@brentp),
+Jason E. Aten (@glycerine),
+Vali Malinoiu (@0x4139),
+Forud Ghafouri (@fzerorubigd),
+Joe Nall (@joenall),
+(@fredim),
+Edd Robinson (@e-dard),
+Alexander Petrov (@alldroll),
+Guy Molinari (@guymolinari),
+Ling Jin (@JinLingChristopher)

vendor/github.com/RoaringBitmap/roaring/v2/LICENSE

@@ -0,0 +1,235 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 by the authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================================
+
+Portions of runcontainer.go are from the Go standard library, which is licensed
+under:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+     copyright notice, this list of conditions and the following disclaimer
+     in the documentation and/or other materials provided with the
+     distribution.
+   * Neither the name of Google Inc. nor the names of its
+     contributors may be used to endorse or promote products derived from
+     this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/RoaringBitmap/roaring/v2/LICENSE-2.0.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 by the authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/RoaringBitmap/roaring/v2/Makefile

@@ -0,0 +1,10 @@
+# Display general help about this command
+help:
+	@echo ""
+	@echo "The following commands are available:"
+	@echo "    make unconvert   : Find unnecessary type conversions"
+	@echo ""
+
+# Find unnecessary type conversions
+unconvert:
+	go tool unconvert -apply ./...

vendor/github.com/RoaringBitmap/roaring/v2/README.md

@@ -0,0 +1,428 @@
+# roaring 
+
+[![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring)
+
+![Go-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-CI/badge.svg)
+![Go-ARM-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-ARM-CI/badge.svg)
+![Go-Windows-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-Windows-CI/badge.svg)
+=============
+
+This is a go version of the Roaring bitmap data structure. 
+
+Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and
+[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas],  [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Redpanda](https://github.com/redpanda-data/redpanda), [Pilosa][pilosa],  [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing.
+
+[lucene]: https://lucene.apache.org/
+[solr]: https://lucene.apache.org/solr/
+[elasticsearch]: https://www.elastic.co/products/elasticsearch
+[druid]: https://druid.apache.org/
+[spark]: https://spark.apache.org/
+[opensearchserver]: http://www.opensearchserver.com
+[anacrolix/torrent]: https://github.com/anacrolix/torrent
+[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home
+[pilosa]: https://www.pilosa.com/
+[kylin]: http://kylin.apache.org/
+[pinot]: http://github.com/linkedin/pinot/wiki
+[vsts]: https://www.visualstudio.com/team-services/
+[atlas]: https://github.com/Netflix/atlas
+[quanta]: https://github.com/disney/quanta
+
+Roaring bitmaps are found to work well in many important applications:
+
+> Use Roaring for bitmap compression whenever possible. Do not use other bitmap compression methods ([Wang et al., SIGMOD 2017](http://db.ucsd.edu/wp-content/uploads/2017/03/sidm338-wangA.pdf))
+
+
+The ``roaring`` Go library is used by
+* [anacrolix/torrent]
+* [InfluxDB](https://www.influxdata.com)
+* [Pilosa](https://www.pilosa.com/)
+* [Bleve](http://www.blevesearch.com)
+* [Weaviate](https://github.com/weaviate/weaviate)
+* [lindb](https://github.com/lindb/lindb)
+* [Elasticell](https://github.com/deepfabric/elasticell)
+* [SourceGraph](https://github.com/sourcegraph/sourcegraph)
+* [M3](https://github.com/m3db/m3)
+* [trident](https://github.com/NetApp/trident)
+* [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/)
+* [FrostDB](https://github.com/polarsignals/frostdb)
+* [Disney Quanta](https://github.com/disney/quanta)
+
+
+
+
+This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com).
+
+
+There are also  [Java](https://github.com/RoaringBitmap/RoaringBitmap) and [C/C++](https://github.com/RoaringBitmap/CRoaring) versions.  The Java, C, C++ and Go version are binary compatible: e.g,  you can save bitmaps
+from a Java program and load them back in Go, and vice versa. We have a [format specification](https://github.com/RoaringBitmap/RoaringFormatSpec).
+
+
+This code is licensed under Apache License, Version 2.0 (ASL2.0).
+
+Copyright 2016-... by the authors.
+
+When should you use a bitmap?
+===================================
+
+
+Sets are a fundamental abstraction in
+software. They can be implemented in various
+ways, as hash sets, as trees, and so forth.
+In databases and search engines, sets are often an integral
+part of indexes. For example, we may need to maintain a set
+of all documents or rows  (represented by numerical identifier)
+that satisfy some property. Besides adding or removing
+elements from the set, we need fast functions
+to compute the intersection, the union, the difference between sets, and so on.
+
+
+To implement a set
+of integers, a particularly appealing strategy is the
+bitmap (also called bitset or bit vector). Using n bits,
+we can represent any set made of the integers from the range
+[0,n): the ith bit is set to one if integer i is present in the set.
+Commodity processors use words of W=32 or W=64 bits. By combining many such words, we can
+support large values of n. Intersections, unions and differences can then be implemented
+ as bitwise AND, OR and ANDNOT operations.
+More complicated set functions can also be implemented as bitwise operations.
+
+When the bitset approach is applicable, it can be orders of
+magnitude faster than other possible implementation of a set (e.g., as a hash set)
+while using several times less memory.
+
+However, a bitset, even a compressed one is not always applicable. For example, if
+you have 1000 random-looking integers, then a simple array might be the best representation.
+We refer to this case as the "sparse" scenario.
+
+When should you use compressed bitmaps?
+===================================
+
+An uncompressed BitSet can use a lot of memory. For example, if you take a BitSet
+and set the bit at position 1,000,000 to true and you have just over 100kB. That is over 100kB
+to store the position of one bit. This is wasteful  even if you do not care about memory:
+suppose that you need to compute the intersection between this BitSet and another one
+that has a bit at position 1,000,001 to true, then you need to go through all these zeroes,
+whether you like it or not. That can become very wasteful.
+
+This being said, there are definitively cases where attempting to use compressed bitmaps is wasteful.
+For example, if you have a small universe size. E.g., your bitmaps represent sets of integers
+from [0,n) where n is small (e.g., n=64 or n=128). If you can use uncompressed BitSet and
+it does not blow up your memory usage,  then compressed bitmaps are probably not useful
+to you. In fact, if you do not need compression, then a BitSet offers remarkable speed.
+
+The sparse scenario is another use case where compressed bitmaps should not be used.
+Keep in mind that random-looking data is usually not compressible. E.g., if you have a small set of
+32-bit random integers, it is not mathematically possible to use far less than 32 bits per integer,
+and attempts at compression can be counterproductive.
+
+How does Roaring compares with the alternatives?
+==================================================
+
+
+Most alternatives to Roaring are part of a larger family of compressed bitmaps that are run-length-encoded
+bitmaps. They identify long runs of 1s or 0s and they represent them with a marker word.
+If you have a local mix of 1s and 0, you use an uncompressed word.
+
+There are many formats in this family:
+
+* Oracle's BBC is an obsolete format at this point: though it may provide good compression,
+it is likely much slower than more recent alternatives due to excessive branching.
+* WAH is a patented variation on BBC that provides better performance.
+* Concise is a variation on the patented WAH. It some specific instances, it can compress
+much better than WAH (up to 2x better), but it is generally slower.
+* EWAH is both free of patent, and it is faster than all the above. On the downside, it
+does not compress quite as well. It is faster because it allows some form of "skipping"
+over uncompressed words. So though none of these formats are great at random access, EWAH
+is better than the alternatives.
+
+
+
+There is a big problem with these formats however that can hurt you badly in some cases: there is no random access. If you want to check whether a given value is present in the set, you have to start from the beginning and "uncompress" the whole thing. This means that if you want to intersect a big set with a large set, you still have to uncompress the whole big set in the worst case...
+
+Roaring solves this problem. It works in the following manner. It divides the data into chunks of 2<sup>16</sup> integers
+(e.g., [0, 2<sup>16</sup>), [2<sup>16</sup>, 2 x 2<sup>16</sup>), ...). Within a chunk, it can use an uncompressed bitmap, a simple list of integers,
+or a list of runs. Whatever format it uses, they all allow you to check for the presence of any one value quickly
+(e.g., with a binary search). The net result is that Roaring can compute many operations much faster than run-length-encoded
+formats like WAH, EWAH, Concise... Maybe surprisingly, Roaring also generally offers better compression ratios.
+
+
+
+
+
+### References
+
+- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821)
+-  Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin,
+Better bitmap performance with Roaring bitmaps,
+Software: Practice and Experience 46 (5), 2016.[arXiv:1402.6407](http://arxiv.org/abs/1402.6407) This paper used data from http://lemire.me/data/realroaring2014.html
+- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. [arXiv:1603.06549](http://arxiv.org/abs/1603.06549)
+
+### Dependencies
+
+Dependencies are fetched automatically by giving the `-t` flag to `go get`.
+
+they include
+  - github.com/bits-and-blooms/bitset
+  - github.com/mschoch/smat
+  - github.com/glycerine/go-unsnap-stream
+  - github.com/philhofer/fwd
+  - github.com/jtolds/gls
+
+Note that the smat library requires Go 1.15 or better.
+
+#### Installation
+
+  - go get -t github.com/RoaringBitmap/roaring
+
+### Instructions for contributors
+
+Using bash or other common shells:
+```
+$ git clone git@github.com:RoaringBitmap/roaring.git
+$ export GO111MODULE=on 
+$ go mod tidy
+$ go test -v
+```
+
+### Example
+
+Here is a simplified but complete example:
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/RoaringBitmap/roaring/v2"
+    "bytes"
+)
+
+
+func main() {
+    // example inspired by https://github.com/fzandona/goroar
+    fmt.Println("==roaring==")
+    rb1 := roaring.BitmapOf(1, 2, 3, 4, 5, 100, 1000)
+    fmt.Println(rb1.String())
+
+    rb2 := roaring.BitmapOf(3, 4, 1000)
+    fmt.Println(rb2.String())
+
+    rb3 := roaring.New()
+    fmt.Println(rb3.String())
+
+    fmt.Println("Cardinality: ", rb1.GetCardinality())
+
+    fmt.Println("Contains 3? ", rb1.Contains(3))
+
+    rb1.And(rb2)
+
+    rb3.Add(1)
+    rb3.Add(5)
+
+    rb3.Or(rb1)
+
+    // computes union of the three bitmaps in parallel using 4 workers  
+    roaring.ParOr(4, rb1, rb2, rb3)
+    // computes intersection of the three bitmaps in parallel using 4 workers  
+    roaring.ParAnd(4, rb1, rb2, rb3)
+
+
+    // prints 1, 3, 4, 5, 1000
+    i := rb3.Iterator()
+    for i.HasNext() {
+        fmt.Println(i.Next())
+    }
+    fmt.Println()
+
+    // next we include an example of serialization
+    buf := new(bytes.Buffer)
+    rb1.WriteTo(buf) // we omit error handling
+    newrb:= roaring.New()
+    newrb.ReadFrom(buf)
+    if rb1.Equals(newrb) {
+    	fmt.Println("I wrote the content to a byte stream and read it back.")
+    }
+    // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
+}
+```
+
+If you wish to use serialization and handle errors, you might want to
+consider the following sample of code:
+
+```go
+	rb := BitmapOf(1, 2, 3, 4, 5, 100, 1000)
+	buf := new(bytes.Buffer)
+	size,err:=rb.WriteTo(buf)
+	if err != nil {
+		fmt.Println("Failed writing") // return or panic
+	}
+	newrb:= New()
+	size,err=newrb.ReadFrom(buf)
+	if err != nil {
+		fmt.Println("Failed reading") // return or panic
+	}
+	// if buf is an untrusted source, you should validate the result
+	// (this adds a bit of complexity but it is necessary for security)
+	if newrb.Validate() != nil {
+		fmt.Println("Failed validation") // return or panic
+	}
+	if ! rb.Equals(newrb) {
+		fmt.Println("Cannot retrieve serialized version")
+	}
+```
+
+Given N integers in [0,x), then the serialized size in bytes of
+a Roaring bitmap should never exceed this bound:
+
+`` 8 + 9 * ((long)x+65535)/65536 + 2 * N ``
+
+That is, given a fixed overhead for the universe size (x), Roaring
+bitmaps never use more than 2 bytes per integer. You can call
+``BoundSerializedSizeInBytes`` for a more precise estimate.
+
+### 64-bit Roaring
+
+By default, roaring is used to stored unsigned 32-bit integers. However, we also offer
+an extension dedicated to 64-bit integers. It supports roughly the same functions:
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/RoaringBitmap/roaring/v2/roaring64"
+    "bytes"
+)
+
+
+func main() {
+    // example inspired by https://github.com/fzandona/goroar
+    fmt.Println("==roaring64==")
+    rb1 := roaring64.BitmapOf(1, 2, 3, 4, 5, 100, 1000)
+    fmt.Println(rb1.String())
+
+    rb2 := roaring64.BitmapOf(3, 4, 1000)
+    fmt.Println(rb2.String())
+
+    rb3 := roaring64.New()
+    fmt.Println(rb3.String())
+
+    fmt.Println("Cardinality: ", rb1.GetCardinality())
+
+    fmt.Println("Contains 3? ", rb1.Contains(3))
+
+    rb1.And(rb2)
+
+    rb3.Add(1)
+    rb3.Add(5)
+
+    rb3.Or(rb1)
+
+
+
+    // prints 1, 3, 4, 5, 1000
+    i := rb3.Iterator()
+    for i.HasNext() {
+        fmt.Println(i.Next())
+    }
+    fmt.Println()
+
+    // next we include an example of serialization
+    buf := new(bytes.Buffer)
+    rb1.WriteTo(buf) // we omit error handling
+    newrb:= roaring64.New()
+    newrb.ReadFrom(buf)
+    if rb1.Equals(newrb) {
+    	fmt.Println("I wrote the content to a byte stream and read it back.")
+    }
+    // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
+}
+```
+
+Only the 32-bit roaring format is standard and cross-operable between Java, C++, C and Go. There is no guarantee that the 64-bit versions are compatible.
+
+### Documentation
+
+Current documentation is available at https://pkg.go.dev/github.com/RoaringBitmap/roaring and https://pkg.go.dev/github.com/RoaringBitmap/roaring/roaring64
+
+### Goroutine safety
+
+In general, it should not generally be considered safe to access
+the same bitmaps using different goroutines--they are left
+unsynchronized for performance. Should you want to access
+a Bitmap from more than one goroutine, you should
+provide synchronization. Typically this is done by using channels to pass
+the *Bitmap around (in Go style; so there is only ever one owner),
+or by using `sync.Mutex` to serialize operations on Bitmaps.
+
+### Coverage
+
+We test our software. For a report on our test coverage, see
+
+https://coveralls.io/github/RoaringBitmap/roaring?branch=master
+
+### Benchmark
+
+Type
+
+         go test -bench Benchmark -run -
+
+To run benchmarks on [Real Roaring Datasets](https://github.com/RoaringBitmap/real-roaring-datasets)
+run the following:
+
+```sh
+go get github.com/RoaringBitmap/real-roaring-datasets
+BENCH_REAL_DATA=1 go test -bench BenchmarkRealData -run -
+```
+
+### Interactive use
+
+You can use roaring with gore:
+
+- go install github.com/x-motemen/gore/cmd/gore@latest
+- Make sure that ``$GOPATH/bin`` is in your ``$PATH``.
+
+```go
+$ gore
+gore version 0.2.6  :help for help
+gore> :import github.com/RoaringBitmap/roaring
+gore> x:=roaring.New()
+gore> x.Add(1)
+gore> x.String()
+"{1}"
+```
+
+
+### Fuzzy testing
+
+You can help us test further the library with fuzzy testing:
+
+         go get github.com/dvyukov/go-fuzz/go-fuzz
+         go get github.com/dvyukov/go-fuzz/go-fuzz-build
+         go test -tags=gofuzz -run=TestGenerateSmatCorpus
+         go-fuzz-build github.com/RoaringBitmap/roaring
+         go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -func FuzzSmat
+
+Let it run, and if the # of crashers is > 0, check out the reports in
+the workdir where you should be able to find the panic goroutine stack
+traces.
+
+You may also replace `-func FuzzSmat`  by `-func FuzzSerializationBuffer` or `-func FuzzSerializationStream`.
+
+### Alternative in Go
+
+There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring
+
+For an alternative implementation in Go, see https://github.com/fzandona/goroar
+The two versions were written independently.
+
+
+### Mailing list/discussion group
+
+https://groups.google.com/g/roaring-bitmaps
+
+
+### Further reading
+
+<p>Mastering Programming: From Testing to Performance in Go</p>
+<div><a href="https://www.amazon.com/dp/B0FMPGSWR5"><img style="margin-left: auto; margin-right: auto;" src="https://m.media-amazon.com/images/I/61feneHS7kL._SL1499_.jpg" alt="" width="250px" /></a></div>

vendor/github.com/RoaringBitmap/roaring/v2/fastaggregation.go

@@ -0,0 +1,320 @@
+package roaring
+
+import (
+	"container/heap"
+)
+
+// Or function that requires repairAfterLazy
+func lazyOR(x1, x2 *Bitmap) *Bitmap {
+	answer := NewBitmap()
+	pos1 := 0
+	pos2 := 0
+	length1 := x1.highlowcontainer.size()
+	length2 := x2.highlowcontainer.size()
+main:
+	for (pos1 < length1) && (pos2 < length2) {
+		s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
+		s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
+
+		for {
+			if s1 < s2 {
+				answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
+				pos1++
+				if pos1 == length1 {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+			} else if s1 > s2 {
+				answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
+				pos2++
+				if pos2 == length2 {
+					break main
+				}
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			} else {
+				c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
+				answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
+				pos1++
+				pos2++
+				if (pos1 == length1) || (pos2 == length2) {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			}
+		}
+	}
+	if pos1 == length1 {
+		answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
+	} else if pos2 == length2 {
+		answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
+	}
+	return answer
+}
+
+// In-place Or function that requires repairAfterLazy
+func (x1 *Bitmap) lazyOR(x2 *Bitmap) *Bitmap {
+	pos1 := 0
+	pos2 := 0
+	length1 := x1.highlowcontainer.size()
+	length2 := x2.highlowcontainer.size()
+main:
+	for (pos1 < length1) && (pos2 < length2) {
+		s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
+		s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
+
+		for {
+			if s1 < s2 {
+				pos1++
+				if pos1 == length1 {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+			} else if s1 > s2 {
+				x1.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone())
+				pos2++
+				pos1++
+				length1++
+				if pos2 == length2 {
+					break main
+				}
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			} else {
+				c1 := x1.highlowcontainer.getWritableContainerAtIndex(pos1)
+				// runContainer16.lazyIOR falls back to a slow ior path
+				// (O(N log R) per merged element); promote to bitmapContainer
+				// first, whose lazy union is O(1024) regardless of cardinality.
+				// See https://github.com/RoaringBitmap/roaring/issues/81.
+				if rc, ok := c1.(*runContainer16); ok && !rc.isFull() {
+					c1 = rc.toBitmapContainer()
+				}
+				x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2))
+				x1.highlowcontainer.needCopyOnWrite[pos1] = false
+				pos1++
+				pos2++
+				if (pos1 == length1) || (pos2 == length2) {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			}
+		}
+	}
+	if pos1 == length1 {
+		x1.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
+	}
+	return x1
+}
+
+// to be called after lazy aggregates
+func (x1 *Bitmap) repairAfterLazy() {
+	for pos := 0; pos < x1.highlowcontainer.size(); pos++ {
+		c := x1.highlowcontainer.getContainerAtIndex(pos)
+		switch c.(type) {
+		case *bitmapContainer:
+			if c.(*bitmapContainer).cardinality == invalidCardinality {
+				c = x1.highlowcontainer.getWritableContainerAtIndex(pos)
+				c.(*bitmapContainer).computeCardinality()
+				if c.(*bitmapContainer).getCardinality() <= arrayDefaultMaxSize {
+					x1.highlowcontainer.setContainerAtIndex(pos, c.(*bitmapContainer).toArrayContainer())
+				} else if c.(*bitmapContainer).isFull() {
+					x1.highlowcontainer.setContainerAtIndex(pos, newRunContainer16Range(0, MaxUint16))
+				}
+			}
+		}
+	}
+}
+
+// FastAnd computes the intersection between many bitmaps quickly
+// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
+// of manually calling "And" many times.
+//
+// Performance hints: if you have very large and tiny bitmaps,
+// it may be beneficial performance-wise to put a tiny bitmap
+// in first position.
+func FastAnd(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+	answer := And(bitmaps[0], bitmaps[1])
+	for _, bm := range bitmaps[2:] {
+		answer.And(bm)
+	}
+	return answer
+}
+
+// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly.
+// It might also be faster than calling Or repeatedly.
+func FastOr(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+	answer := lazyOR(bitmaps[0], bitmaps[1])
+	for _, bm := range bitmaps[2:] {
+		answer = answer.lazyOR(bm)
+	}
+	// here is where repairAfterLazy is called.
+	answer.repairAfterLazy()
+	return answer
+}
+
+// HeapOr computes the union between many bitmaps quickly using a heap.
+// It might be faster than calling Or repeatedly.
+func HeapOr(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	}
+	// TODO:  for better speed, we could do the operation lazily, see Java implementation
+	pq := make(priorityQueue, len(bitmaps))
+	for i, bm := range bitmaps {
+		pq[i] = &item{bm, i}
+	}
+	heap.Init(&pq)
+
+	for pq.Len() > 1 {
+		x1 := heap.Pop(&pq).(*item)
+		x2 := heap.Pop(&pq).(*item)
+		heap.Push(&pq, &item{Or(x1.value, x2.value), 0})
+	}
+	return heap.Pop(&pq).(*item).value
+}
+
+// HeapXor computes the symmetric difference between many bitmaps quickly (as opposed to calling Xor repeated).
+// Internally, this function uses a heap.
+// It might be faster than calling Xor repeatedly.
+func HeapXor(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	}
+
+	pq := make(priorityQueue, len(bitmaps))
+	for i, bm := range bitmaps {
+		pq[i] = &item{bm, i}
+	}
+	heap.Init(&pq)
+
+	for pq.Len() > 1 {
+		x1 := heap.Pop(&pq).(*item)
+		x2 := heap.Pop(&pq).(*item)
+		heap.Push(&pq, &item{Xor(x1.value, x2.value), 0})
+	}
+	return heap.Pop(&pq).(*item).value
+}
+
+// AndAny provides a result equivalent to x1.And(FastOr(bitmaps)).
+// It's optimized to minimize allocations. It also might be faster than separate calls.
+func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) {
+	if len(bitmaps) == 0 {
+		return
+	} else if len(bitmaps) == 1 {
+		x1.And(bitmaps[0])
+		return
+	}
+
+	type withPos struct {
+		bitmap *roaringArray
+		pos    int
+		key    uint16
+	}
+	filters := make([]withPos, 0, len(bitmaps))
+
+	for _, b := range bitmaps {
+		if b.highlowcontainer.size() > 0 {
+			filters = append(filters, withPos{
+				bitmap: &b.highlowcontainer,
+				pos:    0,
+				key:    b.highlowcontainer.getKeyAtIndex(0),
+			})
+		}
+	}
+
+	basePos := 0
+	intersections := 0
+	keyContainers := make([]container, 0, len(filters))
+	var (
+		tmpArray   *arrayContainer
+		tmpBitmap  *bitmapContainer
+		minNextKey uint16
+	)
+
+	for basePos < x1.highlowcontainer.size() && len(filters) > 0 {
+		baseKey := x1.highlowcontainer.getKeyAtIndex(basePos)
+
+		// accumulate containers for current key, find next minimal key in filters
+		// and exclude filters that do not have related values anymore
+		i := 0
+		maxPossibleOr := 0
+		minNextKey = MaxUint16
+		for _, f := range filters {
+			if f.key < baseKey {
+				f.pos = f.bitmap.advanceUntil(baseKey, f.pos)
+				if f.pos == f.bitmap.size() {
+					continue
+				}
+				f.key = f.bitmap.getKeyAtIndex(f.pos)
+			}
+
+			if f.key == baseKey {
+				cont := f.bitmap.getContainerAtIndex(f.pos)
+				keyContainers = append(keyContainers, cont)
+				maxPossibleOr += cont.getCardinality()
+
+				f.pos++
+				if f.pos == f.bitmap.size() {
+					continue
+				}
+				f.key = f.bitmap.getKeyAtIndex(f.pos)
+			}
+
+			minNextKey = minOfUint16(minNextKey, f.key)
+			filters[i] = f
+			i++
+		}
+		filters = filters[:i]
+
+		if len(keyContainers) == 0 {
+			basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
+			continue
+		}
+
+		var ored container
+
+		if len(keyContainers) == 1 {
+			ored = keyContainers[0]
+		} else {
+			//TODO: special case for run containers?
+			if maxPossibleOr > arrayDefaultMaxSize {
+				if tmpBitmap == nil {
+					tmpBitmap = newBitmapContainer()
+				}
+				tmpBitmap.resetTo(keyContainers[0])
+				ored = tmpBitmap
+			} else {
+				if tmpArray == nil {
+					tmpArray = newArrayContainerCapacity(maxPossibleOr)
+				}
+				tmpArray.realloc(maxPossibleOr)
+				tmpArray.resetTo(keyContainers[0])
+				ored = tmpArray
+			}
+			for _, c := range keyContainers[1:] {
+				ored = ored.ior(c)
+			}
+		}
+
+		result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored)
+		if !result.isEmpty() {
+			x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false)
+			intersections++
+		}
+
+		keyContainers = keyContainers[:0]
+		basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
+	}
+
+	x1.highlowcontainer.resize(intersections)
+}

vendor/github.com/RoaringBitmap/roaring/v2/internal/byte_input.go

@@ -0,0 +1,215 @@
+package internal
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// ByteInput typed interface around io.Reader or raw bytes
+type ByteInput interface {
+	// Next returns a slice containing the next n bytes from the buffer,
+	// advancing the buffer as if the bytes had been returned by Read.
+	Next(n int) ([]byte, error)
+	// NextReturnsSafeSlice returns true if Next() returns a safe slice as opposed
+	// to a slice that points to an underlying buffer possibly owned by another system.
+	// When NextReturnsSafeSlice returns false, the result from Next() should be copied
+	// before it is modified (i.e., it is immutable).
+	NextReturnsSafeSlice() bool
+	// ReadUInt32 reads uint32 with LittleEndian order
+	ReadUInt32() (uint32, error)
+	// ReadUInt16 reads uint16 with LittleEndian order
+	ReadUInt16() (uint16, error)
+	// GetReadBytes returns read bytes
+	GetReadBytes() int64
+	// SkipBytes skips exactly n bytes
+	SkipBytes(n int) error
+}
+
+// NewByteInputFromReader creates reader wrapper
+func NewByteInputFromReader(reader io.Reader) ByteInput {
+	return &ByteInputAdapter{
+		r:         reader,
+		readBytes: 0,
+	}
+}
+
+// NewByteInput creates raw bytes wrapper
+func NewByteInput(buf []byte) ByteInput {
+	return &ByteBuffer{
+		buf: buf,
+		off: 0,
+	}
+}
+
+// ByteBuffer raw bytes wrapper
+type ByteBuffer struct {
+	buf []byte
+	off int
+}
+
+// NewByteBuffer creates a new ByteBuffer.
+func NewByteBuffer(buf []byte) *ByteBuffer {
+	return &ByteBuffer{
+		buf: buf,
+	}
+}
+
+var _ io.Reader = (*ByteBuffer)(nil)
+
+// Read implements io.Reader.
+func (b *ByteBuffer) Read(p []byte) (int, error) {
+	data, err := b.Next(len(p))
+	if err != nil {
+		return 0, err
+	}
+	copy(p, data)
+	return len(data), nil
+}
+
+// Next returns a slice containing the next n bytes from the reader
+// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned
+func (b *ByteBuffer) Next(n int) ([]byte, error) {
+	m := len(b.buf) - b.off
+
+	if n > m {
+		return nil, io.ErrUnexpectedEOF
+	}
+
+	data := b.buf[b.off : b.off+n]
+	b.off += n
+
+	return data, nil
+}
+
+// NextReturnsSafeSlice returns false since ByteBuffer might hold
+// an array owned by some other systems.
+func (b *ByteBuffer) NextReturnsSafeSlice() bool {
+	return false
+}
+
+// ReadUInt32 reads uint32 with LittleEndian order
+func (b *ByteBuffer) ReadUInt32() (uint32, error) {
+	if len(b.buf)-b.off < 4 {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	v := binary.LittleEndian.Uint32(b.buf[b.off:])
+	b.off += 4
+
+	return v, nil
+}
+
+// ReadUInt16 reads uint16 with LittleEndian order
+func (b *ByteBuffer) ReadUInt16() (uint16, error) {
+	if len(b.buf)-b.off < 2 {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	v := binary.LittleEndian.Uint16(b.buf[b.off:])
+	b.off += 2
+
+	return v, nil
+}
+
+// GetReadBytes returns read bytes
+func (b *ByteBuffer) GetReadBytes() int64 {
+	return int64(b.off)
+}
+
+// SkipBytes skips exactly n bytes
+func (b *ByteBuffer) SkipBytes(n int) error {
+	m := len(b.buf) - b.off
+
+	if n > m {
+		return io.ErrUnexpectedEOF
+	}
+
+	b.off += n
+
+	return nil
+}
+
+// Reset resets the given buffer with a new byte slice
+func (b *ByteBuffer) Reset(buf []byte) {
+	b.buf = buf
+	b.off = 0
+}
+
+// ByteInputAdapter reader wrapper
+type ByteInputAdapter struct {
+	r         io.Reader
+	readBytes int
+	buf       [4]byte
+}
+
+var _ io.Reader = (*ByteInputAdapter)(nil)
+
+// Read implements io.Reader.
+func (b *ByteInputAdapter) Read(buf []byte) (int, error) {
+	m, err := io.ReadAtLeast(b.r, buf, len(buf))
+	b.readBytes += m
+
+	if err != nil {
+		return 0, err
+	}
+
+	return m, nil
+}
+
+// Next returns a slice containing the next n bytes from the buffer,
+// advancing the buffer as if the bytes had been returned by Read.
+func (b *ByteInputAdapter) Next(n int) ([]byte, error) {
+	buf := make([]byte, n)
+	_, err := b.Read(buf)
+
+	if err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+// NextReturnsSafeSlice returns true since ByteInputAdapter always returns a slice
+// allocated with make([]byte, ...)
+func (b *ByteInputAdapter) NextReturnsSafeSlice() bool {
+	return true
+}
+
+// ReadUInt32 reads uint32 with LittleEndian order
+func (b *ByteInputAdapter) ReadUInt32() (uint32, error) {
+	buf := b.buf[:4]
+	_, err := b.Read(buf)
+	if err != nil {
+		return 0, err
+	}
+
+	return binary.LittleEndian.Uint32(buf), nil
+}
+
+// ReadUInt16 reads uint16 with LittleEndian order
+func (b *ByteInputAdapter) ReadUInt16() (uint16, error) {
+	buf := b.buf[:2]
+	_, err := b.Read(buf)
+	if err != nil {
+		return 0, err
+	}
+
+	return binary.LittleEndian.Uint16(buf), nil
+}
+
+// GetReadBytes returns read bytes
+func (b *ByteInputAdapter) GetReadBytes() int64 {
+	return int64(b.readBytes)
+}
+
+// SkipBytes skips exactly n bytes
+func (b *ByteInputAdapter) SkipBytes(n int) error {
+	_, err := b.Next(n)
+
+	return err
+}
+
+// Reset resets the given buffer with a new stream
+func (b *ByteInputAdapter) Reset(stream io.Reader) {
+	b.r = stream
+	b.readBytes = 0
+}

vendor/github.com/RoaringBitmap/roaring/v2/internal/pools.go

@@ -0,0 +1,21 @@
+package internal
+
+import (
+	"sync"
+)
+
+var (
+	// ByteInputAdapterPool shared pool
+	ByteInputAdapterPool = sync.Pool{
+		New: func() interface{} {
+			return &ByteInputAdapter{}
+		},
+	}
+
+	// ByteBufferPool shared pool
+	ByteBufferPool = sync.Pool{
+		New: func() interface{} {
+			return &ByteBuffer{}
+		},
+	}
+)

vendor/github.com/RoaringBitmap/roaring/v2/iter.go

@@ -0,0 +1,159 @@
+package roaring
+
+import (
+	"iter"
+	"math/bits"
+)
+
+// Values returns an iterator that yields the elements of the bitmap in
+// increasing order. Starting with Go 1.23, users can use a for loop to iterate
+// over it.
+func Values(b *Bitmap) iter.Seq[uint32] {
+	return func(yield func(uint32) bool) {
+		it := b.Iterator()
+		for it.HasNext() {
+			if !yield(it.Next()) {
+				return
+			}
+		}
+	}
+}
+
+// Backward returns an iterator that yields the elements of the bitmap in
+// decreasing order. Starting with Go 1.23, users can use a for loop to iterate
+// over it.
+func Backward(b *Bitmap) iter.Seq[uint32] {
+	return func(yield func(uint32) bool) {
+		it := b.ReverseIterator()
+		for it.HasNext() {
+			if !yield(it.Next()) {
+				return
+			}
+		}
+	}
+}
+
+// Unset creates an iterator that yields values in the range [min, max] that are NOT contained in the bitmap.
+// The iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
+func Unset(b *Bitmap, min, max uint32) iter.Seq[uint32] {
+	return func(yield func(uint32) bool) {
+		it := b.UnsetIterator(uint64(min), uint64(max)+1)
+		for it.HasNext() {
+			if !yield(it.Next()) {
+				return
+			}
+		}
+	}
+}
+
+// Ranges iterates contiguous ranges of values present in the bitmap as
+// half-open [start, endExclusive) pairs. endExclusive is uint64 to represent
+// ranges that include MaxUint32. Ranges spanning container boundaries are merged.
+func (b *Bitmap) Ranges() iter.Seq2[uint32, uint64] {
+	return func(yield func(uint32, uint64) bool) {
+		ra := &b.highlowcontainer
+		keys := ra.keys
+		containers := ra.containers
+		n := len(keys)
+
+		var pendingStart, pendingEnd uint64
+		hasPending := false
+
+		emit := func(rStart, rEnd uint64) bool {
+			if hasPending && rStart <= pendingEnd {
+				if rEnd > pendingEnd {
+					pendingEnd = rEnd
+				}
+				return true
+			}
+			if hasPending {
+				if !yield(uint32(pendingStart), pendingEnd) {
+					return false
+				}
+			}
+			pendingStart = rStart
+			pendingEnd = rEnd
+			hasPending = true
+			return true
+		}
+
+		for idx := 0; idx < n; idx++ {
+			hs := uint64(keys[idx]) << 16
+			c := containers[idx]
+
+			switch t := c.(type) {
+			case *runContainer16:
+				for _, iv := range t.iv {
+					if !emit(hs+uint64(iv.start), hs+uint64(iv.start)+uint64(iv.length)+1) {
+						return
+					}
+				}
+
+			case *bitmapContainer:
+				bm := t.bitmap
+				length := uint(len(bm))
+				pos := uint(0)
+
+				for pos < length {
+					w := bm[pos]
+					if w == 0 {
+						pos++
+						continue
+					}
+
+					for w != 0 {
+						lo := uint(bits.TrailingZeros64(w))
+						bitStart := pos*64 + lo
+
+						ones := uint(bits.TrailingZeros64(^(w >> lo)))
+						if lo+ones < 64 {
+							if !emit(hs+uint64(bitStart), hs+uint64(bitStart+ones)) {
+								return
+							}
+							w &= ^((uint64(1) << (lo + ones)) - 1)
+						} else {
+							pos++
+							for pos < length && bm[pos] == 0xFFFFFFFFFFFFFFFF {
+								pos++
+							}
+							var bitEnd uint
+							if pos < length {
+								trailing := uint(bits.TrailingZeros64(^bm[pos]))
+								bitEnd = pos*64 + trailing
+								w = bm[pos] & ^((uint64(1) << trailing) - 1)
+							} else {
+								bitEnd = length * 64
+								w = 0
+							}
+							if !emit(hs+uint64(bitStart), hs+uint64(bitEnd)) {
+								return
+							}
+							continue
+						}
+					}
+					pos++
+				}
+
+			case *arrayContainer:
+				content := t.content
+				i := 0
+				for i < len(content) {
+					start := uint64(content[i])
+					end := start + 1
+					i++
+					for i < len(content) && uint64(content[i]) == end {
+						end++
+						i++
+					}
+					if !emit(hs+start, hs+end) {
+						return
+					}
+				}
+			}
+		}
+
+		if hasPending {
+			yield(uint32(pendingStart), pendingEnd)
+		}
+	}
+}

vendor/github.com/RoaringBitmap/roaring/v2/manyiterator.go

@@ -0,0 +1,32 @@
+package roaring
+
+type manyIterable interface {
+	nextMany(hs uint32, buf []uint32) int
+	nextMany64(hs uint64, buf []uint64) int
+}
+
+func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
+	n := 0
+	l := si.loc
+	s := si.slice
+	for n < len(buf) && l < len(s) {
+		buf[n] = uint32(s[l]) | hs
+		l++
+		n++
+	}
+	si.loc = l
+	return n
+}
+
+func (si *shortIterator) nextMany64(hs uint64, buf []uint64) int {
+	n := 0
+	l := si.loc
+	s := si.slice
+	for n < len(buf) && l < len(s) {
+		buf[n] = uint64(s[l]) | hs
+		l++
+		n++
+	}
+	si.loc = l
+	return n
+}

vendor/github.com/RoaringBitmap/roaring/v2/parallel.go

@@ -0,0 +1,612 @@
+package roaring
+
+import (
+	"container/heap"
+	"fmt"
+	"runtime"
+	"sync"
+)
+
+var defaultWorkerCount = runtime.NumCPU()
+
+type bitmapContainerKey struct {
+	key    uint16
+	idx    int
+	bitmap *Bitmap
+}
+
+type multipleContainers struct {
+	key        uint16
+	containers []container
+	idx        int
+}
+
+type keyedContainer struct {
+	key       uint16
+	container container
+	idx       int
+}
+
+type bitmapContainerHeap []bitmapContainerKey
+
+func (h bitmapContainerHeap) Len() int           { return len(h) }
+func (h bitmapContainerHeap) Less(i, j int) bool { return h[i].key < h[j].key }
+func (h bitmapContainerHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *bitmapContainerHeap) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	*h = append(*h, x.(bitmapContainerKey))
+}
+
+func (h *bitmapContainerHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+func (h bitmapContainerHeap) Peek() bitmapContainerKey {
+	return h[0]
+}
+
+func (h *bitmapContainerHeap) popIncrementing() (key uint16, container container) {
+	k := h.Peek()
+	key = k.key
+	container = k.bitmap.highlowcontainer.containers[k.idx]
+
+	newIdx := k.idx + 1
+	if newIdx < k.bitmap.highlowcontainer.size() {
+		k = bitmapContainerKey{
+			k.bitmap.highlowcontainer.keys[newIdx],
+			newIdx,
+			k.bitmap,
+		}
+		(*h)[0] = k
+		heap.Fix(h, 0)
+	} else {
+		heap.Pop(h)
+	}
+
+	return
+}
+
+func (h *bitmapContainerHeap) Next(containers []container) multipleContainers {
+	if h.Len() == 0 {
+		return multipleContainers{}
+	}
+
+	key, container := h.popIncrementing()
+	containers = append(containers, container)
+
+	for h.Len() > 0 && key == h.Peek().key {
+		_, container = h.popIncrementing()
+		containers = append(containers, container)
+	}
+
+	return multipleContainers{
+		key,
+		containers,
+		-1,
+	}
+}
+
+func newBitmapContainerHeap(bitmaps ...*Bitmap) bitmapContainerHeap {
+	// Initialize heap
+	var h bitmapContainerHeap = make([]bitmapContainerKey, 0, len(bitmaps))
+	for _, bitmap := range bitmaps {
+		if !bitmap.IsEmpty() {
+			key := bitmapContainerKey{
+				bitmap.highlowcontainer.keys[0],
+				0,
+				bitmap,
+			}
+			h = append(h, key)
+		}
+	}
+
+	heap.Init(&h)
+
+	return h
+}
+
+func repairAfterLazy(c container) container {
+	switch t := c.(type) {
+	case *bitmapContainer:
+		if t.cardinality == invalidCardinality {
+			t.computeCardinality()
+		}
+
+		if t.getCardinality() <= arrayDefaultMaxSize {
+			return t.toArrayContainer()
+		} else if c.(*bitmapContainer).isFull() {
+			return newRunContainer16Range(0, MaxUint16)
+		}
+	}
+
+	return c
+}
+
+func toBitmapContainer(c container) container {
+	switch t := c.(type) {
+	case *arrayContainer:
+		return t.toBitmapContainer()
+	case *runContainer16:
+		if !t.isFull() {
+			return t.toBitmapContainer()
+		}
+	}
+	return c
+}
+
+func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) {
+	expectedKeys := -1
+	appendedKeys := 0
+	var keys []uint16
+	var containers []container
+	for appendedKeys != expectedKeys {
+		select {
+		case item := <-resultChan:
+			if len(keys) <= item.idx {
+				keys = append(keys, make([]uint16, item.idx-len(keys)+1)...)
+				containers = append(containers, make([]container, item.idx-len(containers)+1)...)
+			}
+			keys[item.idx] = item.key
+			containers[item.idx] = item.container
+
+			appendedKeys++
+		case msg := <-expectedKeysChan:
+			expectedKeys = msg
+		}
+	}
+	answer := &Bitmap{
+		roaringArray{
+			make([]uint16, 0, expectedKeys),
+			make([]container, 0, expectedKeys),
+			make([]bool, 0, expectedKeys),
+			false,
+		},
+	}
+	for i := range keys {
+		if containers[i] != nil { // in case a resulting container was empty, see ParAnd function
+			answer.highlowcontainer.appendContainer(keys[i], containers[i], false)
+		}
+	}
+
+	bitmapChan <- answer
+}
+
+// ParHeapOr computes the union (OR) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+// ParHeapOr uses a heap to compute the union. For rare cases it might be faster than ParOr
+func ParHeapOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+
+	bitmapCount := len(bitmaps)
+	if bitmapCount == 0 {
+		return NewBitmap()
+	} else if bitmapCount == 1 {
+		return bitmaps[0].Clone()
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+
+	h := newBitmapContainerHeap(bitmaps...)
+
+	bitmapChan := make(chan *Bitmap)
+	inputChan := make(chan multipleContainers, 128)
+	resultChan := make(chan keyedContainer, 32)
+	expectedKeysChan := make(chan int)
+
+	pool := sync.Pool{
+		New: func() interface{} {
+			return make([]container, 0, len(bitmaps))
+		},
+	}
+
+	orFunc := func() {
+		// Assumes only structs with >=2 containers are passed
+		for input := range inputChan {
+			c := toBitmapContainer(input.containers[0]).lazyOR(input.containers[1])
+			for _, next := range input.containers[2:] {
+				c = c.lazyIOR(next)
+			}
+			c = repairAfterLazy(c)
+			kx := keyedContainer{
+				input.key,
+				c,
+				input.idx,
+			}
+			resultChan <- kx
+			pool.Put(input.containers[:0])
+		}
+	}
+
+	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)
+
+	for i := 0; i < parallelism; i++ {
+		go orFunc()
+	}
+
+	idx := 0
+	for h.Len() > 0 {
+		ck := h.Next(pool.Get().([]container))
+		if len(ck.containers) == 1 {
+			resultChan <- keyedContainer{
+				ck.key,
+				ck.containers[0],
+				idx,
+			}
+			pool.Put(ck.containers[:0])
+		} else {
+			ck.idx = idx
+			inputChan <- ck
+		}
+		idx++
+	}
+	expectedKeysChan <- idx
+
+	bitmap := <-bitmapChan
+
+	close(inputChan)
+	close(resultChan)
+	close(expectedKeysChan)
+
+	return bitmap
+}
+
+// ParAnd computes the intersection (AND) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+	bitmapCount := len(bitmaps)
+	if bitmapCount == 0 {
+		return NewBitmap()
+	} else if bitmapCount == 1 {
+		return bitmaps[0].Clone()
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+
+	h := newBitmapContainerHeap(bitmaps...)
+
+	bitmapChan := make(chan *Bitmap)
+	inputChan := make(chan multipleContainers, 128)
+	resultChan := make(chan keyedContainer, 32)
+	expectedKeysChan := make(chan int)
+
+	andFunc := func() {
+		// Assumes only structs with >=2 containers are passed
+		for input := range inputChan {
+			c := input.containers[0].and(input.containers[1])
+			for _, next := range input.containers[2:] {
+				if c.isEmpty() {
+					break
+				}
+				c = c.iand(next)
+			}
+
+			// Send a nil explicitly if the result of the intersection is an empty container
+			if c.isEmpty() {
+				c = nil
+			}
+
+			kx := keyedContainer{
+				input.key,
+				c,
+				input.idx,
+			}
+			resultChan <- kx
+		}
+	}
+
+	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)
+
+	for i := 0; i < parallelism; i++ {
+		go andFunc()
+	}
+
+	idx := 0
+	for h.Len() > 0 {
+		ck := h.Next(make([]container, 0, 4))
+		if len(ck.containers) == bitmapCount {
+			ck.idx = idx
+			inputChan <- ck
+			idx++
+		}
+	}
+	expectedKeysChan <- idx
+
+	bitmap := <-bitmapChan
+
+	close(inputChan)
+	close(resultChan)
+	close(expectedKeysChan)
+
+	return bitmap
+}
+
+// ParOr computes the union (OR) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+	var lKey uint16 = MaxUint16
+	var hKey uint16
+
+	bitmapsFiltered := bitmaps[:0]
+	for _, b := range bitmaps {
+		if !b.IsEmpty() {
+			bitmapsFiltered = append(bitmapsFiltered, b)
+		}
+	}
+	bitmaps = bitmapsFiltered
+
+	for _, b := range bitmaps {
+		lKey = minOfUint16(lKey, b.highlowcontainer.keys[0])
+		hKey = maxOfUint16(hKey, b.highlowcontainer.keys[b.highlowcontainer.size()-1])
+	}
+
+	if lKey == MaxUint16 && hKey == 0 {
+		return New()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+
+	keyRange := int(hKey) - int(lKey) + 1
+	if keyRange == 1 {
+		// revert to FastOr. Since the key range is 0
+		// no container-level aggregation parallelism is achievable
+		return FastOr(bitmaps...)
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+
+	var chunkSize int
+	var chunkCount int
+	if parallelism*4 > keyRange {
+		chunkSize = 1
+		chunkCount = keyRange
+	} else {
+		chunkCount = parallelism * 4
+		chunkSize = (keyRange + chunkCount - 1) / chunkCount
+	}
+
+	if chunkCount*chunkSize < keyRange {
+		// it's fine to panic to indicate an implementation error
+		panic(fmt.Sprintf("invariant check failed: chunkCount * chunkSize < keyRange, %d * %d < %d", chunkCount, chunkSize, keyRange))
+	}
+
+	chunks := make([]*roaringArray, chunkCount)
+
+	chunkSpecChan := make(chan parChunkSpec, minOfInt(maxOfInt(64, 2*parallelism), chunkCount))
+	chunkChan := make(chan parChunk, minOfInt(32, chunkCount))
+
+	orFunc := func() {
+		for spec := range chunkSpecChan {
+			ra := lazyOrOnRange(&bitmaps[0].highlowcontainer, &bitmaps[1].highlowcontainer, spec.start, spec.end)
+			for _, b := range bitmaps[2:] {
+				ra = lazyIOrOnRange(ra, &b.highlowcontainer, spec.start, spec.end)
+			}
+
+			for i, c := range ra.containers {
+				ra.containers[i] = repairAfterLazy(c)
+			}
+
+			chunkChan <- parChunk{ra, spec.idx}
+		}
+	}
+
+	for i := 0; i < parallelism; i++ {
+		go orFunc()
+	}
+
+	go func() {
+		for i := 0; i < chunkCount; i++ {
+			spec := parChunkSpec{
+				start: uint16(int(lKey) + i*chunkSize),
+				end:   uint16(minOfInt(int(lKey)+(i+1)*chunkSize-1, int(hKey))),
+				idx:   i,
+			}
+			chunkSpecChan <- spec
+		}
+	}()
+
+	chunksRemaining := chunkCount
+	for chunk := range chunkChan {
+		chunks[chunk.idx] = chunk.ra
+		chunksRemaining--
+		if chunksRemaining == 0 {
+			break
+		}
+	}
+	close(chunkChan)
+	close(chunkSpecChan)
+
+	containerCount := 0
+	for _, chunk := range chunks {
+		containerCount += chunk.size()
+	}
+
+	result := Bitmap{
+		roaringArray{
+			containers:      make([]container, containerCount),
+			keys:            make([]uint16, containerCount),
+			needCopyOnWrite: make([]bool, containerCount),
+		},
+	}
+
+	resultOffset := 0
+	for _, chunk := range chunks {
+		copy(result.highlowcontainer.containers[resultOffset:], chunk.containers)
+		copy(result.highlowcontainer.keys[resultOffset:], chunk.keys)
+		copy(result.highlowcontainer.needCopyOnWrite[resultOffset:], chunk.needCopyOnWrite)
+		resultOffset += chunk.size()
+	}
+
+	return &result
+}
+
+type parChunkSpec struct {
+	start uint16
+	end   uint16
+	idx   int
+}
+
+type parChunk struct {
+	ra  *roaringArray
+	idx int
+}
+
+func (c parChunk) size() int {
+	return c.ra.size()
+}
+
+func parNaiveStartAt(ra *roaringArray, start uint16, last uint16) int {
+	for idx, key := range ra.keys {
+		if key >= start && key <= last {
+			return idx
+		} else if key > last {
+			break
+		}
+	}
+	return ra.size()
+}
+
+func lazyOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray {
+	answer := newRoaringArray()
+	length1 := ra1.size()
+	length2 := ra2.size()
+
+	idx1 := parNaiveStartAt(ra1, start, last)
+	idx2 := parNaiveStartAt(ra2, start, last)
+
+	var key1 uint16
+	var key2 uint16
+	if idx1 < length1 && idx2 < length2 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		key2 = ra2.getKeyAtIndex(idx2)
+
+		for key1 <= last && key2 <= last {
+
+			if key1 < key2 {
+				answer.appendCopy(*ra1, idx1)
+				idx1++
+				if idx1 == length1 {
+					break
+				}
+				key1 = ra1.getKeyAtIndex(idx1)
+			} else if key1 > key2 {
+				answer.appendCopy(*ra2, idx2)
+				idx2++
+				if idx2 == length2 {
+					break
+				}
+				key2 = ra2.getKeyAtIndex(idx2)
+			} else {
+				c1 := ra1.getFastContainerAtIndex(idx1, false)
+
+				answer.appendContainer(key1, c1.lazyOR(ra2.getContainerAtIndex(idx2)), false)
+				idx1++
+				idx2++
+				if idx1 == length1 || idx2 == length2 {
+					break
+				}
+
+				key1 = ra1.getKeyAtIndex(idx1)
+				key2 = ra2.getKeyAtIndex(idx2)
+			}
+		}
+	}
+
+	if idx2 < length2 {
+		key2 = ra2.getKeyAtIndex(idx2)
+		for key2 <= last {
+			answer.appendCopy(*ra2, idx2)
+			idx2++
+			if idx2 == length2 {
+				break
+			}
+			key2 = ra2.getKeyAtIndex(idx2)
+		}
+	}
+
+	if idx1 < length1 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		for key1 <= last {
+			answer.appendCopy(*ra1, idx1)
+			idx1++
+			if idx1 == length1 {
+				break
+			}
+			key1 = ra1.getKeyAtIndex(idx1)
+		}
+	}
+	return answer
+}
+
+func lazyIOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray {
+	length1 := ra1.size()
+	length2 := ra2.size()
+
+	idx1 := 0
+	idx2 := parNaiveStartAt(ra2, start, last)
+
+	var key1 uint16
+	var key2 uint16
+	if idx1 < length1 && idx2 < length2 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		key2 = ra2.getKeyAtIndex(idx2)
+
+		for key1 <= last && key2 <= last {
+			if key1 < key2 {
+				idx1++
+				if idx1 >= length1 {
+					break
+				}
+				key1 = ra1.getKeyAtIndex(idx1)
+			} else if key1 > key2 {
+				ra1.insertNewKeyValueAt(idx1, key2, ra2.getContainerAtIndex(idx2))
+				ra1.needCopyOnWrite[idx1] = true
+				idx2++
+				idx1++
+				length1++
+				if idx2 >= length2 {
+					break
+				}
+				key2 = ra2.getKeyAtIndex(idx2)
+			} else {
+				c1 := ra1.getFastContainerAtIndex(idx1, true)
+
+				ra1.containers[idx1] = c1.lazyIOR(ra2.getContainerAtIndex(idx2))
+				ra1.needCopyOnWrite[idx1] = false
+				idx1++
+				idx2++
+				if idx1 >= length1 || idx2 >= length2 {
+					break
+				}
+
+				key1 = ra1.getKeyAtIndex(idx1)
+				key2 = ra2.getKeyAtIndex(idx2)
+			}
+		}
+	}
+	if idx2 < length2 {
+		key2 = ra2.getKeyAtIndex(idx2)
+		for key2 <= last {
+			ra1.appendCopy(*ra2, idx2)
+			idx2++
+			if idx2 >= length2 {
+				break
+			}
+			key2 = ra2.getKeyAtIndex(idx2)
+		}
+	}
+	return ra1
+}

vendor/github.com/RoaringBitmap/roaring/v2/popcnt_avx2_amd64.go

@@ -0,0 +1,67 @@
+//go:build amd64 && !appengine
+// +build amd64,!appengine
+
+package roaring
+
+// The functions below are implemented in popcnt_avx2_amd64.s using AVX2.
+// They are only used when the CPU supports AVX2 (see useAVX2); otherwise the
+// pure-Go fallbacks in popcnt_slices.go are used. This keeps behavior identical
+// on every target: appengine and non-amd64 builds compile popcnt_generic.go
+// instead, and amd64 CPUs without AVX2 take the scalar path at runtime.
+
+//go:noescape
+func _hasAVX2() bool
+
+//go:noescape
+func _popcntSliceAVX2(s []uint64) uint64
+
+//go:noescape
+func _popcntMaskSliceAVX2(s, m []uint64) uint64
+
+//go:noescape
+func _popcntAndSliceAVX2(s, m []uint64) uint64
+
+//go:noescape
+func _popcntOrSliceAVX2(s, m []uint64) uint64
+
+//go:noescape
+func _popcntXorSliceAVX2(s, m []uint64) uint64
+
+// useAVX2 selects the AVX2 assembly implementations when the running CPU
+// supports AVX2. It is evaluated once at package initialization.
+var useAVX2 = _hasAVX2()
+
+func popcntSlice(s []uint64) uint64 {
+	if useAVX2 {
+		return _popcntSliceAVX2(s)
+	}
+	return popcntSliceGo(s)
+}
+
+func popcntMaskSlice(s, m []uint64) uint64 {
+	if useAVX2 {
+		return _popcntMaskSliceAVX2(s, m)
+	}
+	return popcntMaskSliceGo(s, m)
+}
+
+func popcntAndSlice(s, m []uint64) uint64 {
+	if useAVX2 {
+		return _popcntAndSliceAVX2(s, m)
+	}
+	return popcntAndSliceGo(s, m)
+}
+
+func popcntOrSlice(s, m []uint64) uint64 {
+	if useAVX2 {
+		return _popcntOrSliceAVX2(s, m)
+	}
+	return popcntOrSliceGo(s, m)
+}
+
+func popcntXorSlice(s, m []uint64) uint64 {
+	if useAVX2 {
+		return _popcntXorSliceAVX2(s, m)
+	}
+	return popcntXorSliceGo(s, m)
+}

vendor/github.com/RoaringBitmap/roaring/v2/popcnt_avx2_amd64.s

@@ -0,0 +1,360 @@
+// +build amd64,!appengine
+
+//go:build amd64 && !appengine
+
+#include "textflag.h"
+
+// AVX2 population-count routines for amd64. They count the set bits across a
+// []uint64 (the backing storage of a bitmap container), optionally combining
+// each pair of words with a boolean op first: And, Or, Xor, and Mask (s &^ m).
+//
+// Algorithm (Mula/Lemire VPSHUFB nibble lookup)
+// ---------------------------------------------
+// AVX2 has no single "popcount a whole vector" instruction, so each byte's
+// popcount is taken from a 16-entry lookup table indexed by a 4-bit nibble:
+// a byte is split into its low and high nibble, each nibble is looked up (one
+// VPSHUFB performs all 32 lookups in a 256-bit register at once), and VPSADBW
+// then combines the two results and sums each group of 8 byte-counts into a
+// 64-bit lane total that is accumulated (COUNTBLOCK).
+// After the loop the four lane totals are summed (HSUM) into a scalar register.
+// Each iteration handles 256 bits (4 uint64); a scalar POPCNTQ tail handles
+// the trailing len%4 words, so any slice length is counted correctly.
+//
+// Go assembler conventions used below
+// -----------------------------------
+//   - Operands are written source(s) first, destination LAST. So
+//     "VPAND Ymask, Ydata, Ylo" means Ylo = Ydata AND Ymask.
+//   - Yn are the 256-bit AVX registers; Xn aliases the low 128 bits of Yn.
+//   - Arguments/results are read from the frame pointer (FP). A Go slice is a
+//     3-word header {ptr,len,cap}: s_base+0(FP), s_len+8(FP); a second slice
+//     argument starts at +24(FP). The uint64 result slot follows the args
+//     (e.g. ret+24(FP) for one slice arg, ret+48(FP) for two).
+//   - Every routine is a leaf (makes no calls): NOSPLIT with a $0 local frame.
+//   - Loads/stores use VMOVDQU (unaligned): container slices are only 8-byte
+//     aligned, not 32. Generic (VEX-encoded) AVX instructions impose no
+//     alignment requirement on a memory source either, so the second input of
+//     the two-operand loops is read straight out of memory by VPAND/VPOR/
+//     VPXOR/VPANDN instead of being loaded into a register first.
+//     VZEROUPPER precedes every RET to avoid the AVX<->SSE transition penalty
+//     in any non-VEX SSE code that runs afterwards.
+
+// lutmask is a 17-byte read-only blob (the linker pads it out to whatever its
+// alignment requires) holding the two constants used by every routine:
+//   bytes  0..15 - the nibble popcount table, i.e. table[i] = number of set
+//                  bits in the 4-bit value i. Read low-byte-first, the first
+//                  qword 0x0302020102010100 is the bytes {0,1,1,2,1,2,2,3} for
+//                  nibbles 0..7, and 0x0403030203020201 is {1,2,2,3,2,3,3,4}
+//                  for nibbles 8..15. VPSHUFB indexes within each 128-bit lane
+//                  independently and so needs the table in both lanes, but
+//                  VBROADCASTI128 duplicates the 16 bytes at load time; only
+//                  one copy has to be stored.
+//   byte     16  - 0x0F, the mask that isolates the low nibble of each byte,
+//                  splatted to all 32 bytes by VPBROADCASTB.
+// RODATA|NOPTR marks it read-only and pointer-free (so the GC ignores it).
+DATA lutmask<>+0(SB)/8, $0x0302020102010100
+DATA lutmask<>+8(SB)/8, $0x0403030203020201
+DATA lutmask<>+16(SB)/1, $0x0f
+GLOBL lutmask<>(SB), RODATA|NOPTR, $17
+
+// Register aliases. Ylut1/Ylut2/Ymask are constants set up once per call (see
+// SETUP); Yacc is the running accumulator of lane totals; Ydata holds the
+// current input vector; Ylo/Yhi are scratch used by COUNTBLOCK. Ydata is dead
+// once its nibbles have been extracted, so Yhi shares its register: only five
+// architectural registers are needed.
+#define Ylut1 Y0
+#define Ylut2 Y1
+#define Ymask Y2
+#define Yacc Y3
+#define Ydata Y4
+#define Yhi Y4
+#define Ylo Y5
+
+// Low 128-bit halves of Yacc and Ylo, used as scratch by HSUM.
+#define Xacc X3
+#define Xtmp X5
+
+// COUNTBLOCK folds the popcount of the 32 bytes currently in Ydata into the
+// accumulator Yacc. Line by line:
+//   VPAND  Ymask,Ydata,Ylo   : Ylo = low nibble of every byte
+//   VPSRLW $4,Ydata,Yhi      : shift each 16-bit lane right by 4...
+//   VPAND  Ymask,Yhi,Yhi     : ...then mask, leaving the high nibble of each byte
+//   VPSHUFB Ylo,Ylut1,Ylo    : Ylo[b] = B + popcount(low nibble of byte b)
+//   VPSHUFB Yhi,Ylut2,Yhi    : Yhi[b] = B - popcount(high nibble of byte b)
+//   VPSADBW Ylo,Yhi,Ylo      : sum each group of 8 bytes -> 4 lane totals
+//   VPADDQ  Ylo,Yacc,Yacc    : add the 4 lane totals into the accumulator
+//
+// VPSADBW computes |a-b| per byte and sums each group of 8, so it can do the
+// work of the per-byte add as well: feeding it the two nibble counts directly
+// yields (B + lo) - (B - hi) = lo + hi, and the separate VPADDB the naive
+// version needs (with a zero second VPSADBW operand) disappears along with its
+// latency. That is why SETUP builds two tables, one biased up by B and one
+// subtracted from B. The bias must satisfy 4 <= B <= 251 so that neither
+// table's entries (max nibble popcount is 4) wrap around as unsigned bytes and
+// so that a >= b always holds, making the absolute value a no-op; B = 15 is
+// used simply because Ymask already holds 15 in every byte.
+//
+// Per-byte counts max at 8 and lane totals at 512, so accumulating across the
+// whole loop never overflows the 64-bit lanes.
+#define COUNTBLOCK \
+	VPAND Ymask, Ydata, Ylo \
+	VPSRLW $4, Ydata, Yhi \
+	VPAND Ymask, Yhi, Yhi \
+	VPSHUFB Ylo, Ylut1, Ylo \
+	VPSHUFB Yhi, Ylut2, Yhi \
+	VPSADBW Ylo, Yhi, Ylo \
+	VPADDQ Ylo, Yacc, Yacc
+
+// SETUP builds the two biased lookup tables and the nibble mask, and zeroes
+// Yacc (the accumulator). Run once per routine, after the check that the
+// vector loop runs at least one iteration. Ylut1 first holds the raw table, so
+// the VPSUBB must come before the VPADDB that overwrites it.
+#define SETUP \
+	VBROADCASTI128 lutmask<>+0(SB), Ylut1 \
+	VPBROADCASTB lutmask<>+16(SB), Ymask \
+	VPXOR Yacc, Yacc, Yacc \
+	VPSUBB Ylut1, Ymask, Ylut2 \
+	VPADDB Ylut1, Ymask, Ylut1
+
+// HSUM reduces Yacc's four 64-bit lane totals to a single sum in AX.
+// VEXTRACTI128 pulls Yacc's high 128 bits into Xtmp and the two halves are
+// added, leaving two qwords in Xacc; VPSHUFD $0x4e then swaps those two qwords
+// so a second VPADDQ puts their total in the low qword, which one MOVQ moves
+// out. Finishing the reduction in SIMD avoids VPEXTRQ, which is 2 uops on both
+// AMD Zen and Intel, against 1 each for VPSHUFD and VPADDQ.
+#define HSUM \
+	VEXTRACTI128 $1, Yacc, Xtmp \
+	VPADDQ Xtmp, Xacc, Xacc \
+	VPSHUFD $0x4e, Xacc, Xtmp \
+	VPADDQ Xtmp, Xacc, Xacc \
+	MOVQ Xacc, DX \
+	ADDQ DX, AX
+
+// func _popcntSliceAVX2(s []uint64) uint64
+// Returns the total number of set bits in s. This is the canonical routine;
+// the And/Or/Xor/Mask variants below share its structure and differ only by
+// the boolean op applied before counting.
+TEXT ·_popcntSliceAVX2(SB), NOSPLIT, $0-32
+	MOVQ s_base+0(FP), SI   // SI = &s[0]
+	MOVQ s_len+8(FP), CX    // CX = len(s), in 64-bit words
+	XORL AX, AX             // AX = running result
+	MOVQ CX, R8
+	SHRQ $2, R8             // R8 = len/4 = number of full 256-bit blocks
+	JZ slicetail            // fewer than 4 words: skip the vector loop
+	SETUP                   // load tables/mask; zero Yacc
+sliceloop:
+	VMOVDQU (SI), Ydata     // load 4 words (32 bytes)
+	COUNTBLOCK              // Yacc += popcount(those 32 bytes)
+	ADDQ $32, SI            // advance to the next block
+	DECQ R8
+	JNZ sliceloop
+	HSUM                    // AX += sum of Yacc's lane totals
+slicetail:
+	ANDL $3, CX             // CX = len % 4 = leftover words (0..3)
+	JZ slicedone
+slicetailloop:
+	POPCNTQ (SI), DX        // scalar popcount of one word
+	ADDQ DX, AX
+	ADDQ $8, SI
+	DECL CX
+	JNZ slicetailloop
+slicedone:
+	VZEROUPPER              // clear upper YMM state before returning
+	MOVQ AX, ret+24(FP)     // return AX
+	RET
+
+// func _popcntAndSliceAVX2(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] & m[i]). Mirrors _popcntSliceAVX2 but ANDs
+// a vector of s with the matching bytes of m before counting. s and m are
+// assumed to have equal length.
+TEXT ·_popcntAndSliceAVX2(SB), NOSPLIT, $0-56
+	MOVQ s_base+0(FP), SI    // SI = &s[0]
+	MOVQ m_base+24(FP), DI   // DI = &m[0]
+	MOVQ s_len+8(FP), CX     // CX = len
+	XORL AX, AX
+	MOVQ CX, R8
+	SHRQ $2, R8
+	JZ andtail
+	SETUP
+andloop:
+	VMOVDQU (SI), Ydata
+	VPAND (DI), Ydata, Ydata // Ydata = s & m
+	COUNTBLOCK
+	ADDQ $32, SI
+	ADDQ $32, DI
+	DECQ R8
+	JNZ andloop
+	HSUM
+andtail:
+	ANDL $3, CX
+	JZ anddone
+andtailloop:
+	MOVQ (SI), DX
+	ANDQ (DI), DX            // s & m, one word
+	POPCNTQ DX, DX
+	ADDQ DX, AX
+	ADDQ $8, SI
+	ADDQ $8, DI
+	DECL CX
+	JNZ andtailloop
+anddone:
+	VZEROUPPER
+	MOVQ AX, ret+48(FP)      // +48: result follows two 24-byte slice headers
+	RET
+
+// func _popcntOrSliceAVX2(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] | m[i]); see _popcntAndSliceAVX2 for the
+// shared structure.
+TEXT ·_popcntOrSliceAVX2(SB), NOSPLIT, $0-56
+	MOVQ s_base+0(FP), SI
+	MOVQ m_base+24(FP), DI
+	MOVQ s_len+8(FP), CX
+	XORL AX, AX
+	MOVQ CX, R8
+	SHRQ $2, R8
+	JZ ortail
+	SETUP
+orloop:
+	VMOVDQU (SI), Ydata
+	VPOR (DI), Ydata, Ydata  // Ydata = s | m
+	COUNTBLOCK
+	ADDQ $32, SI
+	ADDQ $32, DI
+	DECQ R8
+	JNZ orloop
+	HSUM
+ortail:
+	ANDL $3, CX
+	JZ ordone
+ortailloop:
+	MOVQ (SI), DX
+	ORQ (DI), DX             // s | m, one word
+	POPCNTQ DX, DX
+	ADDQ DX, AX
+	ADDQ $8, SI
+	ADDQ $8, DI
+	DECL CX
+	JNZ ortailloop
+ordone:
+	VZEROUPPER
+	MOVQ AX, ret+48(FP)
+	RET
+
+// func _popcntXorSliceAVX2(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] ^ m[i]); see _popcntAndSliceAVX2 for the
+// shared structure.
+TEXT ·_popcntXorSliceAVX2(SB), NOSPLIT, $0-56
+	MOVQ s_base+0(FP), SI
+	MOVQ m_base+24(FP), DI
+	MOVQ s_len+8(FP), CX
+	XORL AX, AX
+	MOVQ CX, R8
+	SHRQ $2, R8
+	JZ xortail
+	SETUP
+xorloop:
+	VMOVDQU (SI), Ydata
+	VPXOR (DI), Ydata, Ydata // Ydata = s ^ m
+	COUNTBLOCK
+	ADDQ $32, SI
+	ADDQ $32, DI
+	DECQ R8
+	JNZ xorloop
+	HSUM
+xortail:
+	ANDL $3, CX
+	JZ xordone
+xortailloop:
+	MOVQ (SI), DX
+	XORQ (DI), DX            // s ^ m, one word
+	POPCNTQ DX, DX
+	ADDQ DX, AX
+	ADDQ $8, SI
+	ADDQ $8, DI
+	DECL CX
+	JNZ xortailloop
+xordone:
+	VZEROUPPER
+	MOVQ AX, ret+48(FP)
+	RET
+
+// func _popcntMaskSliceAVX2(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] &^ m[i]) == popcount(s & ~m). Same structure
+// as _popcntAndSliceAVX2; the combine is VPANDN, which computes (NOT first) AND
+// second. Only VPANDN's second source may come from memory, and it is the
+// operand that is *not* negated, so here it is m that is loaded into a register
+// and s that is read straight out of memory:
+// "VPANDN (SI), Ydata, Ydata" with Ydata = m gives (NOT m) AND s = s &^ m.
+TEXT ·_popcntMaskSliceAVX2(SB), NOSPLIT, $0-56
+	MOVQ s_base+0(FP), SI
+	MOVQ m_base+24(FP), DI
+	MOVQ s_len+8(FP), CX
+	XORL AX, AX
+	MOVQ CX, R8
+	SHRQ $2, R8
+	JZ masktail
+	SETUP
+maskloop:
+	VMOVDQU (DI), Ydata        // Ydata = m
+	VPANDN (SI), Ydata, Ydata  // Ydata = s &^ m  (= (NOT m) AND s)
+	COUNTBLOCK
+	ADDQ $32, SI
+	ADDQ $32, DI
+	DECQ R8
+	JNZ maskloop
+	HSUM
+masktail:
+	ANDL $3, CX
+	JZ maskdone
+masktailloop:
+	MOVQ (DI), R10
+	NOTQ R10                 // ~m
+	MOVQ (SI), DX
+	ANDQ R10, DX             // s &^ m = s & ~m, one word
+	POPCNTQ DX, DX
+	ADDQ DX, AX
+	ADDQ $8, SI
+	ADDQ $8, DI
+	DECL CX
+	JNZ masktailloop
+maskdone:
+	VZEROUPPER
+	MOVQ AX, ret+48(FP)
+	RET
+
+// func _hasAVX2() bool
+// Reports whether the CPU supports AVX2 and the OS has enabled the wide (YMM)
+// register state. All three checks must pass; otherwise the Go wrappers fall
+// back to the scalar implementation. Note CPUID clobbers AX/BX/CX/DX.
+//
+// Each check complements the feature word and then TESTs the bits of interest:
+// ZF is set exactly when every required bit was set in the original value. That
+// needs only one large immediate instead of the two an AND/CMP pair would
+// encode, and it lets all three checks converge on a single SETEQ, which stores
+// the final ZF straight into the bool result. Since this runs once per process,
+// code size matters more here than the (negligible) speed difference.
+TEXT ·_hasAVX2(SB), NOSPLIT, $0-1
+	// CPUID leaf 1: require OSXSAVE (ECX bit 27) and AVX (ECX bit 28).
+	MOVL $1, AX
+	XORL CX, CX
+	CPUID
+	NOTL CX
+	TESTL $0x18000000, CX
+	JNE noavx2
+	// XGETBV(0): the OS must have enabled saving of SSE and AVX/YMM state, i.e.
+	// XCR0 bits 1 and 2. Without this the YMM registers would be corrupted
+	// across a context switch even though the CPU supports the instructions.
+	XORL CX, CX
+	XGETBV
+	NOTL AX
+	TESTL $0x6, AX
+	JNE noavx2
+	// CPUID leaf 7, sub-leaf 0: require AVX2 itself (EBX bit 5). The sub-leaf is
+	// selected via ECX, which must be 0.
+	MOVL $7, AX
+	XORL CX, CX
+	CPUID
+	NOTL BX
+	TESTL $0x20, BX
+noavx2:
+	SETEQ ret+0(FP)          // ZF is still set by whichever TESTL ran last
+	RET

vendor/github.com/RoaringBitmap/roaring/v2/popcnt_generic.go

@@ -0,0 +1,24 @@
+//go:build (!amd64 && !arm64) || appengine
+// +build !amd64,!arm64 appengine
+
+package roaring
+
+func popcntSlice(s []uint64) uint64 {
+	return popcntSliceGo(s)
+}
+
+func popcntMaskSlice(s, m []uint64) uint64 {
+	return popcntMaskSliceGo(s, m)
+}
+
+func popcntAndSlice(s, m []uint64) uint64 {
+	return popcntAndSliceGo(s, m)
+}
+
+func popcntOrSlice(s, m []uint64) uint64 {
+	return popcntOrSliceGo(s, m)
+}
+
+func popcntXorSlice(s, m []uint64) uint64 {
+	return popcntXorSliceGo(s, m)
+}

vendor/github.com/RoaringBitmap/roaring/v2/popcnt_neon_arm64.go

@@ -0,0 +1,65 @@
+//go:build arm64 && !appengine
+// +build arm64,!appengine
+
+package roaring
+
+// The functions below are implemented in popcnt_neon_arm64.s using NEON
+// (Advanced SIMD). NEON is mandatory in the ARMv8-A baseline that every arm64
+// CPU implements, so — unlike the amd64 AVX2 path, which is gated on a runtime
+// _hasAVX2 check — these routines are always used on arm64. The pure-Go
+// fallbacks in popcnt_slices.go remain in use on other architectures and on
+// appengine builds, which compile popcnt_generic.go instead.
+
+//go:noescape
+func _popcntSliceNEON(s []uint64) uint64
+
+//go:noescape
+func _popcntMaskSliceNEON(s, m []uint64) uint64
+
+//go:noescape
+func _popcntAndSliceNEON(s, m []uint64) uint64
+
+//go:noescape
+func _popcntOrSliceNEON(s, m []uint64) uint64
+
+//go:noescape
+func _popcntXorSliceNEON(s, m []uint64) uint64
+
+// useNEON is always true on arm64; it exists so tests can force the scalar
+// fallback path and to mirror the amd64 dispatch structure.
+var useNEON = true
+
+func popcntSlice(s []uint64) uint64 {
+	if useNEON {
+		return _popcntSliceNEON(s)
+	}
+	return popcntSliceGo(s)
+}
+
+func popcntMaskSlice(s, m []uint64) uint64 {
+	if useNEON {
+		return _popcntMaskSliceNEON(s, m)
+	}
+	return popcntMaskSliceGo(s, m)
+}
+
+func popcntAndSlice(s, m []uint64) uint64 {
+	if useNEON {
+		return _popcntAndSliceNEON(s, m)
+	}
+	return popcntAndSliceGo(s, m)
+}
+
+func popcntOrSlice(s, m []uint64) uint64 {
+	if useNEON {
+		return _popcntOrSliceNEON(s, m)
+	}
+	return popcntOrSliceGo(s, m)
+}
+
+func popcntXorSlice(s, m []uint64) uint64 {
+	if useNEON {
+		return _popcntXorSliceNEON(s, m)
+	}
+	return popcntXorSliceGo(s, m)
+}

vendor/github.com/RoaringBitmap/roaring/v2/popcnt_neon_arm64.s

@@ -0,0 +1,340 @@
+// +build arm64,!appengine
+
+//go:build arm64 && !appengine
+
+#include "textflag.h"
+
+// NEON (Advanced SIMD) population-count routines for arm64. They count the set
+// bits across a []uint64 (the backing storage of a bitmap container), optionally
+// combining each pair of words with a boolean op first: And, Or, Xor, and Mask
+// (s &^ m). NEON is mandatory in the ARMv8-A baseline that every arm64 CPU
+// implements, so unlike the amd64 AVX2 code there is no runtime feature check:
+// these routines are always used on arm64 (see popcnt_neon_arm64.go).
+//
+// Algorithm (VCNT byte popcount + widening accumulation)
+// ------------------------------------------------------
+// arm64 has a dedicated per-byte popcount instruction, VCNT, which replaces each
+// byte of a 128-bit register with the popcount (0..8) of the input byte. Turning
+// those per-byte counts into a running total means widening and accumulating,
+// and the loop is shaped to keep the arithmetic units busy:
+//   - each iteration loads four 16-byte vectors (64 bytes = 8 words) and VCNTs
+//     them independently, then sums the four with byte-wise VADD. Four counts of
+//     at most 8 sum to at most 32, so no byte lane overflows.
+//   - the summed bytes are folded into 16-bit lanes with add-wide: VUADDW takes
+//     the low 8 bytes into partial accumulator V16 and VUADDW2 the high 8 into
+//     V18. Two separate accumulators keep those adds off each other's
+//     dependency chain. The obvious-looking alternative, accumulating pairs of
+//     byte lanes straight into the 16-bit accumulator with UADALP, would save
+//     an instruction per iteration but is a pessimisation in practice: it has
+//     no Go assembler mnemonic, and it issues at roughly 2.4 per cycle on an
+//     Apple M4 where VADD/VCNT/VUADDW all issue at 4 per cycle. Measured, it
+//     costs this routine about 20%.
+//   - a 16-bit lane would eventually overflow, so every INNERMAX iterations the
+//     partials are drained (widened again) into a 4x32-bit accumulator (V17)
+//     that cannot realistically overflow, and the partials are re-zeroed.
+//   - at the end VUADDLV sums the four 32-bit lanes into a scalar.
+// A scalar-width NEON tail (VCNT + VUADDLV on one 64-bit word at a time) mops up
+// the trailing len%8 words, so any slice length is counted correctly.
+//
+// Go assembler conventions used below
+// -----------------------------------
+//   - Operands are written source(s) first, destination LAST. So
+//     "VAND V4.B16, V0.B16, V0.B16" means V0 = V0 AND V4.
+//   - Vn.B16/H8/H4/S4/D1 name the arrangement (element size x count) an
+//     instruction operates on: B16 = 16 bytes, H8/H4 = 8/4 halfwords, S4 = 4
+//     words, D1 = 1 doubleword. The same physical register is viewed either way.
+//   - VLD1.P post-increments the pointer register by the number of bytes loaded.
+//   - Arguments/results are read from the frame pointer (FP). A Go slice is a
+//     3-word header {ptr,len,cap}: s_base+0(FP), s_len+8(FP); a second slice
+//     argument starts at +24(FP). The uint64 result slot follows the args
+//     (ret+24(FP) for one slice arg, ret+48(FP) for two).
+//   - Every routine is a leaf (makes no calls): NOSPLIT with a $0 local frame.
+
+// INNERMAX bounds how many 64-byte iterations fold into the 16-bit partial
+// accumulators before they are drained into the wider one. Each iteration adds
+// at most 32 (four byte-popcounts of at most 8) to a 16-bit lane, and
+// 1024*32 = 32768 stays well under the 65535 lane limit.
+#define INNERMAX $1024
+
+// FOLD4 assumes 64 bytes of input (post-combine) sit in V0..V3 and folds their
+// popcount into the partial accumulators V16/V18. VADD sums the four VCNT
+// results byte-wise (each lane 0..32); VUADDW/VUADDW2 then widen the low/high
+// halves into the two 16-bit accumulators.
+#define FOLD4 \
+	VCNT V0.B16, V0.B16 \
+	VCNT V1.B16, V1.B16 \
+	VCNT V2.B16, V2.B16 \
+	VCNT V3.B16, V3.B16 \
+	VADD V1.B16, V0.B16, V0.B16 \
+	VADD V3.B16, V2.B16, V2.B16 \
+	VADD V2.B16, V0.B16, V0.B16 \
+	VUADDW V0.B8, V16.H8, V16.H8 \
+	VUADDW2 V0.B16, V18.H8, V18.H8
+
+// ZEROPART re-zeroes the two 16-bit partial accumulators at the start of each
+// INNERMAX batch.
+#define ZEROPART \
+	VEOR V16.B16, V16.B16, V16.B16 \
+	VEOR V18.B16, V18.B16, V18.B16
+
+// DRAIN widens the 16-bit partials V16/V18 into the 32-bit accumulator V17
+// (VUADDW low four halfwords, VUADDW2 high four, for each) and re-zeroes them.
+#define DRAIN \
+	VUADDW V16.H4, V17.S4, V17.S4 \
+	VUADDW2 V16.H8, V17.S4, V17.S4 \
+	VUADDW V18.H4, V17.S4, V17.S4 \
+	VUADDW2 V18.H8, V17.S4, V17.S4 \
+	ZEROPART
+
+// REDUCE sums the four 32-bit lanes of V17 into a scalar and adds it to R2 (the
+// running result). VUADDLV over .S4 yields a 64-bit sum; VMOV lifts it to a GPR.
+#define REDUCE \
+	VUADDLV V17.S4, V0 \
+	VMOV V0.D[0], R4 \
+	ADD R4, R2, R2
+
+// TAILWORD popcounts the single 64-bit word already loaded into V0's low lane
+// and adds it to R2: VCNT counts each of the 8 bytes, VUADDLV sums them.
+#define TAILWORD \
+	VCNT V0.B8, V0.B8 \
+	VUADDLV V0.B8, V0 \
+	VMOV V0.S[0], R4 \
+	ADD R4, R2, R2
+
+// func _popcntSliceNEON(s []uint64) uint64
+// Returns the total number of set bits in s. This is the canonical routine; the
+// And/Or/Xor/Mask variants below share its structure and differ only by the
+// boolean op applied to the two inputs before counting.
+TEXT ·_popcntSliceNEON(SB), NOSPLIT, $0-32
+	MOVD s_base+0(FP), R0            // R0 = &s[0]
+	MOVD s_len+8(FP), R1            // R1 = len(s), in 64-bit words
+	MOVD $0, R2                    // R2 = running result
+	VEOR V17.B16, V17.B16, V17.B16 // zero the 32-bit accumulator
+	LSR $3, R1, R3                 // R3 = len/8 = number of 64-byte blocks
+	CBZ R3, sltail                 // fewer than 8 words: skip the vector loop
+slblock:
+	MOVD INNERMAX, R4              // R4 = min(remaining blocks, INNERMAX)
+	CMP R4, R3
+	BHS slinner
+	MOVD R3, R4
+slinner:
+	SUB R4, R3, R3                 // R3 -= this batch's block count
+	ZEROPART
+slloop:
+	VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] // load 8 words (64 bytes)
+	FOLD4                          // partials += popcount(those 64 bytes)
+	SUBS $1, R4, R4
+	BNE slloop
+	DRAIN                          // fold partials into V17, re-zero them
+	CBNZ R3, slblock               // more blocks remain
+	REDUCE                         // R2 += sum of V17's lanes
+sltail:
+	AND $7, R1, R1                 // leftover words (0..7)
+	CBZ R1, sldone
+sltailloop:
+	VLD1.P 8(R0), [V0.D1]          // load one word, advance R0 by 8
+	TAILWORD
+	SUBS $1, R1, R1
+	BNE sltailloop
+sldone:
+	MOVD R2, ret+24(FP)
+	RET
+
+// func _popcntAndSliceNEON(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] & m[i]). Mirrors _popcntSliceNEON but loads
+// four vectors from each of s and m and ANDs them before counting. s and m are
+// assumed to have equal length.
+TEXT ·_popcntAndSliceNEON(SB), NOSPLIT, $0-56
+	MOVD s_base+0(FP), R0           // R0 = &s[0]
+	MOVD m_base+24(FP), R1          // R1 = &m[0]
+	MOVD s_len+8(FP), R5           // R5 = len
+	MOVD $0, R2
+	VEOR V17.B16, V17.B16, V17.B16
+	LSR $3, R5, R3
+	CBZ R3, andtail
+andblock:
+	MOVD INNERMAX, R4
+	CMP R4, R3
+	BHS andinner
+	MOVD R3, R4
+andinner:
+	SUB R4, R3, R3
+	ZEROPART
+andloop:
+	VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16]
+	VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16]
+	VAND V4.B16, V0.B16, V0.B16    // V0 = s & m
+	VAND V5.B16, V1.B16, V1.B16
+	VAND V6.B16, V2.B16, V2.B16
+	VAND V7.B16, V3.B16, V3.B16
+	FOLD4
+	SUBS $1, R4, R4
+	BNE andloop
+	DRAIN
+	CBNZ R3, andblock
+	REDUCE
+andtail:
+	AND $7, R5, R5
+	CBZ R5, anddone
+andtailloop:
+	VLD1.P 8(R0), [V0.D1]
+	VLD1.P 8(R1), [V1.D1]
+	VAND V1.B8, V0.B8, V0.B8       // s & m, one word
+	TAILWORD
+	SUBS $1, R5, R5
+	BNE andtailloop
+anddone:
+	MOVD R2, ret+48(FP)            // +48: result follows two 24-byte slice headers
+	RET
+
+// func _popcntOrSliceNEON(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] | m[i]); see _popcntAndSliceNEON for the
+// shared structure.
+TEXT ·_popcntOrSliceNEON(SB), NOSPLIT, $0-56
+	MOVD s_base+0(FP), R0
+	MOVD m_base+24(FP), R1
+	MOVD s_len+8(FP), R5
+	MOVD $0, R2
+	VEOR V17.B16, V17.B16, V17.B16
+	LSR $3, R5, R3
+	CBZ R3, ortail
+orblock:
+	MOVD INNERMAX, R4
+	CMP R4, R3
+	BHS orinner
+	MOVD R3, R4
+orinner:
+	SUB R4, R3, R3
+	ZEROPART
+orloop:
+	VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16]
+	VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16]
+	VORR V4.B16, V0.B16, V0.B16    // V0 = s | m
+	VORR V5.B16, V1.B16, V1.B16
+	VORR V6.B16, V2.B16, V2.B16
+	VORR V7.B16, V3.B16, V3.B16
+	FOLD4
+	SUBS $1, R4, R4
+	BNE orloop
+	DRAIN
+	CBNZ R3, orblock
+	REDUCE
+ortail:
+	AND $7, R5, R5
+	CBZ R5, ordone
+ortailloop:
+	VLD1.P 8(R0), [V0.D1]
+	VLD1.P 8(R1), [V1.D1]
+	VORR V1.B8, V0.B8, V0.B8       // s | m, one word
+	TAILWORD
+	SUBS $1, R5, R5
+	BNE ortailloop
+ordone:
+	MOVD R2, ret+48(FP)
+	RET
+
+// func _popcntXorSliceNEON(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] ^ m[i]); see _popcntAndSliceNEON for the
+// shared structure.
+TEXT ·_popcntXorSliceNEON(SB), NOSPLIT, $0-56
+	MOVD s_base+0(FP), R0
+	MOVD m_base+24(FP), R1
+	MOVD s_len+8(FP), R5
+	MOVD $0, R2
+	VEOR V17.B16, V17.B16, V17.B16
+	LSR $3, R5, R3
+	CBZ R3, xortail
+xorblock:
+	MOVD INNERMAX, R4
+	CMP R4, R3
+	BHS xorinner
+	MOVD R3, R4
+xorinner:
+	SUB R4, R3, R3
+	ZEROPART
+xorloop:
+	VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16]
+	VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16]
+	VEOR V4.B16, V0.B16, V0.B16    // V0 = s ^ m
+	VEOR V5.B16, V1.B16, V1.B16
+	VEOR V6.B16, V2.B16, V2.B16
+	VEOR V7.B16, V3.B16, V3.B16
+	FOLD4
+	SUBS $1, R4, R4
+	BNE xorloop
+	DRAIN
+	CBNZ R3, xorblock
+	REDUCE
+xortail:
+	AND $7, R5, R5
+	CBZ R5, xordone
+xortailloop:
+	VLD1.P 8(R0), [V0.D1]
+	VLD1.P 8(R1), [V1.D1]
+	VEOR V1.B8, V0.B8, V0.B8       // s ^ m, one word
+	TAILWORD
+	SUBS $1, R5, R5
+	BNE xortailloop
+xordone:
+	MOVD R2, ret+48(FP)
+	RET
+
+// func _popcntMaskSliceNEON(s, m []uint64) uint64
+// Returns the sum of popcount(s[i] &^ m[i]) == popcount(s & ~m). Same structure
+// as _popcntAndSliceNEON, except that s &^ m is formed with a single instruction
+// per vector instead of an invert-then-AND pair.
+//
+// arm64 does have a vector and-not, BIC, but the Go assembler exposes no
+// mnemonic for it. VBIT fits just as well and is spelled. BIT ("bitwise insert
+// if true") selects bit by bit between the destination and one source, under
+// the control of the other source:
+//
+//	Vd<i> = Vm<i> ? Vn<i> : Vd<i>     i.e.  Vd = (Vm & Vn) | (^Vm & Vd)
+//
+// Holding Vn at zero reduces that to "clear every bit of Vd that is set in Vm",
+// which is exactly Vd &^= Vm. So with Vd = s and Vm = m the mask is applied in
+// place in one op. V15 -- which used to hold all-ones so that m could be
+// inverted with VEOR -- is now simply kept at zero to serve as that Vn.
+TEXT ·_popcntMaskSliceNEON(SB), NOSPLIT, $0-56
+	MOVD s_base+0(FP), R0
+	MOVD m_base+24(FP), R1
+	MOVD s_len+8(FP), R5
+	MOVD $0, R2
+	VEOR V15.B16, V15.B16, V15.B16 // V15 = 0, the "insert" source for VBIT
+	VEOR V17.B16, V17.B16, V17.B16
+	LSR $3, R5, R3
+	CBZ R3, masktail
+maskblock:
+	MOVD INNERMAX, R4
+	CMP R4, R3
+	BHS maskinner
+	MOVD R3, R4
+maskinner:
+	SUB R4, R3, R3
+	ZEROPART
+maskloop:
+	VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16]
+	VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16]
+	VBIT V4.B16, V15.B16, V0.B16   // V0 = s &^ m
+	VBIT V5.B16, V15.B16, V1.B16
+	VBIT V6.B16, V15.B16, V2.B16
+	VBIT V7.B16, V15.B16, V3.B16
+	FOLD4
+	SUBS $1, R4, R4
+	BNE maskloop
+	DRAIN
+	CBNZ R3, maskblock
+	REDUCE
+masktail:
+	AND $7, R5, R5
+	CBZ R5, maskdone
+masktailloop:
+	VLD1.P 8(R0), [V0.D1]
+	VLD1.P 8(R1), [V1.D1]
+	VBIT V1.B8, V15.B8, V0.B8      // s &^ m, one word
+	TAILWORD
+	SUBS $1, R5, R5
+	BNE masktailloop
+maskdone:
+	MOVD R2, ret+48(FP)
+	RET

vendor/github.com/RoaringBitmap/roaring/v2/popcnt_slices.go

@@ -0,0 +1,43 @@
+package roaring
+
+import "math/bits"
+
+func popcntSliceGo(s []uint64) uint64 {
+	cnt := uint64(0)
+	for _, x := range s {
+		cnt += uint64(bits.OnesCount64(x))
+	}
+	return cnt
+}
+
+func popcntMaskSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] &^ m[i]))
+	}
+	return cnt
+}
+
+func popcntAndSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] & m[i]))
+	}
+	return cnt
+}
+
+func popcntOrSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] | m[i]))
+	}
+	return cnt
+}
+
+func popcntXorSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] ^ m[i]))
+	}
+	return cnt
+}

vendor/github.com/RoaringBitmap/roaring/v2/priorityqueue.go

@@ -0,0 +1,101 @@
+package roaring
+
+import "container/heap"
+
+/////////////
+// The priorityQueue is used to keep Bitmaps sorted.
+////////////
+
+type item struct {
+	value *Bitmap
+	index int
+}
+
+type priorityQueue []*item
+
+func (pq priorityQueue) Len() int { return len(pq) }
+
+func (pq priorityQueue) Less(i, j int) bool {
+	return pq[i].value.GetSizeInBytes() < pq[j].value.GetSizeInBytes()
+}
+
+func (pq priorityQueue) Swap(i, j int) {
+	pq[i], pq[j] = pq[j], pq[i]
+	pq[i].index = i
+	pq[j].index = j
+}
+
+func (pq *priorityQueue) Push(x interface{}) {
+	n := len(*pq)
+	item := x.(*item)
+	item.index = n
+	*pq = append(*pq, item)
+}
+
+func (pq *priorityQueue) Pop() interface{} {
+	old := *pq
+	n := len(old)
+	item := old[n-1]
+	item.index = -1 // for safety
+	*pq = old[0 : n-1]
+	return item
+}
+
+func (pq *priorityQueue) update(item *item, value *Bitmap) {
+	item.value = value
+	heap.Fix(pq, item.index)
+}
+
+/////////////
+// The containerPriorityQueue is used to keep the containers of various Bitmaps sorted.
+////////////
+
+type containeritem struct {
+	value    *Bitmap
+	keyindex int
+	index    int
+}
+
+type containerPriorityQueue []*containeritem
+
+func (pq containerPriorityQueue) Len() int { return len(pq) }
+
+func (pq containerPriorityQueue) Less(i, j int) bool {
+	k1 := pq[i].value.highlowcontainer.getKeyAtIndex(pq[i].keyindex)
+	k2 := pq[j].value.highlowcontainer.getKeyAtIndex(pq[j].keyindex)
+	if k1 != k2 {
+		return k1 < k2
+	}
+	c1 := pq[i].value.highlowcontainer.getContainerAtIndex(pq[i].keyindex)
+	c2 := pq[j].value.highlowcontainer.getContainerAtIndex(pq[j].keyindex)
+
+	return c1.getCardinality() > c2.getCardinality()
+}
+
+func (pq containerPriorityQueue) Swap(i, j int) {
+	pq[i], pq[j] = pq[j], pq[i]
+	pq[i].index = i
+	pq[j].index = j
+}
+
+func (pq *containerPriorityQueue) Push(x interface{}) {
+	n := len(*pq)
+	item := x.(*containeritem)
+	item.index = n
+	*pq = append(*pq, item)
+}
+
+func (pq *containerPriorityQueue) Pop() interface{} {
+	old := *pq
+	n := len(old)
+	item := old[n-1]
+	item.index = -1 // for safety
+	*pq = old[0 : n-1]
+	return item
+}
+
+//func (pq *containerPriorityQueue) update(item *containeritem, value *Bitmap, keyindex int) {
+//	item.value = value
+//	item.keyindex = keyindex
+//	heap.Fix(pq, item.index)
+//}

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/BSI_BENCHMARKS.md

@@ -0,0 +1,55 @@
+# BSI64 Benchmarks
+
+These notes capture local benchmark results for the BSI64 `BatchEqual` and
+comparison paths. They are intended as reproducible PR evidence, not as
+contractual performance guarantees.
+
+Environment:
+
+- CPU: 12th Gen Intel(R) Core(TM) i7-1255U
+- OS/arch: linux/amd64
+- Package: `github.com/RoaringBitmap/roaring/v2/roaring64`
+
+Commands:
+
+```sh
+go test ./roaring64 -count=1
+go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64BatchEqual' -benchmem -count 3
+go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64Compare(Big)?Value|BenchmarkBSI64BatchEqual(Big)?LargeAgeFixture' -benchmem -count 1
+go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64CompareBSISameRow' -benchmem -count=5
+go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64GetBigValue' -benchmem -count=3
+go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64BatchEqual.*LargeFixture' -benchmem -benchtime=2s -count=5
+```
+
+Representative results:
+
+| Benchmark | Before | After | Notes |
+| --- | ---: | ---: | --- |
+| `BenchmarkBSI64BatchEqualLargeAgeFixture` | ~13-14s/op, ~12.4GB/op | ~145-205ms/op, ~25.5MB/op | Avoids row-by-row `GetBigValue` for int64-width values. |
+| `BenchmarkBSI64BatchEqualM128Scattered` | ~1.25s/op, ~458MB/op | ~11-17ms/op, ~12.5MB/op | Detects complete bit-cube value patterns. |
+| `BenchmarkBSI64CompareValueEQLargeAgeFixture` | ~4.44s/op, ~461MB/op | ~100-118ms/op, ~19.7MB/op | `EQ` delegates to optimized `BatchEqual`. |
+| `BenchmarkBSI64CompareValueRangeLargeAgeFixture` | ~7.49s/op, ~501MB/op | ~204-224ms/op, ~122.6MB/op | Uses bitmap-native signed int64 comparison. |
+| `BenchmarkBSI64CompareValueGELargeAgeFixture` | ~3.45s/op, ~500MB/op | ~168-184ms/op, ~82.3MB/op | Uses bitmap-native signed int64 comparison. |
+| `BenchmarkBSI64CompareBSISameRowBitwise` | ~127-168ms/op, ~69.7MB/op | ~568-795us/op, ~619KB/op | Compares two BSI values per column ID through bitplane algebra instead of row-by-row `GetBigValue`. |
+| `BenchmarkBSI64GetBigValuesLargeFixture` | ~69-92ms/op, ~35.6MB/op, ~1.3M allocs/op for a row-by-row `GetBigValue` loop | ~23-34ms/op, ~8.2MB/op, ~200k allocs/op | Extracts aligned BSI values for a column batch by walking bit-slices once. |
+| `BenchmarkBSI64BatchEqualValuesLargeFixture` | ~5.4-7.1ms/op for `BatchEqual` plus `GetBigValues`; ~10.9-13.0ms/op for `BatchEqual` plus row-by-row `GetValue` | ~1.6-2.3ms/op, ~2.0MB/op, ~432 allocs/op | Emits matched column IDs and int64 values directly from trie leaves, avoiding a second value lookup pass. |
+
+Compatibility:
+
+- Public method signatures are unchanged.
+- `CompareBigValue` and `BatchEqualBig` internally delegate to the optimized
+  int64 paths only when the BSI and query values fit in signed 64-bit space.
+- True wider-than-64-bit values continue to use the existing generic paths.
+- `BatchEqualBig` now keys values by sign and magnitude so positive and negative
+  values with the same magnitude do not collide.
+- `GetBigValues` returns values aligned to the requested column IDs, with nil
+  entries for missing values, while preserving `GetBigValue` semantics.
+- `BatchEqualValues` returns matched column IDs and int64 values for `BatchEqual`
+  shapes, optionally restricted by a found set. Result order is intentionally
+  unspecified.
+
+Follow-up:
+
+- This change is scoped to `roaring64`. The 32-bit `BitSliceIndexing` package
+  already has separate `BatchEqual` coverage, and `CompareValue` parity can be
+  addressed in a follow-up PR with its own benchmarks and signed-value tests.

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/Makefile

@@ -0,0 +1,106 @@
+.PHONY: help all test format fmtcheck vet lint     qa deps clean nuke ser fetch-real-roaring-datasets
+
+
+
+
+
+
+
+
+# Display general help about this command
+help:
+	@echo ""
+	@echo "The following commands are available:"
+	@echo ""
+	@echo "    make qa          : Run all the tests"
+	@echo "    make test        : Run the unit tests"
+	@echo ""
+	@echo "    make format      : Format the source code"
+	@echo "    make fmtcheck    : Check if the source code has been formatted"
+	@echo "    make vet         : Check for suspicious constructs"
+	@echo "    make lint        : Check for style errors"
+	@echo ""
+	@echo "    make deps        : Get the dependencies"
+	@echo "    make clean       : Remove any build artifact"
+	@echo "    make nuke        : Deletes any intermediate file"
+	@echo ""
+	@echo "    make fuzz-smat   : Fuzzy testing with smat"
+	@echo "    make fuzz-stream : Fuzzy testing with stream deserialization"
+	@echo "    make fuzz-buffer : Fuzzy testing with buffer deserialization"
+	@echo ""
+
+# Alias for help target
+all: help
+test:
+	go test
+# Format the source code
+format:
+	@find ./ -type f -name "*.go" -exec gofmt -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+	@mkdir -p target
+	@find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff
+	@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+	GOPATH=$(GOPATH) go vet ./...
+
+# Check for style errors
+lint:
+	GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./...
+
+
+
+
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+	GOPATH=$(GOPATH) go get github.com/stretchr/testify
+	GOPATH=$(GOPATH) go get github.com/bits-and-blooms/bitset
+	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+	GOPATH=$(GOPATH) go get github.com/mschoch/smat
+	GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz
+	GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build
+	GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream
+	GOPATH=$(GOPATH) go get github.com/philhofer/fwd
+	GOPATH=$(GOPATH) go get github.com/jtolds/gls
+
+fuzz-smat:
+	go test -tags=gofuzz -run=TestGenerateSmatCorpus
+	go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-stream:
+	go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+fuzz-buffer:
+	go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+# Remove any build artifact
+clean:
+	GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+	rm -rf ./target
+	GOPATH=$(GOPATH) go clean -i ./...
+
+
+cover:
+	go test -coverprofile=coverage.out
+	go tool cover -html=coverage.out
+
+fetch-real-roaring-datasets:
+	# pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets
+	git submodule init
+	git submodule update

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/fastaggregation64.go

@@ -0,0 +1,31 @@
+package roaring64
+
+// FastAnd computes the intersection between many bitmaps quickly
+// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
+// of manually calling "And" many times.
+func FastAnd(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+	answer := And(bitmaps[0], bitmaps[1])
+	for _, bm := range bitmaps[2:] {
+		answer.And(bm)
+	}
+	return answer
+}
+
+// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly.
+func FastOr(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+	answer := Or(bitmaps[0], bitmaps[1])
+	for _, bm := range bitmaps[2:] {
+		answer.Or(bm)
+	}
+	return answer
+}

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/iter.go

@@ -0,0 +1,31 @@
+package roaring64
+
+import "iter"
+
+// Values returns an iterator that yields the elements of the bitmap in
+// increasing order. Starting with Go 1.23, users can use a for loop to iterate
+// over it.
+func Values(b *Bitmap) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		it := b.Iterator()
+		for it.HasNext() {
+			if !yield(it.Next()) {
+				return
+			}
+		}
+	}
+}
+
+// Backward returns an iterator that yields the elements of the bitmap in
+// decreasing order. Starting with Go 1.23, users can use a for loop to iterate
+// over it.
+func Backward(b *Bitmap) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		it := b.ReverseIterator()
+		for it.HasNext() {
+			if !yield(it.Next()) {
+				return
+			}
+		}
+	}
+}

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/iterables64.go

@@ -0,0 +1,214 @@
+package roaring64
+
+import (
+	"github.com/RoaringBitmap/roaring/v2"
+)
+
+// IntIterable64 allows you to iterate over the values in a Bitmap
+type IntIterable64 interface {
+	HasNext() bool
+	Next() uint64
+}
+
+// IntPeekable64 allows you to look at the next value without advancing and
+// advance as long as the next value is smaller than minval
+type IntPeekable64 interface {
+	IntIterable64
+	// PeekNext peeks the next value without advancing the iterator
+	PeekNext() uint64
+	// AdvanceIfNeeded advances as long as the next value is smaller than minval
+	AdvanceIfNeeded(minval uint64)
+}
+
+type intIterator struct {
+	pos              int
+	hs               uint64
+	iter             roaring.IntPeekable
+	highlowcontainer *roaringArray64
+
+	// These embedded iterators per container type help reduce load in the GC.
+	// This way, instead of making up-to 4 billion allocations per full iteration
+	// we get a single allocation and simply reinitialize the embedded iterator
+	// and point to it in the generic `iter` member on each key bound.
+	bitmapIter roaring.IntIterator
+}
+
+// HasNext returns true if there are more integers to iterate over
+func (ii *intIterator) HasNext() bool {
+	return ii.pos < ii.highlowcontainer.size()
+}
+
+func (ii *intIterator) init() {
+	if ii.highlowcontainer.size() > ii.pos {
+		ii.hs = uint64(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 32
+		ii.bitmapIter.Initialize(ii.highlowcontainer.getContainerAtIndex(ii.pos))
+		ii.iter = &ii.bitmapIter
+	}
+}
+
+// Next returns the next integer
+func (ii *intIterator) Next() uint64 {
+	lowbits := ii.iter.Next()
+	x := uint64(lowbits) | ii.hs
+	if !ii.iter.HasNext() {
+		ii.pos = ii.pos + 1
+		ii.init()
+	}
+	return x
+}
+
+// PeekNext peeks the next value without advancing the iterator
+func (ii *intIterator) PeekNext() uint64 {
+	return uint64(ii.iter.PeekNext()&maxLowBit) | ii.hs
+}
+
+// AdvanceIfNeeded advances as long as the next value is smaller than minval
+func (ii *intIterator) AdvanceIfNeeded(minval uint64) {
+	to := minval >> 32
+
+	for ii.HasNext() && (ii.hs>>32) < to {
+		ii.pos++
+		ii.init()
+	}
+
+	if ii.HasNext() && (ii.hs>>32) == to {
+		ii.iter.AdvanceIfNeeded(lowbits(minval))
+
+		if !ii.iter.HasNext() {
+			ii.pos++
+			ii.init()
+		}
+	}
+}
+
+// IntIterator64 is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
+type IntIterator64 = intIterator
+
+// Initialize configures the existing iterator so that it can iterate through the values of
+// the provided bitmap.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+func (ii *intIterator) Initialize(a *Bitmap) {
+	ii.pos = 0
+	ii.highlowcontainer = &a.highlowcontainer
+	ii.init()
+}
+
+func newIntIterator(a *Bitmap) *intIterator {
+	p := new(intIterator)
+	p.Initialize(a)
+	return p
+}
+
+type intReverseIterator struct {
+	pos              int
+	hs               uint64
+	iter             roaring.IntIterable
+	highlowcontainer *roaringArray64
+
+	// Stack-allocated embedded iterator to reduce GC pressure.
+	bitmapIter roaring.IntReverseIterator
+}
+
+// HasNext returns true if there are more integers to iterate over
+func (ii *intReverseIterator) HasNext() bool {
+	return ii.pos >= 0
+}
+
+func (ii *intReverseIterator) init() {
+	if ii.pos >= 0 {
+		ii.hs = uint64(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 32
+		ii.bitmapIter.Initialize(ii.highlowcontainer.getContainerAtIndex(ii.pos))
+		ii.iter = &ii.bitmapIter
+	} else {
+		ii.iter = nil
+	}
+}
+
+// Next returns the next integer
+func (ii *intReverseIterator) Next() uint64 {
+	x := uint64(ii.iter.Next()) | ii.hs
+	if !ii.iter.HasNext() {
+		ii.pos = ii.pos - 1
+		ii.init()
+	}
+	return x
+}
+
+// IntReverseIterator64 is meant to allow you to iterate through the values of a bitmap in reverse, see Initialize(a *Bitmap)
+type IntReverseIterator64 = intReverseIterator
+
+// Initialize configures the existing iterator so that it can iterate through the values of
+// the provided bitmap in reverse.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+func (ii *intReverseIterator) Initialize(a *Bitmap) {
+	ii.highlowcontainer = &a.highlowcontainer
+	ii.pos = a.highlowcontainer.size() - 1
+	ii.init()
+}
+
+func newIntReverseIterator(a *Bitmap) *intReverseIterator {
+	p := new(intReverseIterator)
+	p.Initialize(a)
+	return p
+}
+
+// ManyIntIterable64 allows you to iterate over the values in a Bitmap
+type ManyIntIterable64 interface {
+	// pass in a buffer to fill up with values, returns how many values were returned
+	NextMany([]uint64) int
+}
+
+type manyIntIterator struct {
+	pos              int
+	hs               uint64
+	iter             roaring.ManyIntIterable
+	highlowcontainer *roaringArray64
+
+	// Stack-allocated embedded iterator to reduce GC pressure.
+	bitmapIter roaring.ManyIntIterator
+}
+
+func (ii *manyIntIterator) init() {
+	if ii.highlowcontainer.size() > ii.pos {
+		ii.hs = uint64(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 32
+		ii.bitmapIter.Initialize(ii.highlowcontainer.getContainerAtIndex(ii.pos))
+		ii.iter = &ii.bitmapIter
+	} else {
+		ii.iter = nil
+	}
+}
+
+func (ii *manyIntIterator) NextMany(buf []uint64) int {
+	n := 0
+	for n < len(buf) {
+		if ii.iter == nil {
+			break
+		}
+		moreN := ii.iter.NextMany64(ii.hs, buf[n:])
+		n += moreN
+		if moreN == 0 {
+			ii.pos = ii.pos + 1
+			ii.init()
+		}
+	}
+
+	return n
+}
+
+// ManyIntIterator64 is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
+type ManyIntIterator64 = manyIntIterator
+
+// Initialize configures the existing iterator so that it can iterate through the values of
+// the provided bitmap.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+func (ii *manyIntIterator) Initialize(a *Bitmap) {
+	ii.pos = 0
+	ii.highlowcontainer = &a.highlowcontainer
+	ii.init()
+}
+
+func newManyIntIterator(a *Bitmap) *manyIntIterator {
+	p := new(manyIntIterator)
+	p.Initialize(a)
+	return p
+}

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/parallel64.go

@@ -0,0 +1,297 @@
+package roaring64
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/RoaringBitmap/roaring/v2"
+)
+
+var defaultWorkerCount = runtime.NumCPU()
+
+// ParOr computes the union (OR) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+	var lKey uint32 = maxUint32
+	var hKey uint32
+
+	bitmapsFiltered := bitmaps[:0]
+	for _, b := range bitmaps {
+		if !b.IsEmpty() {
+			bitmapsFiltered = append(bitmapsFiltered, b)
+		}
+	}
+	bitmaps = bitmapsFiltered
+
+	for _, b := range bitmaps {
+		lKey = minOfUint32(lKey, b.highlowcontainer.keys[0])
+		hKey = maxOfUint32(hKey, b.highlowcontainer.keys[b.highlowcontainer.size()-1])
+	}
+
+	if lKey == maxUint32 && hKey == 0 {
+		return New()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0]
+	}
+	// The following might overflow and we do not want that!
+	// as it might lead to a channel of size 0 later which,
+	// on some systems, would block indefinitely.
+	keyRange := uint64(hKey) - uint64(lKey) + 1
+	if keyRange == 1 {
+		// All bitmaps have the same key,
+		// we can merge the 32-bit roaring bitmaps in parallel
+		var bms32s = make([]*roaring.Bitmap, 0, len(bitmaps))
+		for _, b := range bitmaps {
+			bms32s = append(bms32s, b.highlowcontainer.containers...)
+		}
+		return roaring32AsRoaring64(roaring.ParOr(parallelism, bms32s...), lKey)
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+	// We cannot use int since int is 32-bit on 32-bit systems.
+	var chunkSize int64
+	var chunkCount int64
+	if int64(parallelism)*4 > int64(keyRange) {
+		chunkSize = 1
+		chunkCount = int64(keyRange)
+	} else {
+		chunkCount = int64(parallelism) * 4
+		chunkSize = (int64(keyRange) + chunkCount - 1) / chunkCount
+	}
+
+	if chunkCount*chunkSize < int64(keyRange) {
+		// it's fine to panic to indicate an implementation error
+		panic(fmt.Sprintf("invariant check failed: chunkCount * chunkSize < keyRange, %d * %d < %d", chunkCount, chunkSize, keyRange))
+	}
+
+	chunks := make([]*roaringArray64, chunkCount)
+
+	chunkSpecChan := make(chan parChunkSpec, minOfInt(maxOfInt(64, 2*parallelism), int(chunkCount)))
+	chunkChan := make(chan parChunk, minOfInt(32, int(chunkCount)))
+
+	orFunc := func() {
+		for spec := range chunkSpecChan {
+			ra := orOnRange(&bitmaps[0].highlowcontainer, &bitmaps[1].highlowcontainer, spec.start, spec.end)
+			for _, b := range bitmaps[2:] {
+				ra = iorOnRange(ra, &b.highlowcontainer, spec.start, spec.end)
+			}
+
+			chunkChan <- parChunk{ra, spec.idx}
+		}
+	}
+
+	for i := 0; i < parallelism; i++ {
+		go orFunc()
+	}
+
+	go func() {
+		for i := int64(0); i < chunkCount; i++ {
+			spec := parChunkSpec{
+				start: uint32(int64(lKey) + i*chunkSize),
+				end:   uint32(minOfInt64(int64(lKey)+(i+1)*chunkSize-1, int64(hKey))),
+				idx:   int(i),
+			}
+			chunkSpecChan <- spec
+		}
+	}()
+
+	chunksRemaining := chunkCount
+	for chunk := range chunkChan {
+		chunks[chunk.idx] = chunk.ra
+		chunksRemaining--
+		if chunksRemaining == 0 {
+			break
+		}
+	}
+	close(chunkChan)
+	close(chunkSpecChan)
+
+	containerCount := 0
+	for _, chunk := range chunks {
+		containerCount += chunk.size()
+	}
+
+	result := Bitmap{
+		roaringArray64{
+			containers:      make([]*roaring.Bitmap, containerCount),
+			keys:            make([]uint32, containerCount),
+			needCopyOnWrite: make([]bool, containerCount),
+		},
+	}
+
+	resultOffset := 0
+	for _, chunk := range chunks {
+		copy(result.highlowcontainer.containers[resultOffset:], chunk.containers)
+		copy(result.highlowcontainer.keys[resultOffset:], chunk.keys)
+		copy(result.highlowcontainer.needCopyOnWrite[resultOffset:], chunk.needCopyOnWrite)
+		resultOffset += chunk.size()
+	}
+
+	return &result
+}
+
+type parChunkSpec struct {
+	start uint32
+	end   uint32
+	idx   int
+}
+
+type parChunk struct {
+	ra  *roaringArray64
+	idx int
+}
+
+func (c parChunk) size() int {
+	return c.ra.size()
+}
+
+// parNaiveStartAt returns the index of the first key that is inclusive between start and last
+// Returns the size if there is no such key
+func parNaiveStartAt(ra *roaringArray64, start uint32, last uint32) int {
+	for idx, key := range ra.keys {
+		if key >= start && key <= last {
+			return idx
+		} else if key > last {
+			break
+		}
+	}
+	return ra.size()
+}
+
+func orOnRange(ra1, ra2 *roaringArray64, start, last uint32) *roaringArray64 {
+	answer := &roaringArray64{}
+	length1 := ra1.size()
+	length2 := ra2.size()
+
+	idx1 := parNaiveStartAt(ra1, start, last)
+	idx2 := parNaiveStartAt(ra2, start, last)
+
+	var key1 uint32
+	var key2 uint32
+	if idx1 < length1 && idx2 < length2 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		key2 = ra2.getKeyAtIndex(idx2)
+
+		for key1 <= last && key2 <= last {
+			if key1 < key2 {
+				answer.appendCopy(*ra1, idx1)
+				idx1++
+				if idx1 == length1 {
+					break
+				}
+				key1 = ra1.getKeyAtIndex(idx1)
+			} else if key1 > key2 {
+				answer.appendCopy(*ra2, idx2)
+				idx2++
+				if idx2 == length2 {
+					break
+				}
+				key2 = ra2.getKeyAtIndex(idx2)
+			} else {
+				c1 := ra1.getContainerAtIndex(idx1)
+
+				// answer.appendContainer(key1, c1.lazyOR(ra2.getContainerAtIndex(idx2)), false)
+				answer.appendContainer(key1, roaring.Or(c1, ra2.getContainerAtIndex(idx2)), false)
+				idx1++
+				idx2++
+				if idx1 == length1 || idx2 == length2 {
+					break
+				}
+
+				key1 = ra1.getKeyAtIndex(idx1)
+				key2 = ra2.getKeyAtIndex(idx2)
+			}
+		}
+	}
+
+	if idx2 < length2 {
+		key2 = ra2.getKeyAtIndex(idx2)
+		for key2 <= last {
+			answer.appendCopy(*ra2, idx2)
+			idx2++
+			if idx2 == length2 {
+				break
+			}
+			key2 = ra2.getKeyAtIndex(idx2)
+		}
+	}
+
+	if idx1 < length1 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		for key1 <= last {
+			answer.appendCopy(*ra1, idx1)
+			idx1++
+			if idx1 == length1 {
+				break
+			}
+			key1 = ra1.getKeyAtIndex(idx1)
+		}
+	}
+	return answer
+}
+
+func iorOnRange(ra1, ra2 *roaringArray64, start, last uint32) *roaringArray64 {
+	length1 := ra1.size()
+	length2 := ra2.size()
+
+	idx1 := 0
+	idx2 := parNaiveStartAt(ra2, start, last)
+
+	var key1 uint32
+	var key2 uint32
+	if idx1 < length1 && idx2 < length2 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		key2 = ra2.getKeyAtIndex(idx2)
+
+		for key1 <= last && key2 <= last {
+			if key1 < key2 {
+				idx1++
+				if idx1 >= length1 {
+					break
+				}
+				key1 = ra1.getKeyAtIndex(idx1)
+			} else if key1 > key2 {
+				ra1.insertNewKeyValueAt(idx1, key2, ra2.getContainerAtIndex(idx2))
+				ra1.needCopyOnWrite[idx1] = true
+				idx2++
+				idx1++
+				length1++
+				if idx2 >= length2 {
+					break
+				}
+				key2 = ra2.getKeyAtIndex(idx2)
+			} else {
+				c1 := ra1.getWritableContainerAtIndex(idx1)
+
+				// ra1.containers[idx1] = c1.lazyIOR(ra2.getContainerAtIndex(idx2))
+				c1.Or(ra2.getContainerAtIndex(idx2))
+				ra1.setContainerAtIndex(idx1, c1)
+
+				ra1.needCopyOnWrite[idx1] = false
+				idx1++
+				idx2++
+				if idx1 >= length1 || idx2 >= length2 {
+					break
+				}
+
+				key1 = ra1.getKeyAtIndex(idx1)
+				key2 = ra2.getKeyAtIndex(idx2)
+			}
+		}
+	}
+	if idx2 < length2 {
+		key2 = ra2.getKeyAtIndex(idx2)
+		for key2 <= last {
+			ra1.appendCopy(*ra2, idx2)
+			idx2++
+			if idx2 >= length2 {
+				break
+			}
+			key2 = ra2.getKeyAtIndex(idx2)
+		}
+	}
+	return ra1
+}

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/roaringarray64.go

@@ -0,0 +1,464 @@
+package roaring64
+
+import (
+	"errors"
+
+	"github.com/RoaringBitmap/roaring/v2"
+)
+
+type roaringArray64 struct {
+	keys            []uint32
+	containers      []*roaring.Bitmap
+	needCopyOnWrite []bool
+	copyOnWrite     bool
+}
+
+var (
+	// ErrKeySortOrder reports that container keys are out of order.
+	ErrKeySortOrder = errors.New("keys were out of order")
+	// ErrCardinalityConstraint reports inconsistent array cardinality metadata.
+	ErrCardinalityConstraint = errors.New("size of arrays was not coherent")
+)
+
+// runOptimize compresses the element containers to minimize space consumed.
+// Q: how does this interact with copyOnWrite and needCopyOnWrite?
+// A: since we aren't changing the logical content, just the representation,
+//
+//	we don't bother to check the needCopyOnWrite bits. We replace
+//	(possibly all) elements of ra.containers in-place with space
+//	optimized versions.
+func (ra *roaringArray64) runOptimize() {
+	for i := range ra.containers {
+		ra.containers[i].RunOptimize()
+	}
+}
+
+func (ra *roaringArray64) appendContainer(key uint32, value *roaring.Bitmap, mustCopyOnWrite bool) {
+	ra.keys = append(ra.keys, key)
+	ra.containers = append(ra.containers, value)
+	ra.needCopyOnWrite = append(ra.needCopyOnWrite, mustCopyOnWrite)
+}
+
+func (ra *roaringArray64) appendWithoutCopy(sa roaringArray64, startingindex int) {
+	mustCopyOnWrite := sa.needCopyOnWrite[startingindex]
+	ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], mustCopyOnWrite)
+}
+
+func (ra *roaringArray64) appendCopy(sa roaringArray64, startingindex int) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := (ra.copyOnWrite && sa.copyOnWrite) || sa.needsCopyOnWrite(startingindex)
+	if !copyonwrite {
+		// since there is no copy-on-write, we need to clone the container (this is important)
+		ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex].Clone(), copyonwrite)
+	} else {
+		ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex].Clone(), copyonwrite)
+		if !sa.needsCopyOnWrite(startingindex) {
+			sa.setNeedsCopyOnWrite(startingindex)
+		}
+	}
+}
+
+func (ra *roaringArray64) appendWithoutCopyMany(sa roaringArray64, startingindex, end int) {
+	for i := startingindex; i < end; i++ {
+		ra.appendWithoutCopy(sa, i)
+	}
+}
+
+func (ra *roaringArray64) appendCopyMany(sa roaringArray64, startingindex, end int) {
+	for i := startingindex; i < end; i++ {
+		ra.appendCopy(sa, i)
+	}
+}
+
+func (ra *roaringArray64) appendCopiesUntil(sa roaringArray64, stoppingKey uint32) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := ra.copyOnWrite && sa.copyOnWrite
+
+	for i := 0; i < sa.size(); i++ {
+		if sa.keys[i] >= stoppingKey {
+			break
+		}
+		thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i)
+		if thiscopyonewrite {
+			ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite)
+			if !sa.needsCopyOnWrite(i) {
+				sa.setNeedsCopyOnWrite(i)
+			}
+
+		} else {
+			// since there is no copy-on-write, we need to clone the container (this is important)
+			ra.appendContainer(sa.keys[i], sa.containers[i].Clone(), thiscopyonewrite)
+		}
+	}
+}
+
+func (ra *roaringArray64) appendCopiesAfter(sa roaringArray64, beforeStart uint32) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := ra.copyOnWrite && sa.copyOnWrite
+
+	startLocation := sa.getIndex(beforeStart)
+	if startLocation >= 0 {
+		startLocation++
+	} else {
+		startLocation = -startLocation - 1
+	}
+
+	for i := startLocation; i < sa.size(); i++ {
+		thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i)
+		if thiscopyonewrite {
+			ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite)
+			if !sa.needsCopyOnWrite(i) {
+				sa.setNeedsCopyOnWrite(i)
+			}
+		} else {
+			// since there is no copy-on-write, we need to clone the container (this is important)
+			ra.appendContainer(sa.keys[i], sa.containers[i].Clone(), thiscopyonewrite)
+		}
+	}
+}
+
+func (ra *roaringArray64) removeIndexRange(begin, end int) {
+	if end <= begin {
+		return
+	}
+
+	r := end - begin
+
+	copy(ra.keys[begin:], ra.keys[end:])
+	copy(ra.containers[begin:], ra.containers[end:])
+	copy(ra.needCopyOnWrite[begin:], ra.needCopyOnWrite[end:])
+
+	ra.resize(len(ra.keys) - r)
+}
+
+func (ra *roaringArray64) resize(newsize int) {
+	for k := newsize; k < len(ra.containers); k++ {
+		ra.keys[k] = 0
+		ra.needCopyOnWrite[k] = false
+		ra.containers[k] = nil
+	}
+
+	ra.keys = ra.keys[:newsize]
+	ra.containers = ra.containers[:newsize]
+	ra.needCopyOnWrite = ra.needCopyOnWrite[:newsize]
+}
+
+func (ra *roaringArray64) clear() {
+	ra.resize(0)
+	ra.copyOnWrite = false
+}
+
+func (ra *roaringArray64) clone() *roaringArray64 {
+	sa := roaringArray64{}
+	sa.copyOnWrite = ra.copyOnWrite
+
+	// this is where copyOnWrite is used.
+	if ra.copyOnWrite {
+		sa.keys = make([]uint32, len(ra.keys))
+		copy(sa.keys, ra.keys)
+		sa.containers = make([]*roaring.Bitmap, len(ra.containers))
+		copy(sa.containers, ra.containers)
+		sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite))
+
+		ra.markAllAsNeedingCopyOnWrite()
+		sa.markAllAsNeedingCopyOnWrite()
+
+		// sa.needCopyOnWrite is shared
+	} else {
+		// make a full copy
+
+		sa.keys = make([]uint32, len(ra.keys))
+		copy(sa.keys, ra.keys)
+
+		sa.containers = make([]*roaring.Bitmap, len(ra.containers))
+		for i := range sa.containers {
+			sa.containers[i] = ra.containers[i].Clone()
+		}
+
+		sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite))
+	}
+	return &sa
+}
+
+// clone all containers which have needCopyOnWrite set to true
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to.
+func (ra *roaringArray64) cloneCopyOnWriteContainers() {
+	for i, needCopyOnWrite := range ra.needCopyOnWrite {
+		if needCopyOnWrite {
+			ra.containers[i] = ra.containers[i].Clone()
+			ra.needCopyOnWrite[i] = false
+		}
+	}
+}
+
+// unused function:
+// func (ra *roaringArray64) containsKey(x uint32) bool {
+//	return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
+// }
+
+func (ra *roaringArray64) getContainer(x uint32) *roaring.Bitmap {
+	i := ra.binarySearch(0, int64(len(ra.keys)), x)
+	if i < 0 {
+		return nil
+	}
+	return ra.containers[i]
+}
+
+func (ra *roaringArray64) getContainerAtIndex(i int) *roaring.Bitmap {
+	return ra.containers[i]
+}
+
+func (ra *roaringArray64) getWritableContainerAtIndex(i int) *roaring.Bitmap {
+	if ra.needCopyOnWrite[i] {
+		ra.containers[i] = ra.containers[i].Clone()
+		ra.needCopyOnWrite[i] = false
+	}
+	return ra.containers[i]
+}
+
+func (ra *roaringArray64) getIndex(x uint32) int {
+	// before the binary search, we optimize for frequent cases
+	size := len(ra.keys)
+	if (size == 0) || (ra.keys[size-1] == x) {
+		return size - 1
+	}
+	return ra.binarySearch(0, int64(size), x)
+}
+
+func (ra *roaringArray64) getKeyAtIndex(i int) uint32 {
+	return ra.keys[i]
+}
+
+func (ra *roaringArray64) insertNewKeyValueAt(i int, key uint32, value *roaring.Bitmap) {
+	ra.keys = append(ra.keys, 0)
+	ra.containers = append(ra.containers, nil)
+
+	copy(ra.keys[i+1:], ra.keys[i:])
+	copy(ra.containers[i+1:], ra.containers[i:])
+
+	ra.keys[i] = key
+	ra.containers[i] = value
+
+	ra.needCopyOnWrite = append(ra.needCopyOnWrite, false)
+	copy(ra.needCopyOnWrite[i+1:], ra.needCopyOnWrite[i:])
+	ra.needCopyOnWrite[i] = false
+}
+
+func (ra *roaringArray64) remove(key uint32) bool {
+	i := ra.binarySearch(0, int64(len(ra.keys)), key)
+	if i >= 0 { // if a new key
+		ra.removeAtIndex(i)
+		return true
+	}
+	return false
+}
+
+func (ra *roaringArray64) removeAtIndex(i int) {
+	copy(ra.keys[i:], ra.keys[i+1:])
+	copy(ra.containers[i:], ra.containers[i+1:])
+
+	copy(ra.needCopyOnWrite[i:], ra.needCopyOnWrite[i+1:])
+
+	ra.resize(len(ra.keys) - 1)
+}
+
+func (ra *roaringArray64) setContainerAtIndex(i int, c *roaring.Bitmap) {
+	ra.containers[i] = c
+}
+
+func (ra *roaringArray64) replaceKeyAndContainerAtIndex(i int, key uint32, c *roaring.Bitmap, mustCopyOnWrite bool) {
+	ra.keys[i] = key
+	ra.containers[i] = c
+	ra.needCopyOnWrite[i] = mustCopyOnWrite
+}
+
+func (ra *roaringArray64) size() int {
+	return len(ra.keys)
+}
+
+func (ra *roaringArray64) binarySearch(begin, end int64, ikey uint32) int {
+	low := begin
+	high := end - 1
+	for low+16 <= high {
+		middleIndex := low + (high-low)/2 // avoid overflow
+		middleValue := ra.keys[middleIndex]
+
+		if middleValue < ikey {
+			low = middleIndex + 1
+		} else if middleValue > ikey {
+			high = middleIndex - 1
+		} else {
+			return int(middleIndex)
+		}
+	}
+	for ; low <= high; low++ {
+		val := ra.keys[low]
+		if val >= ikey {
+			if val == ikey {
+				return int(low)
+			}
+			break
+		}
+	}
+	return -int(low + 1)
+}
+
+func (ra *roaringArray64) equals(o interface{}) bool {
+	srb, ok := o.(roaringArray64)
+	if ok {
+
+		if srb.size() != ra.size() {
+			return false
+		}
+		for i, k := range ra.keys {
+			if k != srb.keys[i] {
+				return false
+			}
+		}
+
+		for i, c := range ra.containers {
+			if !c.Equals(srb.containers[i]) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func (ra *roaringArray64) hasRunCompression() bool {
+	for _, c := range ra.containers {
+		if c.HasRunCompression() {
+			return true
+		}
+	}
+	return false
+}
+
+/**
+ * Find the smallest integer index strictly larger than pos such that array[index].key&gt;=min. If none can
+ * be found, return size. Based on code by O. Kaser.
+ *
+ * @param min minimal value
+ * @param pos index to exceed
+ * @return the smallest index greater than pos such that array[index].key is at least as large as
+ *         min, or size if it is not possible.
+ */
+func (ra *roaringArray64) advanceUntil(min uint32, pos int) int {
+	lower := pos + 1
+
+	if lower >= len(ra.keys) || ra.keys[lower] >= min {
+		return lower
+	}
+
+	spansize := 1
+
+	for lower+spansize < len(ra.keys) && ra.keys[lower+spansize] < min {
+		spansize *= 2
+	}
+	var upper int
+	if lower+spansize < len(ra.keys) {
+		upper = lower + spansize
+	} else {
+		upper = len(ra.keys) - 1
+	}
+
+	if ra.keys[upper] == min {
+		return upper
+	}
+
+	if ra.keys[upper] < min {
+		// means
+		// array
+		// has no
+		// item
+		// >= min
+		// pos = array.length;
+		return len(ra.keys)
+	}
+
+	// we know that the next-smallest span was too small
+	lower += (spansize >> 1)
+
+	mid := 0
+	for lower+1 != upper {
+		mid = (lower + upper) >> 1
+		if ra.keys[mid] == min {
+			return mid
+		} else if ra.keys[mid] < min {
+			lower = mid
+		} else {
+			upper = mid
+		}
+	}
+	return upper
+}
+
+func (ra *roaringArray64) markAllAsNeedingCopyOnWrite() {
+	for i := range ra.needCopyOnWrite {
+		ra.needCopyOnWrite[i] = true
+	}
+}
+
+func (ra *roaringArray64) needsCopyOnWrite(i int) bool {
+	return ra.needCopyOnWrite[i]
+}
+
+func (ra *roaringArray64) setNeedsCopyOnWrite(i int) {
+	ra.needCopyOnWrite[i] = true
+}
+
+// should be dirt cheap
+func (ra *roaringArray64) serializedSizeInBytes() uint64 {
+	answer := uint64(8)
+	for _, c := range ra.containers {
+		answer += 4
+		answer += c.GetSerializedSizeInBytes()
+	}
+	return answer
+}
+
+func (ra *roaringArray64) checkKeysSorted() bool {
+	if len(ra.keys) == 0 || len(ra.keys) == 1 {
+		return true
+	}
+	previous := ra.keys[0]
+	for nextIdx := 1; nextIdx < len(ra.keys); nextIdx++ {
+		next := ra.keys[nextIdx]
+		if previous >= next {
+			return false
+		}
+		previous = next
+
+	}
+	return true
+}
+
+// validate checks the referential integrity
+// ensures len(keys) == len(containers), recurses and checks each container type
+func (ra *roaringArray64) validate() error {
+	if !ra.checkKeysSorted() {
+		return ErrKeySortOrder
+	}
+
+	if len(ra.keys) != len(ra.containers) {
+		return ErrCardinalityConstraint
+	}
+
+	if len(ra.keys) != len(ra.needCopyOnWrite) {
+		return ErrCardinalityConstraint
+	}
+
+	for _, maps := range ra.containers {
+		err := maps.Validate()
+		if err != nil {
+			return err
+		}
+		if maps.IsEmpty() {
+			return errors.New("empty container")
+		}
+	}
+
+	return nil
+}

vendor/github.com/RoaringBitmap/roaring/v2/roaring64/util.go

@@ -0,0 +1,49 @@
+package roaring64
+
+import "github.com/RoaringBitmap/roaring/v2"
+
+func highbits(x uint64) uint32 {
+	return uint32(x >> 32)
+}
+
+func lowbits(x uint64) uint32 {
+	return uint32(x & maxLowBit)
+}
+
+const maxLowBit = roaring.MaxUint32
+const maxUint32 = roaring.MaxUint32
+
+func minOfInt64(a, b int64) int64 {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func minOfInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxOfInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func maxOfUint32(a, b uint32) uint32 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func minOfUint32(a, b uint32) uint32 {
+	if a < b {
+		return a
+	}
+	return b
+}

vendor/github.com/RoaringBitmap/roaring/v2/roaringarray.go

@@ -0,0 +1,943 @@
+package roaring
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/RoaringBitmap/roaring/v2/internal"
+)
+
+type container interface {
+	// addOffset returns the (low, high) parts of the shifted container.
+	// Whenever one of them would be empty, nil will be returned instead to
+	// avoid unnecessary allocations.
+	addOffset(uint16) (container, container)
+
+	clone() container
+	and(container) container
+	andCardinality(container) int
+	iand(container) container // i stands for inplace
+	andNot(container) container
+	iandNot(container) container // i stands for inplace
+	isEmpty() bool
+	getCardinality() int
+	// rank returns the number of integers that are
+	// smaller or equal to x. rank(infinity) would be getCardinality().
+	rank(uint16) int
+
+	// getCardinalityInRange returns the number of integers that are
+	// within the half-open range [start, end). It is equivalent to
+	// rank(end-1) - rank(start-1) but may be faster.
+	getCardinalityInRange(start, end uint) int
+
+	iadd(x uint16) bool                   // inplace, returns true if x was new.
+	iaddReturnMinimized(uint16) container // may change return type to minimize storage.
+
+	iaddRange(start, endx int) container // i stands for inplace, range is [firstOfRange,endx)
+
+	iremove(x uint16) bool                   // inplace, returns true if x was present.
+	iremoveReturnMinimized(uint16) container // may change return type to minimize storage.
+
+	not(start, final int) container        // range is [firstOfRange,lastOfRange)
+	inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx)
+	xor(r container) container
+	ixor(r container) container // i stands for inplace
+	getShortIterator() shortPeekable
+	getUnsetIterator() shortPeekable
+	iterate(cb func(x uint16) bool) bool
+	getReverseIterator() shortIterable
+	getManyIterator() manyIterable
+	contains(i uint16) bool
+	maximum() uint16
+	minimum() uint16
+
+	// equals is now logical equals; it does not require the
+	// same underlying container types, but compares across
+	// any of the implementations.
+	equals(r container) bool
+
+	fillLeastSignificant16bits(array []uint32, i int, mask uint32) int
+	or(r container) container
+	orCardinality(r container) int
+	isFull() bool
+	ior(r container) container   // i stands for inplace
+	intersects(r container) bool // whether the two containers intersect
+	lazyOR(r container) container
+	lazyIOR(r container) container
+	getSizeInBytes() int
+	iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange)
+	selectInt(x uint16) int                  // selectInt returns the xth integer in the container
+	serializedSizeInBytes() int
+	writeTo(io.Writer) (int, error)
+
+	numberOfRuns() int
+	toEfficientContainer() container
+	String() string
+	containerType() contype
+
+	safeMinimum() (uint16, error)
+	safeMaximum() (uint16, error)
+	nextValue(x uint16) int
+	previousValue(x uint16) int
+	nextAbsentValue(x uint16) int
+	previousAbsentValue(x uint16) int
+	validate() error
+}
+
+type contype uint8
+
+const (
+	bitmapContype contype = iota
+	arrayContype
+	run16Contype
+	run32Contype
+)
+
+var (
+	ErrKeySortOrder          = errors.New("keys were out of order")
+	ErrCardinalityConstraint = errors.New("size of arrays was not coherent")
+)
+
+// careful: range is [firstOfRange,lastOfRange]
+func rangeOfOnes(start, last int) container {
+	if start > MaxUint16 {
+		panic("rangeOfOnes called with start > MaxUint16")
+	}
+	if last > MaxUint16 {
+		panic("rangeOfOnes called with last > MaxUint16")
+	}
+	if start < 0 {
+		panic("rangeOfOnes called with start < 0")
+	}
+	if last < 0 {
+		panic("rangeOfOnes called with last < 0")
+	}
+	return newRunContainer16Range(uint16(start), uint16(last)).toEfficientContainer()
+}
+
+type roaringArray struct {
+	keys            []uint16
+	containers      []container `msg:"-"` // don't try to serialize directly.
+	needCopyOnWrite []bool
+	copyOnWrite     bool
+}
+
+func newRoaringArray() *roaringArray {
+	return &roaringArray{}
+}
+
+// runOptimize compresses the element containers to minimize space consumed.
+// Q: how does this interact with copyOnWrite and needCopyOnWrite?
+// A: since we aren't changing the logical content, just the representation,
+//
+//	we don't bother to check the needCopyOnWrite bits. We replace
+//	(possibly all) elements of ra.containers in-place with space
+//	optimized versions.
+func (ra *roaringArray) runOptimize() {
+	for i := range ra.containers {
+		ra.containers[i] = ra.containers[i].toEfficientContainer()
+	}
+}
+
+func (ra *roaringArray) appendContainer(key uint16, value container, mustCopyOnWrite bool) {
+	ra.keys = append(ra.keys, key)
+	ra.containers = append(ra.containers, value)
+	ra.needCopyOnWrite = append(ra.needCopyOnWrite, mustCopyOnWrite)
+}
+
+func (ra *roaringArray) appendWithoutCopy(sa roaringArray, startingindex int) {
+	mustCopyOnWrite := sa.needCopyOnWrite[startingindex]
+	ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], mustCopyOnWrite)
+}
+
+func (ra *roaringArray) appendCopy(sa roaringArray, startingindex int) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := (ra.copyOnWrite && sa.copyOnWrite) || sa.needsCopyOnWrite(startingindex)
+	if !copyonwrite {
+		// since there is no copy-on-write, we need to clone the container (this is important)
+		ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex].clone(), copyonwrite)
+	} else {
+		ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], copyonwrite)
+		if !sa.needsCopyOnWrite(startingindex) {
+			sa.setNeedsCopyOnWrite(startingindex)
+		}
+	}
+}
+
+func (ra *roaringArray) appendWithoutCopyMany(sa roaringArray, startingindex, end int) {
+	for i := startingindex; i < end; i++ {
+		ra.appendWithoutCopy(sa, i)
+	}
+}
+
+func (ra *roaringArray) appendCopyMany(sa roaringArray, startingindex, end int) {
+	for i := startingindex; i < end; i++ {
+		ra.appendCopy(sa, i)
+	}
+}
+
+func (ra *roaringArray) appendCopiesUntil(sa roaringArray, stoppingKey uint16) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := ra.copyOnWrite && sa.copyOnWrite
+
+	for i := 0; i < sa.size(); i++ {
+		if sa.keys[i] >= stoppingKey {
+			break
+		}
+		thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i)
+		if thiscopyonewrite {
+			ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite)
+			if !sa.needsCopyOnWrite(i) {
+				sa.setNeedsCopyOnWrite(i)
+			}
+
+		} else {
+			// since there is no copy-on-write, we need to clone the container (this is important)
+			ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite)
+		}
+	}
+}
+
+func (ra *roaringArray) appendCopiesAfter(sa roaringArray, beforeStart uint16) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := ra.copyOnWrite && sa.copyOnWrite
+
+	startLocation := sa.getIndex(beforeStart)
+	if startLocation >= 0 {
+		startLocation++
+	} else {
+		startLocation = -startLocation - 1
+	}
+
+	for i := startLocation; i < sa.size(); i++ {
+		thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i)
+		if thiscopyonewrite {
+			ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite)
+			if !sa.needsCopyOnWrite(i) {
+				sa.setNeedsCopyOnWrite(i)
+			}
+		} else {
+			// since there is no copy-on-write, we need to clone the container (this is important)
+			ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite)
+		}
+	}
+}
+
+func (ra *roaringArray) removeIndexRange(begin, end int) {
+	if end <= begin {
+		return
+	}
+
+	r := end - begin
+
+	copy(ra.keys[begin:], ra.keys[end:])
+	copy(ra.containers[begin:], ra.containers[end:])
+	copy(ra.needCopyOnWrite[begin:], ra.needCopyOnWrite[end:])
+
+	ra.resize(len(ra.keys) - r)
+}
+
+func (ra *roaringArray) resize(newsize int) {
+	for k := newsize; k < len(ra.containers); k++ {
+		ra.containers[k] = nil
+	}
+
+	ra.keys = ra.keys[:newsize]
+	ra.containers = ra.containers[:newsize]
+	ra.needCopyOnWrite = ra.needCopyOnWrite[:newsize]
+}
+
+func (ra *roaringArray) clear() {
+	ra.resize(0)
+	ra.copyOnWrite = false
+}
+
+func (ra *roaringArray) clone() *roaringArray {
+	sa := roaringArray{}
+	sa.copyOnWrite = ra.copyOnWrite
+
+	// this is where copyOnWrite is used.
+	if ra.copyOnWrite {
+		sa.keys = make([]uint16, len(ra.keys))
+		copy(sa.keys, ra.keys)
+		sa.containers = make([]container, len(ra.containers))
+		copy(sa.containers, ra.containers)
+		sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite))
+
+		ra.markAllAsNeedingCopyOnWrite()
+		sa.markAllAsNeedingCopyOnWrite()
+
+		// sa.needCopyOnWrite is shared
+	} else {
+		// make a full copy
+
+		sa.keys = make([]uint16, len(ra.keys))
+		copy(sa.keys, ra.keys)
+
+		sa.containers = make([]container, len(ra.containers))
+		for i := range sa.containers {
+			sa.containers[i] = ra.containers[i].clone()
+		}
+
+		sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite))
+	}
+	return &sa
+}
+
+// clone all containers which have needCopyOnWrite set to true
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to.
+func (ra *roaringArray) cloneCopyOnWriteContainers() {
+	for i, needCopyOnWrite := range ra.needCopyOnWrite {
+		if needCopyOnWrite {
+			ra.containers[i] = ra.containers[i].clone()
+			ra.needCopyOnWrite[i] = false
+		}
+	}
+}
+
+// unused function:
+//func (ra *roaringArray) containsKey(x uint16) bool {
+//	return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
+//}
+
+// getContainer returns the container with key `x`
+// if no such container exists `nil` is returned
+func (ra *roaringArray) getContainer(x uint16) container {
+	i := ra.binarySearch(0, int64(len(ra.keys)), x)
+	if i < 0 {
+		return nil
+	}
+	return ra.containers[i]
+}
+
+func (ra *roaringArray) getContainerAtIndex(i int) container {
+	return ra.containers[i]
+}
+
+func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) container {
+	c := ra.getContainerAtIndex(i)
+	switch t := c.(type) {
+	case *arrayContainer:
+		c = t.toBitmapContainer()
+	case *runContainer16:
+		if !t.isFull() {
+			c = t.toBitmapContainer()
+		}
+	case *bitmapContainer:
+		if needsWriteable && ra.needCopyOnWrite[i] {
+			c = ra.containers[i].clone()
+		}
+	}
+	return c
+}
+
+// getUnionedWritableContainer switches behavior for in-place Or
+// depending on whether the container requires a copy on write.
+// If it does using the non-inplace or() method leads to fewer allocations.
+func (ra *roaringArray) getUnionedWritableContainer(pos int, other container) container {
+	if ra.needCopyOnWrite[pos] {
+		return ra.getContainerAtIndex(pos).or(other)
+	}
+	return ra.getContainerAtIndex(pos).ior(other)
+}
+
+func (ra *roaringArray) getWritableContainerAtIndex(i int) container {
+	if ra.needCopyOnWrite[i] {
+		ra.containers[i] = ra.containers[i].clone()
+		ra.needCopyOnWrite[i] = false
+	}
+	return ra.containers[i]
+}
+
+// getIndex returns the index of the container with key `x`
+// if no such container exists a negative value is returned
+func (ra *roaringArray) getIndex(x uint16) int {
+	// Todo : test
+	// before the binary search, we optimize for frequent cases
+	size := len(ra.keys)
+	if (size == 0) || (ra.keys[size-1] == x) {
+		return size - 1
+	}
+	return ra.binarySearch(0, int64(size), x)
+}
+
+func (ra *roaringArray) getKeyAtIndex(i int) uint16 {
+	return ra.keys[i]
+}
+
+func (ra *roaringArray) insertNewKeyValueAt(i int, key uint16, value container) {
+	ra.keys = append(ra.keys, 0)
+	ra.containers = append(ra.containers, nil)
+
+	copy(ra.keys[i+1:], ra.keys[i:])
+	copy(ra.containers[i+1:], ra.containers[i:])
+
+	ra.keys[i] = key
+	ra.containers[i] = value
+
+	ra.needCopyOnWrite = append(ra.needCopyOnWrite, false)
+	copy(ra.needCopyOnWrite[i+1:], ra.needCopyOnWrite[i:])
+	ra.needCopyOnWrite[i] = false
+}
+
+// copyOrSourceContainerAt returns the container (and its copy-on-write flag) to
+// store for a source-only key. Keys beyond the receiver's last key are the
+// trailing suffix: they may be shared under copy-on-write, matching appendCopy.
+// Interior source-only keys are always cloned so that later receiver mutations
+// cannot leak into the source.
+func (ra *roaringArray) copyOrSourceContainerAt(other *roaringArray, index int, receiverLastKey uint16) (container, bool) {
+	if other.keys[index] > receiverLastKey {
+		copyOnWrite := (ra.copyOnWrite && other.copyOnWrite) || other.needsCopyOnWrite(index)
+		if copyOnWrite {
+			if !other.needsCopyOnWrite(index) {
+				other.setNeedsCopyOnWrite(index)
+			}
+			return other.containers[index], true
+		}
+	}
+	return other.containers[index].clone(), false
+}
+
+// mergeBulk finishes an in-place union (xor == false) or symmetric difference
+// (xor == true) once the receiver's structure must change and continuing in
+// place would shift the aligned suffix once per changed key -- quadratic when
+// many keys are interleaved. It merges the two suffixes forward into fresh
+// slices in a single pass instead.
+//
+// The change that triggers it is a source-only key that must be inserted, or
+// (xor only) an aligned pair that cancelled to an empty container. dst is the
+// write cursor: the prefix [0, dst) is already final and copied over unchanged.
+// left/right are the receiver/source scan positions; the caller advances them
+// past an already-consumed aligned-empty pair. For a union a source-only key is
+// always inserted, so dst == left; the xor caller may pass dst < left to drop
+// the emptied container.
+//
+// Like every other roaringArray operation it assumes both arrays already hold
+// their keys in sorted order; that invariant is enforced at the load boundary
+// (Validate), not re-checked here.
+func (ra *roaringArray) mergeBulk(other *roaringArray, dst, left, right int, xor bool) {
+	length1 := ra.size()
+	length2 := other.size()
+	receiverLastKey := ra.keys[length1-1]
+
+	// First pass over the keys only (cheap, no container work): count the
+	// distinct keys of the two suffixes. That is the exact result size for a
+	// union and, for a xor, a tight upper bound (aligned pairs may cancel). So
+	// the appends below never reallocate, without grossly over-allocating when
+	// many aligned containers cancel to empty.
+	distinct := 0
+	l, r := left, right
+	for l < length1 && r < length2 {
+		if ra.keys[l] < other.keys[r] {
+			l++
+		} else if ra.keys[l] > other.keys[r] {
+			r++
+		} else {
+			l++
+			r++
+		}
+		distinct++
+	}
+	distinct += (length1 - l) + (length2 - r)
+	total := dst + distinct
+	keys := make([]uint16, dst, total)
+	containers := make([]container, dst, total)
+	needCopyOnWrite := make([]bool, dst, total)
+	copy(keys, ra.keys[:dst])
+	copy(containers, ra.containers[:dst])
+	copy(needCopyOnWrite, ra.needCopyOnWrite[:dst])
+
+	for left < length1 && right < length2 {
+		s1 := ra.keys[left]
+		s2 := other.keys[right]
+		if s1 < s2 {
+			keys = append(keys, s1)
+			containers = append(containers, ra.containers[left])
+			needCopyOnWrite = append(needCopyOnWrite, ra.needCopyOnWrite[left])
+			left++
+		} else if s1 > s2 {
+			c, cow := ra.copyOrSourceContainerAt(other, right, receiverLastKey)
+			keys = append(keys, s2)
+			containers = append(containers, c)
+			needCopyOnWrite = append(needCopyOnWrite, cow)
+			right++
+		} else {
+			// Union of two non-empty containers is never empty, so the
+			// isEmpty check only ever drops a container for xor.
+			var c container
+			if xor {
+				c = ra.getWritableContainerAtIndex(left).ixor(other.containers[right])
+			} else {
+				c = ra.getUnionedWritableContainer(left, other.containers[right])
+			}
+			if !c.isEmpty() {
+				keys = append(keys, s1)
+				containers = append(containers, c)
+				needCopyOnWrite = append(needCopyOnWrite, false)
+			}
+			left++
+			right++
+		}
+	}
+	for ; left < length1; left++ {
+		keys = append(keys, ra.keys[left])
+		containers = append(containers, ra.containers[left])
+		needCopyOnWrite = append(needCopyOnWrite, ra.needCopyOnWrite[left])
+	}
+	for ; right < length2; right++ {
+		c, cow := ra.copyOrSourceContainerAt(other, right, receiverLastKey)
+		keys = append(keys, other.keys[right])
+		containers = append(containers, c)
+		needCopyOnWrite = append(needCopyOnWrite, cow)
+	}
+
+	ra.keys = keys
+	ra.containers = containers
+	ra.needCopyOnWrite = needCopyOnWrite
+}
+
+func (ra *roaringArray) remove(key uint16) bool {
+	i := ra.binarySearch(0, int64(len(ra.keys)), key)
+	if i >= 0 { // if a new key
+		ra.removeAtIndex(i)
+		return true
+	}
+	return false
+}
+
+func (ra *roaringArray) removeAtIndex(i int) {
+	copy(ra.keys[i:], ra.keys[i+1:])
+	copy(ra.containers[i:], ra.containers[i+1:])
+
+	copy(ra.needCopyOnWrite[i:], ra.needCopyOnWrite[i+1:])
+
+	ra.resize(len(ra.keys) - 1)
+}
+
+func (ra *roaringArray) setContainerAtIndex(i int, c container) {
+	ra.containers[i] = c
+}
+
+func (ra *roaringArray) replaceKeyAndContainerAtIndex(i int, key uint16, c container, mustCopyOnWrite bool) {
+	ra.keys[i] = key
+	ra.containers[i] = c
+	ra.needCopyOnWrite[i] = mustCopyOnWrite
+}
+
+func (ra *roaringArray) size() int {
+	return len(ra.keys)
+}
+
+// binarySearch returns the index of the key.
+// negative value returned if not found
+func (ra *roaringArray) binarySearch(begin, end int64, ikey uint16) int {
+	// TODO: add unit tests
+	low := begin
+	high := end - 1
+	for low+16 <= high {
+		middleIndex := low + (high-low)/2 // avoid overflow
+		middleValue := ra.keys[middleIndex]
+
+		if middleValue < ikey {
+			low = middleIndex + 1
+		} else if middleValue > ikey {
+			high = middleIndex - 1
+		} else {
+			return int(middleIndex)
+		}
+	}
+	for ; low <= high; low++ {
+		val := ra.keys[low]
+		if val >= ikey {
+			if val == ikey {
+				return int(low)
+			}
+			break
+		}
+	}
+	return -int(low + 1)
+}
+
+func (ra *roaringArray) equals(o interface{}) bool {
+	srb, ok := o.(roaringArray)
+	if ok {
+
+		if srb.size() != ra.size() {
+			return false
+		}
+		for i, k := range ra.keys {
+			if k != srb.keys[i] {
+				return false
+			}
+		}
+
+		for i, c := range ra.containers {
+			if !c.equals(srb.containers[i]) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func (ra *roaringArray) headerSize() uint64 {
+	size := uint64(len(ra.keys))
+	if ra.hasRunCompression() {
+		if size < noOffsetThreshold { // for small bitmaps, we omit the offsets
+			return 4 + (size+7)/8 + 4*size
+		}
+		return 4 + (size+7)/8 + 8*size // - 4 because we pack the size with the cookie
+	}
+	return 4 + 4 + 8*size
+}
+
+// should be dirt cheap
+func (ra *roaringArray) serializedSizeInBytes() uint64 {
+	answer := ra.headerSize()
+	for _, c := range ra.containers {
+		answer += uint64(c.serializedSizeInBytes())
+	}
+	return answer
+}
+
+// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
+	hasRun := ra.hasRunCompression()
+	isRunSizeInBytes := 0
+	cookieSize := 8
+	if hasRun {
+		cookieSize = 4
+		isRunSizeInBytes = (len(ra.keys) + 7) / 8
+	}
+	descriptiveHeaderSize := 4 * len(ra.keys)
+	preambleSize := cookieSize + isRunSizeInBytes + descriptiveHeaderSize
+
+	buf := make([]byte, preambleSize+4*len(ra.keys))
+
+	nw := 0
+
+	if hasRun {
+		binary.LittleEndian.PutUint16(buf[0:], uint16(serialCookie))
+		nw += 2
+		binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1))
+		nw += 2
+		// compute isRun bitmap without temporary allocation
+		runbitmapslice := buf[nw : nw+isRunSizeInBytes]
+		for i, c := range ra.containers {
+			switch c.(type) {
+			case *runContainer16:
+				runbitmapslice[i/8] |= 1 << (uint(i) % 8)
+			}
+		}
+		nw += isRunSizeInBytes
+	} else {
+		binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer))
+		nw += 4
+		binary.LittleEndian.PutUint32(buf[4:], uint32(len(ra.keys)))
+		nw += 4
+	}
+
+	// descriptive header
+	for i, key := range ra.keys {
+		binary.LittleEndian.PutUint16(buf[nw:], key)
+		nw += 2
+		c := ra.containers[i]
+		binary.LittleEndian.PutUint16(buf[nw:], uint16(c.getCardinality()-1))
+		nw += 2
+	}
+
+	startOffset := int64(preambleSize + 4*len(ra.keys))
+	if !hasRun || (len(ra.keys) >= noOffsetThreshold) {
+		// offset header
+		for _, c := range ra.containers {
+			binary.LittleEndian.PutUint32(buf[nw:], uint32(startOffset))
+			nw += 4
+			switch rc := c.(type) {
+			case *runContainer16:
+				startOffset += 2 + int64(len(rc.iv))*4
+			default:
+				startOffset += int64(getSizeInBytesFromCardinality(c.getCardinality()))
+			}
+		}
+	}
+
+	written, err := w.Write(buf[:nw])
+	if err != nil {
+		return n, err
+	}
+	n += int64(written)
+
+	for _, c := range ra.containers {
+		written, err := c.writeTo(w)
+		if err != nil {
+			return n, err
+		}
+		n += int64(written)
+	}
+	return n, nil
+}
+
+// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+func (ra *roaringArray) toBytes() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := ra.writeTo(&buf)
+	return buf.Bytes(), err
+}
+
+// Reads a serialized roaringArray from a byte slice.
+func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte) (int64, error) {
+	var cookie uint32
+	var err error
+	if len(cookieHeader) > 0 && len(cookieHeader) != 4 {
+		return int64(len(cookieHeader)), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: incorrect size of cookie header")
+	}
+	if len(cookieHeader) == 4 {
+		cookie = binary.LittleEndian.Uint32(cookieHeader)
+	} else {
+		cookie, err = stream.ReadUInt32()
+		if err != nil {
+			return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
+		}
+	}
+	// If NextReturnsSafeSlice is false, then willNeedCopyOnWrite should be true
+	willNeedCopyOnWrite := !stream.NextReturnsSafeSlice()
+
+	var size uint32
+	var isRunBitmap []byte
+
+	if cookie&0x0000FFFF == serialCookie {
+		size = cookie>>16 + 1
+		// create is-run-container bitmap
+		isRunBitmapSize := (int(size) + 7) / 8
+		isRunBitmap, err = stream.Next(isRunBitmapSize)
+		if err != nil {
+			return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err)
+		}
+	} else if cookie == serialCookieNoRunContainer {
+		size, err = stream.ReadUInt32()
+		if err != nil {
+			return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err)
+		}
+	} else {
+		return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+	}
+
+	if size > (1 << 16) {
+		return stream.GetReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
+	}
+
+	// descriptive header
+	buf, err := stream.Next(2 * 2 * int(size))
+	if err != nil {
+		return stream.GetReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err)
+	}
+
+	keycard := byteSliceAsUint16Slice(buf)
+
+	if isRunBitmap == nil || size >= noOffsetThreshold {
+		if err := stream.SkipBytes(int(size) * 4); err != nil {
+			return stream.GetReadBytes(), fmt.Errorf("failed to skip bytes: %s", err)
+		}
+	}
+
+	// Allocate slices upfront as number of containers is known
+	if cap(ra.containers) >= int(size) {
+		ra.containers = ra.containers[:size]
+	} else {
+		ra.containers = make([]container, size)
+	}
+
+	if cap(ra.keys) >= int(size) {
+		ra.keys = ra.keys[:size]
+	} else {
+		ra.keys = make([]uint16, size)
+	}
+
+	if cap(ra.needCopyOnWrite) >= int(size) {
+		ra.needCopyOnWrite = ra.needCopyOnWrite[:size]
+	} else {
+		ra.needCopyOnWrite = make([]bool, size)
+	}
+
+	for i := uint32(0); i < size; i++ {
+		key := keycard[2*i]
+		card := int(keycard[2*i+1]) + 1
+		ra.keys[i] = key
+		ra.needCopyOnWrite[i] = willNeedCopyOnWrite
+
+		if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
+			// run container
+			nr, err := stream.ReadUInt16()
+			if err != nil {
+				return 0, fmt.Errorf("failed to read runtime container size: %s", err)
+			}
+
+			buf, err := stream.Next(int(nr) * 4)
+			if err != nil {
+				return stream.GetReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err)
+			}
+
+			nb := runContainer16{
+				iv: byteSliceAsInterval16Slice(buf),
+			}
+
+			ra.containers[i] = &nb
+		} else if card > arrayDefaultMaxSize {
+			// bitmap container
+			buf, err := stream.Next(arrayDefaultMaxSize * 2)
+			if err != nil {
+				return stream.GetReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err)
+			}
+
+			nb := bitmapContainer{
+				cardinality: card,
+				bitmap:      byteSliceAsUint64Slice(buf),
+			}
+
+			ra.containers[i] = &nb
+		} else {
+			// array container
+			buf, err := stream.Next(card * 2)
+			if err != nil {
+				return stream.GetReadBytes(), fmt.Errorf("failed to read array container: %s", err)
+			}
+
+			nb := arrayContainer{
+				byteSliceAsUint16Slice(buf),
+			}
+
+			ra.containers[i] = &nb
+		}
+	}
+
+	return stream.GetReadBytes(), nil
+}
+
+func (ra *roaringArray) hasRunCompression() bool {
+	for _, c := range ra.containers {
+		switch c.(type) {
+		case *runContainer16:
+			return true
+		}
+	}
+	return false
+}
+
+/**
+ * Find the smallest integer index larger than pos such that array[index].key&gt;=min. If none can
+ * be found, return size. Based on code by O. Kaser.
+ *
+ * @param min minimal value
+ * @param pos index to exceed
+ * @return the smallest index greater than pos such that array[index].key is at least as large as
+ *         min, or size if it is not possible.
+ */
+func (ra *roaringArray) advanceUntil(min uint16, pos int) int {
+	lower := pos + 1
+
+	if lower >= len(ra.keys) || ra.keys[lower] >= min {
+		return lower
+	}
+
+	spansize := 1
+
+	for lower+spansize < len(ra.keys) && ra.keys[lower+spansize] < min {
+		spansize *= 2
+	}
+	var upper int
+	if lower+spansize < len(ra.keys) {
+		upper = lower + spansize
+	} else {
+		upper = len(ra.keys) - 1
+	}
+
+	if ra.keys[upper] == min {
+		return upper
+	}
+
+	if ra.keys[upper] < min {
+		// means
+		// array
+		// has no
+		// item
+		// >= min
+		// pos = array.length;
+		return len(ra.keys)
+	}
+
+	// we know that the next-smallest span was too small
+	lower += (spansize >> 1)
+
+	mid := 0
+	for lower+1 != upper {
+		mid = (lower + upper) >> 1
+		if ra.keys[mid] == min {
+			return mid
+		} else if ra.keys[mid] < min {
+			lower = mid
+		} else {
+			upper = mid
+		}
+	}
+	return upper
+}
+
+func (ra *roaringArray) markAllAsNeedingCopyOnWrite() {
+	for i := range ra.needCopyOnWrite {
+		ra.needCopyOnWrite[i] = true
+	}
+}
+
+func (ra *roaringArray) needsCopyOnWrite(i int) bool {
+	return ra.needCopyOnWrite[i]
+}
+
+func (ra *roaringArray) setNeedsCopyOnWrite(i int) {
+	ra.needCopyOnWrite[i] = true
+}
+
+func (ra *roaringArray) checkKeysSorted() bool {
+	if len(ra.keys) == 0 || len(ra.keys) == 1 {
+		return true
+	}
+	previous := ra.keys[0]
+	for nextIdx := 1; nextIdx < len(ra.keys); nextIdx++ {
+		next := ra.keys[nextIdx]
+		if previous >= next {
+			return false
+		}
+		previous = next
+
+	}
+	return true
+}
+
+// validate checks the referential integrity
+// ensures len(keys) == len(containers), recurses and checks each container type
+func (ra *roaringArray) validate() error {
+	if !ra.checkKeysSorted() {
+		return ErrKeySortOrder
+	}
+
+	if len(ra.keys) != len(ra.containers) {
+		return ErrCardinalityConstraint
+	}
+
+	if len(ra.keys) != len(ra.needCopyOnWrite) {
+		return ErrCardinalityConstraint
+	}
+
+	for _, container := range ra.containers {
+		err := container.validate()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

vendor/github.com/RoaringBitmap/roaring/v2/serialization.go

@@ -0,0 +1,18 @@
+package roaring
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// writeTo for runContainer16 follows this
+// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
+	buf := make([]byte, 2+4*len(b.iv))
+	binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv)))
+	for i, v := range b.iv {
+		binary.LittleEndian.PutUint16(buf[2+i*4:], v.start)
+		binary.LittleEndian.PutUint16(buf[2+2+i*4:], v.length)
+	}
+	return stream.Write(buf)
+}

vendor/github.com/RoaringBitmap/roaring/v2/serialization_generic.go

@@ -0,0 +1,145 @@
+//go:build (!amd64 && !386 && !arm && !arm64 && !ppc64le && !mipsle && !mips64le && !mips64p32le && !wasm) || appengine
+// +build !amd64,!386,!arm,!arm64,!ppc64le,!mipsle,!mips64le,!mips64p32le,!wasm appengine
+
+package roaring
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+func (b *arrayContainer) writeTo(stream io.Writer) (int, error) {
+	buf := make([]byte, 2*len(b.content))
+	for i, v := range b.content {
+		base := i * 2
+		buf[base] = byte(v)
+		buf[base+1] = byte(v >> 8)
+	}
+	return stream.Write(buf)
+}
+
+func (b *arrayContainer) readFrom(stream io.Reader) (int, error) {
+	err := binary.Read(stream, binary.LittleEndian, b.content)
+	if err != nil {
+		return 0, err
+	}
+	return 2 * len(b.content), nil
+}
+
+func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+	if b.cardinality <= arrayDefaultMaxSize {
+		return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+	}
+
+	// Write set
+	buf := make([]byte, 8*len(b.bitmap))
+	for i, v := range b.bitmap {
+		base := i * 8
+		buf[base] = byte(v)
+		buf[base+1] = byte(v >> 8)
+		buf[base+2] = byte(v >> 16)
+		buf[base+3] = byte(v >> 24)
+		buf[base+4] = byte(v >> 32)
+		buf[base+5] = byte(v >> 40)
+		buf[base+6] = byte(v >> 48)
+		buf[base+7] = byte(v >> 56)
+	}
+	return stream.Write(buf)
+}
+
+func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) {
+	err := binary.Read(stream, binary.LittleEndian, b.bitmap)
+	if err != nil {
+		return 0, err
+	}
+	b.computeCardinality()
+	return 8 * len(b.bitmap), nil
+}
+
+func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
+	by := make([]byte, len(bc.bitmap)*8)
+	for i := range bc.bitmap {
+		binary.LittleEndian.PutUint64(by[i*8:], bc.bitmap[i])
+	}
+	return by
+}
+
+func uint64SliceAsByteSlice(slice []uint64) []byte {
+	by := make([]byte, len(slice)*8)
+
+	for i, v := range slice {
+		binary.LittleEndian.PutUint64(by[i*8:], v)
+	}
+
+	return by
+}
+
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+	by := make([]byte, len(slice)*2)
+
+	for i, v := range slice {
+		binary.LittleEndian.PutUint16(by[i*2:], v)
+	}
+
+	return by
+}
+
+func interval16SliceAsByteSlice(slice []interval16) []byte {
+	by := make([]byte, len(slice)*4)
+
+	for i, v := range slice {
+		binary.LittleEndian.PutUint16(by[i*2:], v.start)
+		binary.LittleEndian.PutUint16(by[i*2+2:], v.length)
+	}
+
+	return by
+}
+
+func byteSliceAsUint16Slice(slice []byte) []uint16 {
+	if len(slice)%2 != 0 {
+		panic("Slice size should be divisible by 2")
+	}
+
+	b := make([]uint16, len(slice)/2)
+
+	for i := range b {
+		b[i] = binary.LittleEndian.Uint16(slice[2*i:])
+	}
+
+	return b
+}
+
+func byteSliceAsUint64Slice(slice []byte) []uint64 {
+	if len(slice)%8 != 0 {
+		panic("Slice size should be divisible by 8")
+	}
+
+	b := make([]uint64, len(slice)/8)
+
+	for i := range b {
+		b[i] = binary.LittleEndian.Uint64(slice[8*i:])
+	}
+
+	return b
+}
+
+// Converts a byte slice to a interval16 slice.
+// The function assumes that the slice byte buffer is run container data
+// encoded according to Roaring Format Spec
+func byteSliceAsInterval16Slice(byteSlice []byte) []interval16 {
+	if len(byteSlice)%4 != 0 {
+		panic("Slice size should be divisible by 4")
+	}
+
+	intervalSlice := make([]interval16, len(byteSlice)/4)
+
+	for i := range intervalSlice {
+		intervalSlice[i] = interval16{
+			start:  binary.LittleEndian.Uint16(byteSlice[i*4:]),
+			length: binary.LittleEndian.Uint16(byteSlice[i*4+2:]),
+		}
+	}
+
+	return intervalSlice
+}

vendor/github.com/RoaringBitmap/roaring/v2/serialization_littleendian.go

@@ -0,0 +1,564 @@
+//go:build (386 && !appengine) || (amd64 && !appengine) || (arm && !appengine) || (arm64 && !appengine) || (ppc64le && !appengine) || (mipsle && !appengine) || (mips64le && !appengine) || (mips64p32le && !appengine) || (wasm && !appengine)
+// +build 386,!appengine amd64,!appengine arm,!appengine arm64,!appengine ppc64le,!appengine mipsle,!appengine mips64le,!appengine mips64p32le,!appengine wasm,!appengine
+
+package roaring
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"unsafe"
+)
+
+func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) {
+	buf := uint16SliceAsByteSlice(ac.content)
+	return stream.Write(buf)
+}
+
+func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+	if bc.cardinality <= arrayDefaultMaxSize {
+		return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+	}
+	buf := uint64SliceAsByteSlice(bc.bitmap)
+	return stream.Write(buf)
+}
+
+func uint64SliceAsByteSlice(slice []uint64) []byte {
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	const size = unsafe.Sizeof(uint64(0))
+	return unsafe.Slice(((*byte)(unsafe.Pointer(ptr))), int(size)*len(slice))
+}
+
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	const size = unsafe.Sizeof(uint16(0))
+	return unsafe.Slice(((*byte)(unsafe.Pointer(ptr))), int(size)*len(slice))
+}
+
+func interval16SliceAsByteSlice(slice []interval16) []byte {
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	const size = unsafe.Sizeof(interval16{})
+	return unsafe.Slice(((*byte)(unsafe.Pointer(ptr))), int(size)*len(slice))
+}
+
+func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
+	return uint64SliceAsByteSlice(bc.bitmap)
+}
+
+// Deserialization code follows
+
+// //
+// These methods (byteSliceAsUint16Slice,...) do not make copies,
+// they are pointer-based (unsafe). The caller is responsible to
+// ensure that the input slice does not get garbage collected, deleted
+// or modified while you hold the returned slince.
+// //
+func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder
+	const sz = int(unsafe.Sizeof(uint16(0)))
+	if len(slice)%sz != 0 {
+		panic(fmt.Sprintf("Slice size should be divisible by %d", sz))
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*uint16)(unsafe.Pointer(ptr)), len(slice)/sz)
+}
+
+func byteSliceAsUint64Slice(slice []byte) (result []uint64) {
+	const sz = int(unsafe.Sizeof(uint64(0)))
+	if len(slice)%sz != 0 {
+		panic(fmt.Sprintf("Slice size should be divisible by %d", sz))
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*uint64)(unsafe.Pointer(ptr)), len(slice)/sz)
+}
+
+func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
+	const sz = int(unsafe.Sizeof(interval16{}))
+	if len(slice)%sz != 0 {
+		panic(fmt.Sprintf("Slice size should be divisible by %d", sz))
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*interval16)(unsafe.Pointer(ptr)), len(slice)/sz)
+}
+
+func byteSliceAsContainerSlice(slice []byte) (result []container) {
+	var c container
+	containerSize := int(unsafe.Sizeof(c))
+
+	if len(slice)%containerSize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(container)")
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*container)(unsafe.Pointer(ptr)), len(slice)/containerSize)
+}
+
+func byteSliceAsBitsetSlice(slice []byte) (result []bitmapContainer) {
+	const bitsetSize = int(unsafe.Sizeof(bitmapContainer{}))
+	if len(slice)%bitsetSize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(bitmapContainer)")
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*bitmapContainer)(unsafe.Pointer(ptr)), len(slice)/bitsetSize)
+}
+
+func byteSliceAsArraySlice(slice []byte) (result []arrayContainer) {
+	const arraySize = int(unsafe.Sizeof(arrayContainer{}))
+	if len(slice)%arraySize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(arrayContainer)")
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*arrayContainer)(unsafe.Pointer(ptr)), len(slice)/arraySize)
+}
+
+func byteSliceAsRun16Slice(slice []byte) (result []runContainer16) {
+	const run16Size = int(unsafe.Sizeof(runContainer16{}))
+	if len(slice)%run16Size != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(runContainer16)")
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*runContainer16)(unsafe.Pointer(ptr)), len(slice)/run16Size)
+}
+
+func byteSliceAsBoolSlice(slice []byte) (result []bool) {
+	const boolSize = int(unsafe.Sizeof(true))
+	if len(slice)%boolSize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(bool)")
+	}
+	ptr := unsafe.SliceData(slice)
+	if ptr == nil {
+		return nil
+	}
+	return unsafe.Slice((*bool)(unsafe.Pointer(ptr)), len(slice)/boolSize)
+}
+
+// FrozenView creates a static view of a serialized bitmap stored in buf.
+// It uses CRoaring's frozen bitmap format.
+//
+// The format specification is available here:
+// https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/src/roaring.c#L2756-L2783
+//
+// The provided byte array (buf) is expected to be a constant.
+// The function makes the best effort attempt not to copy data.
+// Only little endian is supported. The function will err if it detects a big
+// endian serialized file.
+// You should take care not to modify buff as it will likely result in
+// unexpected program behavior.
+// If said buffer comes from a memory map, it's advisable to give it read
+// only permissions, either at creation or by calling Mprotect from the
+// golang.org/x/sys/unix package.
+//
+// Resulting bitmaps are effectively immutable in the following sense:
+// a copy-on-write marker is used so that when you modify the resulting
+// bitmap, copies of selected data (containers) are made.
+// You should *not* change the copy-on-write status of the resulting
+// bitmaps (SetCopyOnWrite).
+//
+// If buf becomes unavailable, then a bitmap created with
+// FromBuffer would be effectively broken. Furthermore, any
+// bitmap derived from this bitmap (e.g., via Or, And) might
+// also be broken. Thus, before making buf unavailable, you should
+// call CloneCopyOnWriteContainers on all such bitmaps.
+func (rb *Bitmap) FrozenView(buf []byte) error {
+	return rb.highlowcontainer.frozenView(buf)
+}
+
+func (rb *Bitmap) MustFrozenView(buf []byte) error {
+	if err := rb.FrozenView(buf); err != nil {
+		return err
+	}
+	err := rb.Validate()
+
+	return err
+}
+
+/* Verbatim specification from CRoaring.
+ *
+ * FROZEN SERIALIZATION FORMAT DESCRIPTION
+ *
+ * -- (beginning must be aligned by 32 bytes) --
+ * <bitset_data> uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * num_bitset_containers]
+ * <run_data>    rle16_t[total number of rle elements in all run containers]
+ * <array_data>  uint16_t[total number of array elements in all array containers]
+ * <keys>        uint16_t[num_containers]
+ * <counts>      uint16_t[num_containers]
+ * <typecodes>   uint8_t[num_containers]
+ * <header>      uint32_t
+ *
+ * <header> is a 4-byte value which is a bit union of frozenCookie (15 bits)
+ * and the number of containers (17 bits).
+ *
+ * <counts> stores number of elements for every container.
+ * Its meaning depends on container type.
+ * For array and bitset containers, this value is the container cardinality minus one.
+ * For run container, it is the number of rle_t elements (n_runs).
+ *
+ * <bitset_data>,<array_data>,<run_data> are flat arrays of elements of
+ * all containers of respective type.
+ *
+ * <*_data> and <keys> are kept close together because they are not accessed
+ * during deserilization. This may reduce IO in case of large mmaped bitmaps.
+ * All members have their native alignments during deserilization except <header>,
+ * which is not guaranteed to be aligned by 4 bytes.
+ */
+const frozenCookie = 13766
+
+var (
+	// ErrFrozenBitmapInvalidCookie is returned when the header does not contain the frozenCookie.
+	ErrFrozenBitmapInvalidCookie = errors.New("header does not contain the frozenCookie")
+	// ErrFrozenBitmapBigEndian is returned when the header is big endian.
+	ErrFrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported")
+	// ErrFrozenBitmapIncomplete is returned when the buffer is too small to contain a frozen bitmap.
+	ErrFrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap")
+	// ErrFrozenBitmapOverpopulated is returned when the number of containers is too large.
+	ErrFrozenBitmapOverpopulated = errors.New("too many containers")
+	// ErrFrozenBitmapUnexpectedData is returned when the buffer contains unexpected data.
+	ErrFrozenBitmapUnexpectedData = errors.New("spurious data in input")
+	// ErrFrozenBitmapInvalidTypecode is returned when the typecode is invalid.
+	ErrFrozenBitmapInvalidTypecode = errors.New("unrecognized typecode")
+	// ErrFrozenBitmapBufferTooSmall is returned when the buffer is too small.
+	ErrFrozenBitmapBufferTooSmall = errors.New("buffer too small")
+)
+
+func (ra *roaringArray) frozenView(buf []byte) error {
+	if len(buf) < 4 {
+		return ErrFrozenBitmapIncomplete
+	}
+
+	headerBE := binary.BigEndian.Uint32(buf[len(buf)-4:])
+	if headerBE&0x7fff == frozenCookie {
+		return ErrFrozenBitmapBigEndian
+	}
+
+	header := binary.LittleEndian.Uint32(buf[len(buf)-4:])
+	buf = buf[:len(buf)-4]
+
+	if header&0x7fff != frozenCookie {
+		return ErrFrozenBitmapInvalidCookie
+	}
+
+	nCont := int(header >> 15)
+	if nCont > (1 << 16) {
+		return ErrFrozenBitmapOverpopulated
+	}
+
+	// 1 byte per type, 2 bytes per key, 2 bytes per count.
+	if len(buf) < 5*nCont {
+		return ErrFrozenBitmapIncomplete
+	}
+
+	types := buf[len(buf)-nCont:]
+	buf = buf[:len(buf)-nCont]
+
+	counts := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:])
+	buf = buf[:len(buf)-2*nCont]
+
+	keys := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:])
+	buf = buf[:len(buf)-2*nCont]
+
+	nBitmap, nArray, nRun := 0, 0, 0
+	nArrayEl, nRunEl := 0, 0
+	for i, t := range types {
+		switch t {
+		case 1:
+			nBitmap++
+		case 2:
+			nArray++
+			nArrayEl += int(counts[i]) + 1
+		case 3:
+			nRun++
+			nRunEl += int(counts[i])
+		default:
+			return ErrFrozenBitmapInvalidTypecode
+		}
+	}
+
+	if len(buf) < (1<<13)*nBitmap+4*nRunEl+2*nArrayEl {
+		return ErrFrozenBitmapIncomplete
+	}
+
+	bitsetsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBitmap])
+	buf = buf[(1<<13)*nBitmap:]
+
+	runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl])
+	buf = buf[4*nRunEl:]
+
+	arraysArena := byteSliceAsUint16Slice(buf[:2*nArrayEl])
+	buf = buf[2*nArrayEl:]
+
+	if len(buf) != 0 {
+		return ErrFrozenBitmapUnexpectedData
+	}
+
+	var c container
+	containersSz := int(unsafe.Sizeof(c)) * nCont
+	bitsetsSz := int(unsafe.Sizeof(bitmapContainer{})) * nBitmap
+	arraysSz := int(unsafe.Sizeof(arrayContainer{})) * nArray
+	runsSz := int(unsafe.Sizeof(runContainer16{})) * nRun
+	needCOWSz := int(unsafe.Sizeof(true)) * nCont
+
+	bitmapArenaSz := containersSz + bitsetsSz + arraysSz + runsSz + needCOWSz
+	bitmapArena := make([]byte, bitmapArenaSz)
+
+	containers := byteSliceAsContainerSlice(bitmapArena[:containersSz])
+	bitmapArena = bitmapArena[containersSz:]
+
+	bitsets := byteSliceAsBitsetSlice(bitmapArena[:bitsetsSz])
+	bitmapArena = bitmapArena[bitsetsSz:]
+
+	arrays := byteSliceAsArraySlice(bitmapArena[:arraysSz])
+	bitmapArena = bitmapArena[arraysSz:]
+
+	runs := byteSliceAsRun16Slice(bitmapArena[:runsSz])
+	bitmapArena = bitmapArena[runsSz:]
+
+	needCOW := byteSliceAsBoolSlice(bitmapArena)
+
+	iBitset, iArray, iRun := 0, 0, 0
+	for i, t := range types {
+		needCOW[i] = true
+
+		switch t {
+		case 1:
+			containers[i] = &bitsets[iBitset]
+			bitsets[iBitset].cardinality = int(counts[i]) + 1
+			bitsets[iBitset].bitmap = bitsetsArena[:1024]
+			bitsetsArena = bitsetsArena[1024:]
+			iBitset++
+		case 2:
+			containers[i] = &arrays[iArray]
+			sz := int(counts[i]) + 1
+			arrays[iArray].content = arraysArena[:sz]
+			arraysArena = arraysArena[sz:]
+			iArray++
+		case 3:
+			containers[i] = &runs[iRun]
+			runs[iRun].iv = runsArena[:counts[i]]
+			runsArena = runsArena[counts[i]:]
+			iRun++
+		}
+	}
+
+	// Not consuming the full input is a bug.
+	if iBitset != nBitmap || len(bitsetsArena) != 0 ||
+		iArray != nArray || len(arraysArena) != 0 ||
+		iRun != nRun || len(runsArena) != 0 {
+		panic("we missed something")
+	}
+
+	ra.keys = keys
+	ra.containers = containers
+	ra.needCopyOnWrite = needCOW
+	ra.copyOnWrite = true
+
+	return nil
+}
+
+// GetFrozenSizeInBytes returns the size in bytes of the frozen bitmap.
+func (rb *Bitmap) GetFrozenSizeInBytes() uint64 {
+	nBits, nArrayEl, nRunEl := uint64(0), uint64(0), uint64(0)
+	for _, c := range rb.highlowcontainer.containers {
+		switch v := c.(type) {
+		case *bitmapContainer:
+			nBits++
+		case *arrayContainer:
+			nArrayEl += uint64(len(v.content))
+		case *runContainer16:
+			nRunEl += uint64(len(v.iv))
+		}
+	}
+	return 4 + 5*uint64(len(rb.highlowcontainer.containers)) +
+		(nBits << 13) + 2*nArrayEl + 4*nRunEl
+}
+
+// Freeze serializes the bitmap in the CRoaring's frozen format.
+func (rb *Bitmap) Freeze() ([]byte, error) {
+	sz := rb.GetFrozenSizeInBytes()
+	buf := make([]byte, sz)
+	_, err := rb.FreezeTo(buf)
+	return buf, err
+}
+
+// FreezeTo serializes the bitmap in the CRoaring's frozen format.
+func (rb *Bitmap) FreezeTo(buf []byte) (int, error) {
+	containers := rb.highlowcontainer.containers
+	nCont := len(containers)
+
+	nBits, nArrayEl, nRunEl := 0, 0, 0
+	for _, c := range containers {
+		switch v := c.(type) {
+		case *bitmapContainer:
+			nBits++
+		case *arrayContainer:
+			nArrayEl += len(v.content)
+		case *runContainer16:
+			nRunEl += len(v.iv)
+		}
+	}
+
+	serialSize := 4 + 5*nCont + (1<<13)*nBits + 4*nRunEl + 2*nArrayEl
+	if len(buf) < serialSize {
+		return 0, ErrFrozenBitmapBufferTooSmall
+	}
+
+	bitsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBits])
+	buf = buf[(1<<13)*nBits:]
+
+	runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl])
+	buf = buf[4*nRunEl:]
+
+	arraysArena := byteSliceAsUint16Slice(buf[:2*nArrayEl])
+	buf = buf[2*nArrayEl:]
+
+	keys := byteSliceAsUint16Slice(buf[:2*nCont])
+	buf = buf[2*nCont:]
+
+	counts := byteSliceAsUint16Slice(buf[:2*nCont])
+	buf = buf[2*nCont:]
+
+	types := buf[:nCont]
+	buf = buf[nCont:]
+
+	header := uint32(frozenCookie | (nCont << 15))
+	binary.LittleEndian.PutUint32(buf[:4], header)
+
+	copy(keys, rb.highlowcontainer.keys[:])
+
+	for i, c := range containers {
+		switch v := c.(type) {
+		case *bitmapContainer:
+			copy(bitsArena, v.bitmap)
+			bitsArena = bitsArena[1024:]
+			counts[i] = uint16(v.cardinality - 1)
+			types[i] = 1
+		case *arrayContainer:
+			copy(arraysArena, v.content)
+			arraysArena = arraysArena[len(v.content):]
+			elems := len(v.content)
+			counts[i] = uint16(elems - 1)
+			types[i] = 2
+		case *runContainer16:
+			copy(runsArena, v.iv)
+			runs := len(v.iv)
+			runsArena = runsArena[runs:]
+			counts[i] = uint16(runs)
+			types[i] = 3
+		}
+	}
+
+	return serialSize, nil
+}
+
+// WriteFrozenTo serializes the bitmap in the CRoaring's frozen format.
+func (rb *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) {
+	// FIXME: this is a naive version that iterates 4 times through the
+	// containers and allocates 3*len(containers) bytes; it's quite likely
+	// it can be done more efficiently.
+	containers := rb.highlowcontainer.containers
+	written := 0
+
+	for _, c := range containers {
+		c, ok := c.(*bitmapContainer)
+		if !ok {
+			continue
+		}
+		n, err := wr.Write(uint64SliceAsByteSlice(c.bitmap))
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	for _, c := range containers {
+		c, ok := c.(*runContainer16)
+		if !ok {
+			continue
+		}
+		n, err := wr.Write(interval16SliceAsByteSlice(c.iv))
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	for _, c := range containers {
+		c, ok := c.(*arrayContainer)
+		if !ok {
+			continue
+		}
+		n, err := wr.Write(uint16SliceAsByteSlice(c.content))
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	n, err := wr.Write(uint16SliceAsByteSlice(rb.highlowcontainer.keys))
+	written += n
+	if err != nil {
+		return written, err
+	}
+
+	countTypeBuf := make([]byte, 3*len(containers))
+	counts := byteSliceAsUint16Slice(countTypeBuf[:2*len(containers)])
+	types := countTypeBuf[2*len(containers):]
+
+	for i, c := range containers {
+		switch c := c.(type) {
+		case *bitmapContainer:
+			counts[i] = uint16(c.cardinality - 1)
+			types[i] = 1
+		case *arrayContainer:
+			elems := len(c.content)
+			counts[i] = uint16(elems - 1)
+			types[i] = 2
+		case *runContainer16:
+			runs := len(c.iv)
+			counts[i] = uint16(runs)
+			types[i] = 3
+		}
+	}
+
+	n, err = wr.Write(countTypeBuf)
+	written += n
+	if err != nil {
+		return written, err
+	}
+
+	header := uint32(frozenCookie | (len(containers) << 15))
+	if err := binary.Write(wr, binary.LittleEndian, header); err != nil {
+		return written, err
+	}
+	written += 4
+
+	return written, nil
+}

vendor/github.com/RoaringBitmap/roaring/v2/serializationfuzz.go

@@ -0,0 +1,22 @@
+//go:build gofuzz
+// +build gofuzz
+
+package roaring
+
+import "bytes"
+
+func FuzzSerializationStream(data []byte) int {
+	newrb := NewBitmap()
+	if _, err := newrb.ReadFrom(bytes.NewReader(data)); err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzSerializationBuffer(data []byte) int {
+	newrb := NewBitmap()
+	if _, err := newrb.FromBuffer(data); err != nil {
+		return 0
+	}
+	return 1
+}

vendor/github.com/RoaringBitmap/roaring/v2/setutil.go

@@ -0,0 +1,709 @@
+package roaring
+
+func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	if len(set2) == 0 {
+		buffer = buffer[:len(set1)]
+		copy(buffer, set1)
+		return len(set1)
+	}
+	if len(set1) == 0 {
+		return 0
+	}
+	pos := 0
+	k1 := 0
+	k2 := 0
+	buffer = buffer[:cap(buffer)]
+	s1 := set1[k1]
+	s2 := set2[k2]
+	for {
+		if s1 < s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				break
+			}
+			s1 = set1[k1]
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s2 = set2[k2]
+		} else { // if (val1>val2)
+			k2++
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	if 0 == len(set2) {
+		buffer = buffer[:len(set1)]
+		copy(buffer, set1[:])
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		buffer = buffer[:len(set2)]
+		copy(buffer, set2[:])
+		return len(set2)
+	}
+	pos := 0
+	k1 := 0
+	k2 := 0
+	s1 := set1[k1]
+	s2 := set2[k2]
+	buffer = buffer[:cap(buffer)]
+	for {
+		if s1 < s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				for ; k2 < len(set2); k2++ {
+					buffer[pos] = set2[k2]
+					pos++
+				}
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				for ; k2 < len(set2); k2++ {
+					buffer[pos] = set2[k2]
+					pos++
+				}
+				break
+			}
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s1 = set1[k1]
+			s2 = set2[k2]
+		} else { // if (val1>val2)
+			buffer[pos] = s2
+			pos++
+			k2++
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+// union2by2Cardinality computes the cardinality of the union
+func union2by2Cardinality(set1 []uint16, set2 []uint16) int {
+	pos := 0
+	k1 := 0
+	k2 := 0
+	if 0 == len(set2) {
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		return len(set2)
+	}
+	s1 := set1[k1]
+	s2 := set2[k2]
+	for {
+		if s1 < s2 {
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				pos += len(set2) - k2
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			pos++
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				pos += len(set2) - k2
+				break
+			}
+			if k2 >= len(set2) {
+				pos += len(set1) - k1
+				break
+			}
+			s1 = set1[k1]
+			s2 = set2[k2]
+		} else { // if (set1[k1]>set2[k2])
+			pos++
+			k2++
+			if k2 >= len(set2) {
+				pos += len(set1) - k1
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func intersection2by2(
+	set1 []uint16,
+	set2 []uint16,
+	buffer []uint16,
+) int {
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2(set1, set2, buffer)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2(set2, set1, buffer)
+	} else {
+		return localintersect2by2(set1, set2, buffer)
+	}
+}
+
+// intersection2by2Cardinality computes the cardinality of the intersection
+func intersection2by2Cardinality(
+	set1 []uint16,
+	set2 []uint16,
+) int {
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2Cardinality(set1, set2)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2Cardinality(set2, set1)
+	} else {
+		return localintersect2by2Cardinality(set1, set2)
+	}
+}
+
+// intersects2by2 computes whether the two sets intersect
+func intersects2by2(
+	set1 []uint16,
+	set2 []uint16,
+) bool {
+	if (len(set1) == 0) || (len(set2) == 0) {
+		return false
+	}
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2Bool(set1, set2)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2Bool(set2, set1)
+	} else {
+		return intersects2by2Bool(set1, set2)
+	}
+}
+
+func intersects2by2Bool(
+	set1 []uint16,
+	set2 []uint16,
+) bool {
+	index1 := 0
+	index2 := 0
+	value1 := set1[index1]
+	value2 := set2[index2]
+mainwhile:
+	for {
+
+		if value2 < value1 {
+			for {
+				index2++
+				if index2 == len(set2) {
+					break mainwhile
+				}
+				value2 = set2[index2]
+				if value2 >= value1 {
+					break
+				}
+			}
+		}
+		if value1 < value2 {
+			for {
+				index1++
+				if index1 == len(set1) {
+					break mainwhile
+				}
+				value1 = set1[index1]
+				if value1 >= value2 {
+					break
+				}
+			}
+		} else {
+			// (set2[k2] == set1[k1])
+			return true
+		}
+	}
+	return false
+}
+
+func onesidedgallopingintersect2by2Bool(
+	smallset []uint16,
+	largeset []uint16,
+) bool {
+	k1 := 0
+	k2 := 0
+	s1 := largeset[k1]
+	s2 := smallset[k2]
+mainwhile:
+	for {
+		if s1 < s2 {
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+		if s2 < s1 {
+			k2++
+			if k2 == len(smallset) {
+				break mainwhile
+			}
+			s2 = smallset[k2]
+		} else {
+			// (set2[k2] == set1[k1])
+			return true
+		}
+
+	}
+	return false
+}
+
+func localintersect2by2(
+	set1 []uint16,
+	set2 []uint16,
+	buffer []uint16,
+) int {
+	if (len(set1) == 0) || (len(set2) == 0) {
+		return 0
+	}
+	k1 := 0
+	k2 := 0
+	pos := 0
+	buffer = buffer[:cap(buffer)]
+	s1 := set1[k1]
+	s2 := set2[k2]
+mainwhile:
+	for {
+		if s2 < s1 {
+			for {
+				k2++
+				if k2 == len(set2) {
+					break mainwhile
+				}
+				s2 = set2[k2]
+				if s2 >= s1 {
+					break
+				}
+			}
+		}
+		if s1 < s2 {
+			for {
+				k1++
+				if k1 == len(set1) {
+					break mainwhile
+				}
+				s1 = set1[k1]
+				if s1 >= s2 {
+					break
+				}
+			}
+		} else {
+			// (set2[k2] == set1[k1])
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 == len(set1) {
+				break
+			}
+			s1 = set1[k1]
+			k2++
+			if k2 == len(set2) {
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+// / localintersect2by2Cardinality computes the cardinality of the intersection
+func localintersect2by2Cardinality(
+	set1 []uint16,
+	set2 []uint16,
+) int {
+	if (len(set1) == 0) || (len(set2) == 0) {
+		return 0
+	}
+	index1 := 0
+	index2 := 0
+	pos := 0
+	value1 := set1[index1]
+	value2 := set2[index2]
+mainwhile:
+	for {
+		if value2 < value1 {
+			for {
+				index2++
+				if index2 == len(set2) {
+					break mainwhile
+				}
+				value2 = set2[index2]
+				if value2 >= value1 {
+					break
+				}
+			}
+		}
+		if value1 < value2 {
+			for {
+				index1++
+				if index1 == len(set1) {
+					break mainwhile
+				}
+				value1 = set1[index1]
+				if value1 >= value2 {
+					break
+				}
+			}
+		} else {
+			// (set2[k2] == set1[k1])
+			pos++
+			index1++
+			if index1 == len(set1) {
+				break
+			}
+			value1 = set1[index1]
+			index2++
+			if index2 == len(set2) {
+				break
+			}
+			value2 = set2[index2]
+		}
+	}
+	return pos
+}
+
+func advanceUntil(
+	array []uint16,
+	pos int,
+	length int,
+	min uint16,
+) int {
+	lower := pos + 1
+
+	if lower >= length || array[lower] >= min {
+		return lower
+	}
+
+	spansize := 1
+
+	for lower+spansize < length && array[lower+spansize] < min {
+		spansize *= 2
+	}
+	var upper int
+	if lower+spansize < length {
+		upper = lower + spansize
+	} else {
+		upper = length - 1
+	}
+
+	if array[upper] == min {
+		return upper
+	}
+
+	if array[upper] < min {
+		// means
+		// array
+		// has no
+		// item
+		// >= min
+		// pos = array.length;
+		return length
+	}
+
+	// we know that the next-smallest span was too small
+	lower += (spansize >> 1)
+
+	mid := 0
+	for lower+1 != upper {
+		mid = (lower + upper) >> 1
+		if array[mid] == min {
+			return mid
+		} else if array[mid] < min {
+			lower = mid
+		} else {
+			upper = mid
+		}
+	}
+	return upper
+}
+
+func onesidedgallopingintersect2by2(
+	smallset []uint16,
+	largeset []uint16,
+	buffer []uint16,
+) int {
+	if 0 == len(smallset) {
+		return 0
+	}
+	buffer = buffer[:cap(buffer)]
+	k1 := 0
+	k2 := 0
+	pos := 0
+	s1 := largeset[k1]
+	s2 := smallset[k2]
+mainwhile:
+
+	for {
+		if s1 < s2 {
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+		if s2 < s1 {
+			k2++
+			if k2 == len(smallset) {
+				break mainwhile
+			}
+			s2 = smallset[k2]
+		} else {
+
+			buffer[pos] = s2
+			pos++
+			k2++
+			if k2 == len(smallset) {
+				break
+			}
+			s2 = smallset[k2]
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+
+	}
+	return pos
+}
+
+func onesidedgallopingintersect2by2Cardinality(
+	smallset []uint16,
+	largeset []uint16,
+) int {
+	if 0 == len(smallset) {
+		return 0
+	}
+	k1 := 0
+	k2 := 0
+	pos := 0
+	s1 := largeset[k1]
+	s2 := smallset[k2]
+mainwhile:
+
+	for {
+		if s1 < s2 {
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+		if s2 < s1 {
+			k2++
+			if k2 == len(smallset) {
+				break mainwhile
+			}
+			s2 = smallset[k2]
+		} else {
+
+			pos++
+			k2++
+			if k2 == len(smallset) {
+				break
+			}
+			s2 = smallset[k2]
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+
+	}
+	return pos
+}
+
+func binarySearch(array []uint16, ikey uint16) int {
+	low := 0
+	high := len(array) - 1
+	for low+16 <= high {
+		middleIndex := int(uint32(low+high) >> 1)
+		middleValue := array[middleIndex]
+		if middleValue < ikey {
+			low = middleIndex + 1
+		} else if middleValue > ikey {
+			high = middleIndex - 1
+		} else {
+			return middleIndex
+		}
+	}
+	for ; low <= high; low++ {
+		val := array[low]
+		if val >= ikey {
+			if val == ikey {
+				return low
+			}
+			break
+		}
+	}
+	return -(low + 1)
+}
+
+// searchResult provides information about a search request.
+// The values will depend on the context of the search
+type searchResult struct {
+	value      uint16
+	index      int
+	exactMatch bool
+}
+
+// notFound returns a bool depending the search context
+// For cases `previousValue` and `nextValue` if target is present in the slice
+// this function will return `true` otherwise `false`
+// For `nextAbsentValue` and `previousAbsentValue` this will only return `False`
+func (sr *searchResult) notFound() bool {
+	return !sr.exactMatch
+}
+
+// outOfBounds indicates whether the target was outside the lower and upper bounds of the container
+func (sr *searchResult) outOfBounds() bool {
+	return sr.index <= -1
+}
+
+// binarySearchUntil is a helper function around binarySearchUntilWithBounds
+// The user does not have to pass in the lower and upper bound
+// The lower bound is taken to be `0` and the upper bound `len(array)-1`
+func binarySearchUntil(array []uint16, target uint16) searchResult {
+	return binarySearchUntilWithBounds(array, target, 0, len(array)-1)
+}
+
+// binarySearchUntilWithBounds returns a `searchResult`.
+// If an exact match is found the `searchResult{target, <index>, true}` will be returned, where `<index>` is
+// `target`s index in `array`, and `result.notFound()` evaluates to `false`.
+// If a match is not found, but `target` was in-bounds then the result.index will be the closest smaller value
+// Example: [ 8,9,11,12] if the target was 10, then `searchResult{9, 1, false}` will be returned.
+// If `target` was out of bounds `searchResult{0, -1, false}` will be returned.
+func binarySearchUntilWithBounds(array []uint16, target uint16, lowIndex int, maxIndex int) searchResult {
+	highIndex := maxIndex
+
+	closestIndex := -1
+
+	if target < array[lowIndex] {
+		return searchResult{0, closestIndex, false}
+	}
+
+	if target > array[maxIndex] {
+		return searchResult{0, len(array), false}
+	}
+
+	for lowIndex <= highIndex {
+		middleIndex := (lowIndex + highIndex) / 2
+		middleValue := array[middleIndex]
+
+		if middleValue == target {
+			return searchResult{middleValue, middleIndex, true}
+		}
+
+		if target < middleValue {
+
+			if middleIndex > 0 && target > array[middleIndex-1] {
+				return searchResult{array[middleIndex-1], middleIndex - 1, false}
+			}
+
+			highIndex = middleIndex
+		} else {
+			if middleIndex < maxIndex && target < array[middleIndex+1] {
+				return searchResult{middleValue, middleIndex, false}
+			}
+			lowIndex = middleIndex + 1
+		}
+
+	}
+
+	return searchResult{array[closestIndex], closestIndex, false}
+}
+
+// binarySearchPast is a wrapper around binarySearchPastWithBounds
+// The user does not have to pass in the lower and upper bound
+// The lower bound is taken to be `0` and the upper bound `len(array)-1`
+func binarySearchPast(array []uint16, target uint16) searchResult {
+	return binarySearchPastWithBounds(array, target, 0, len(array)-1)
+}
+
+// binarySearchPastWithBounds looks for the smallest value larger than or equal to `target`
+// If `target` is out of bounds a `searchResult` indicating out of bounds is returned
+// `target` does not have to exist in the slice.
+//
+// Example:
+// Suppose the slice is [...10,13...] with `target` equal to 11
+// The searchResult will have searchResult.value = 13
+func binarySearchPastWithBounds(array []uint16, target uint16, lowIndex int, maxIndex int) searchResult {
+	highIndex := maxIndex
+
+	closestIndex := -1
+
+	if target < array[lowIndex] {
+		return searchResult{0, closestIndex, false}
+	}
+
+	if target > array[maxIndex] {
+		return searchResult{0, len(array), false}
+	}
+
+	for lowIndex <= highIndex {
+		middleIndex := (lowIndex + highIndex) / 2
+		middleValue := array[middleIndex]
+
+		if middleValue == target {
+			return searchResult{middleValue, middleIndex, true}
+		}
+
+		if target < middleValue {
+
+			if middleIndex > 0 && target > array[middleIndex-1] {
+				return searchResult{array[middleIndex], middleIndex, false}
+			}
+
+			highIndex = middleIndex
+		} else {
+			if middleIndex < maxIndex && target < array[middleIndex+1] {
+				return searchResult{array[middleIndex+1], middleIndex + 1, false}
+			}
+			lowIndex = middleIndex + 1
+		}
+
+	}
+
+	return searchResult{array[closestIndex], closestIndex, false}
+}

vendor/github.com/RoaringBitmap/roaring/v2/setutil_arm64.go

@@ -0,0 +1,7 @@
+//go:build arm64 && !gccgo && !appengine
+// +build arm64,!gccgo,!appengine
+
+package roaring
+
+//go:noescape
+func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) (size int)

vendor/github.com/RoaringBitmap/roaring/v2/setutil_arm64.s

@@ -0,0 +1,132 @@
+// +build arm64,!gccgo,!appengine
+
+#include "textflag.h"
+
+
+// This implements union2by2 using golang's version of arm64 assembly
+// The algorithm is very similar to the generic one,
+// but makes better use of arm64 features so is notably faster.
+// The basic algorithm structure is as follows:
+// 1. If either set is empty, copy the other set into the buffer and return the length
+// 2. Otherwise, load the first element of each set into a variable (s1 and s2).
+// 3. a. Compare the values of s1 and s2.
+ // b. add the smaller one to the buffer.
+ // c. perform a bounds check before incrementing.
+ // If one set is finished, copy the rest of the other set over.
+ // d. update s1 and or s2 to the next value, continue loop.
+ //
+ // Past the fact of the algorithm, this code makes use of several arm64 features
+ // Condition Codes:
+ // arm64's CMP operation sets 4 bits that can be used for branching,
+ // rather than just true or false.
+ // As a consequence, a single comparison gives enough information to distinguish the three cases
+ //
+ // Post-increment pointers after load/store:
+ // Instructions like `MOVHU.P 2(R0), R6`
+ // increment the register by a specified amount, in this example 2.
+ // Because uint16's are exactly 2 bytes and the length of the slices
+ // is part of the slice header,
+ // there is no need to separately track the index into the slice.
+ // Instead, the code can calculate the final read value and compare against that,
+ // using the post-increment reads to move the pointers along.
+ //
+ // TODO: CALL out to memmove once the list is exhausted.
+ // Right now it moves the necessary shorts so that the remaining count
+ // is a multiple of 4 and then copies 64 bits at a time.
+
+TEXT ·union2by2(SB), NOSPLIT, $0-80
+	// R0, R1, and R2 for the pointers to the three slices
+	MOVD set1+0(FP), R0
+	MOVD set2+24(FP), R1
+	MOVD buffer+48(FP), R2
+
+	//R3 and R4 will be the values at which we will have finished reading set1 and set2.
+	// R3 should be R0 + 2 * set1_len+8(FP)
+	MOVD set1_len+8(FP), R3
+	MOVD set2_len+32(FP), R4
+
+	ADD R3<<1, R0, R3
+	ADD R4<<1, R1, R4
+
+
+	//Rather than counting the number of elements added separately
+	//Save the starting register of buffer.
+	MOVD buffer+48(FP), R5
+
+	// set1 is empty, just flush set2
+	CMP R0, R3
+	BEQ flush_right
+
+	// set2 is empty, just flush set1
+	CMP R1, R4
+	BEQ flush_left
+
+	// R6, R7 are the working space for s1 and s2
+	MOVD ZR, R6
+	MOVD ZR, R7
+
+	MOVHU.P 2(R0), R6
+	MOVHU.P 2(R1), R7
+loop:
+
+	CMP R6, R7
+	BEQ pop_both // R6 == R7
+	BLS pop_right // R6 > R7
+//pop_left: // R6 < R7
+	MOVHU.P R6, 2(R2)
+	CMP R0, R3
+	BEQ pop_then_flush_right
+	MOVHU.P 2(R0), R6
+	JMP loop
+pop_both:
+	MOVHU.P R6, 2(R2) //could also use R7, since they are equal
+	CMP R0, R3
+	BEQ flush_right
+	CMP R1, R4
+	BEQ flush_left
+	MOVHU.P 2(R0), R6
+	MOVHU.P 2(R1), R7
+	JMP loop
+pop_right:
+	MOVHU.P R7, 2(R2)
+	CMP R1, R4
+	BEQ pop_then_flush_left
+	MOVHU.P 2(R1), R7
+	JMP loop
+
+pop_then_flush_right:
+	MOVHU.P R7, 2(R2)
+flush_right:
+	MOVD R1, R0
+	MOVD R4, R3
+	JMP flush_left
+pop_then_flush_left:
+	MOVHU.P R6, 2(R2)
+flush_left:
+	CMP R0, R3
+	BEQ return
+	//figure out how many bytes to slough off. Must be a multiple of two
+	SUB R0, R3, R4
+	ANDS $6, R4
+	BEQ long_flush //handles the 0 mod 8 case
+	SUBS $4, R4, R4 // since possible values are 2, 4, 6, this splits evenly
+	BLT pop_single  // exactly the 2 case
+	MOVW.P 4(R0), R6
+	MOVW.P R6, 4(R2)
+	BEQ long_flush // we're now aligned by 64 bits, as R4==4, otherwise 2 more
+pop_single:
+	MOVHU.P 2(R0), R6
+	MOVHU.P R6, 2(R2)
+long_flush:
+	// at this point we know R3 - R0 is a multiple of 8.
+	CMP R0, R3
+	BEQ return
+	MOVD.P 8(R0), R6
+	MOVD.P R6, 8(R2)
+	JMP long_flush
+return:
+	// number of shorts written is (R5 - R2) >> 1
+	SUB R5, R2
+	LSR $1, R2, R2
+	MOVD R2, size+72(FP)
+	RET

vendor/github.com/RoaringBitmap/roaring/v2/setutil_generic.go

@@ -0,0 +1,64 @@
+//go:build !arm64 || gccgo || appengine
+// +build !arm64 gccgo appengine
+
+package roaring
+
+func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	pos := 0
+	k1 := 0
+	k2 := 0
+	if 0 == len(set2) {
+		buffer = buffer[:len(set1)]
+		copy(buffer, set1[:])
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		buffer = buffer[:len(set2)]
+		copy(buffer, set2[:])
+		return len(set2)
+	}
+	s1 := set1[k1]
+	s2 := set2[k2]
+	buffer = buffer[:cap(buffer)]
+	for {
+		if s1 < s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				copy(buffer[pos:], set2[k2:])
+				pos += len(set2) - k2
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				copy(buffer[pos:], set2[k2:])
+				pos += len(set2) - k2
+				break
+			}
+			if k2 >= len(set2) {
+				copy(buffer[pos:], set1[k1:])
+				pos += len(set1) - k1
+				break
+			}
+			s1 = set1[k1]
+			s2 = set2[k2]
+		} else { // if (set1[k1]>set2[k2])
+			buffer[pos] = s2
+			pos++
+			k2++
+			if k2 >= len(set2) {
+				copy(buffer[pos:], set1[k1:])
+				pos += len(set1) - k1
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}

vendor/github.com/RoaringBitmap/roaring/v2/shortiterator.go

@@ -0,0 +1,102 @@
+package roaring
+
+type shortIterable interface {
+	hasNext() bool
+	next() uint16
+}
+
+type shortPeekable interface {
+	shortIterable
+	peekNext() uint16
+	advanceIfNeeded(minval uint16)
+}
+
+type shortIterator struct {
+	slice []uint16
+	loc   int
+}
+
+func (si *shortIterator) hasNext() bool {
+	return si.loc < len(si.slice)
+}
+
+func (si *shortIterator) next() uint16 {
+	a := si.slice[si.loc]
+	si.loc++
+	return a
+}
+
+func (si *shortIterator) peekNext() uint16 {
+	return si.slice[si.loc]
+}
+
+func (si *shortIterator) advanceIfNeeded(minval uint16) {
+	if si.hasNext() && si.peekNext() < minval {
+		si.loc = advanceUntil(si.slice, si.loc, len(si.slice), minval)
+	}
+}
+
+type reverseIterator struct {
+	slice []uint16
+	loc   int
+}
+
+func (si *reverseIterator) hasNext() bool {
+	return si.loc >= 0
+}
+
+func (si *reverseIterator) next() uint16 {
+	a := si.slice[si.loc]
+	si.loc--
+	return a
+}
+
+type arrayContainerUnsetIterator struct {
+	content []uint16
+	// pos is the index of the next set bit that is >= nextVal.
+	// When nextVal reaches content[pos], pos is incremented.
+	pos     int
+	nextVal int
+}
+
+func (acui *arrayContainerUnsetIterator) next() uint16 {
+	val := acui.nextVal
+	acui.nextVal++
+	for acui.pos < len(acui.content) && uint16(acui.nextVal) >= acui.content[acui.pos] {
+		acui.nextVal++
+		acui.pos++
+	}
+	return uint16(val)
+}
+
+func (acui *arrayContainerUnsetIterator) hasNext() bool {
+	return acui.nextVal < 65536
+}
+
+func (acui *arrayContainerUnsetIterator) peekNext() uint16 {
+	return uint16(acui.nextVal)
+}
+
+func (acui *arrayContainerUnsetIterator) advanceIfNeeded(minval uint16) {
+	if !acui.hasNext() || acui.peekNext() >= minval {
+		return
+	}
+	acui.nextVal = int(minval)
+	acui.pos = binarySearch(acui.content, minval)
+	if acui.pos < 0 {
+		acui.pos = -acui.pos - 1
+	}
+	for acui.pos < len(acui.content) && uint16(acui.nextVal) >= acui.content[acui.pos] {
+		acui.nextVal++
+		acui.pos++
+	}
+}
+
+func newArrayContainerUnsetIterator(content []uint16) *arrayContainerUnsetIterator {
+	acui := &arrayContainerUnsetIterator{content: content, pos: 0, nextVal: 0}
+	for acui.pos < len(acui.content) && uint16(acui.nextVal) >= acui.content[acui.pos] {
+		acui.nextVal++
+		acui.pos++
+	}
+	return acui
+}

vendor/github.com/RoaringBitmap/roaring/v2/smat.go

@@ -0,0 +1,720 @@
+//go:build gofuzz
+// +build gofuzz
+
+/*
+# Instructions for smat testing for roaring
+
+[smat](https://github.com/mschoch/smat) is a framework that provides
+state machine assisted fuzz testing.
+
+To run the smat tests for roaring...
+
+## Prerequisites
+
+Go 1.18 or later (for native fuzzing support).
+
+## Steps
+
+1. Generate initial smat corpus:
+```
+go test -tags=gofuzz -run=TestGenerateSmatCorpus
+```
+You should see a directory `workdir` created with initial corpus files.
+
+2. Run the fuzz test:
+```
+go test -run='^$' -fuzz=FuzzSmat -fuzztime=300s -timeout=60s
+```
+
+Adjust `-fuzztime` as needed for longer or shorter runs. If crashes are found,
+check the test output and the reproducer files in the `workdir` directory.
+You may copy the reproducers to roaring_tests.go
+*/
+
+package roaring
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/bits-and-blooms/bitset"
+	"github.com/mschoch/smat"
+)
+
+// The native fuzz entry point lives in a _test.go file so the go test
+// fuzz engine discovers it. See smat_fuzz_test.go for the fuzz wrapper.
+
+var smatDebug = true
+
+const max_value = 1048576
+const max_pairs = 10
+
+func smatLog(prefix, format string, args ...interface{}) {
+	if smatDebug {
+		fmt.Print(prefix)
+		fmt.Printf(format, args...)
+	}
+}
+
+type smatContext struct {
+	pairs []*smatPair
+
+	// Two registers, x & y.
+	x int
+	y int
+
+	actions int
+	// per-context last action for this fuzz worker
+	lastAction *actionRecord
+}
+
+// actionRecord stores a snapshot of the state just before an action runs.
+type actionRecord struct {
+	Name          string
+	X, Y          int
+	PairSnapshots []string // base64-encoded MarshalBinary of each pair's Bitmap
+}
+
+type smatPair struct {
+	bm *Bitmap
+	bs *bitset.BitSet
+	// parent context (nil if unknown)
+	ctx *smatContext
+}
+
+// ------------------------------------------------------------------
+
+var smatActionMap = smat.ActionMap{
+	smat.ActionID('X'): smatAction("x++", smatWrap(func(c *smatContext) { c.x = (c.x + 1) % max_value })),
+	smat.ActionID('x'): smatAction("x--", smatWrap(func(c *smatContext) { c.x = (c.x - 1 + max_value) % max_value })),
+	smat.ActionID('Y'): smatAction("y++", smatWrap(func(c *smatContext) { c.y = (c.y + 1) % max_value })),
+	smat.ActionID('y'): smatAction("y--", smatWrap(func(c *smatContext) { c.y = (c.y - 1 + max_value) % max_value })),
+	smat.ActionID('*'): smatAction("x*y", smatWrap(func(c *smatContext) { c.x = (c.x * c.y) % max_value })),
+	smat.ActionID('<'): smatAction("x<<", smatWrap(func(c *smatContext) { c.x = (c.x << 1) % max_value })),
+
+	smat.ActionID('^'): smatAction("swap", smatWrap(func(c *smatContext) { c.x, c.y = c.y, c.x })),
+
+	smat.ActionID('['): smatAction(" pushPair", smatWrap(smatPushPair)),
+	smat.ActionID(']'): smatAction(" popPair", smatWrap(smatPopPair)),
+
+	smat.ActionID('B'): smatAction(" setBit", smatWrap(smatSetBit)),
+	smat.ActionID('b'): smatAction(" removeBit", smatWrap(smatRemoveBit)),
+
+	smat.ActionID('o'): smatAction(" or", smatWrap(smatOr)),
+	smat.ActionID('a'): smatAction(" and", smatWrap(smatAnd)),
+	smat.ActionID('z'): smatAction(" xor", smatWrap(smatXor)),
+
+	smat.ActionID('#'): smatAction(" cardinality", smatWrap(smatCardinality)),
+
+	smat.ActionID('O'): smatAction(" orCardinality", smatWrap(smatOrCardinality)),
+	smat.ActionID('A'): smatAction(" andCardinality", smatWrap(smatAndCardinality)),
+	smat.ActionID('Z'): smatAction(" xorCardinality", smatWrap(smatXorCardinality)),
+
+	smat.ActionID('c'): smatAction(" clear", smatWrap(smatClear)),
+	smat.ActionID('r'): smatAction(" runOptimize", smatWrap(smatRunOptimize)),
+
+	smat.ActionID('e'): smatAction(" isEmpty", smatWrap(smatIsEmpty)),
+
+	smat.ActionID('i'): smatAction(" intersects", smatWrap(smatIntersects)),
+
+	smat.ActionID('f'): smatAction(" flip", smatWrap(smatFlip)),
+
+	smat.ActionID('-'): smatAction(" difference", smatWrap(smatDifference)),
+}
+
+var smatRunningPercentActions []smat.PercentAction
+
+func init() {
+	var ids []int
+	for actionId := range smatActionMap {
+		ids = append(ids, int(actionId))
+	}
+	slices.Sort(ids)
+
+	pct := 100 / len(smatActionMap)
+	for _, actionId := range ids {
+		smatRunningPercentActions = append(smatRunningPercentActions,
+			smat.PercentAction{Percent: pct, Action: smat.ActionID(actionId)})
+	}
+
+	smatActionMap[smat.ActionID('S')] = smatAction("SETUP", smatSetupFunc)
+	smatActionMap[smat.ActionID('T')] = smatAction("TEARDOWN", smatTeardownFunc)
+}
+
+// We only have one smat state: running.
+func smatRunning(next byte) smat.ActionID {
+	return smat.PercentExecute(next, smatRunningPercentActions...)
+}
+
+func smatAction(name string, f func(ctx smat.Context) (smat.State, error)) func(smat.Context) (smat.State, error) {
+	return func(ctx smat.Context) (smat.State, error) {
+		c := ctx.(*smatContext)
+
+		// Snapshot all pairs' bitmaps (base64 of MarshalBinary) before action
+		rec := actionRecord{Name: name, X: c.x, Y: c.y}
+		if len(c.pairs) > 0 {
+			rec.PairSnapshots = make([]string, 0, len(c.pairs))
+			for _, pair := range c.pairs {
+				if pair == nil || pair.bm == nil {
+					rec.PairSnapshots = append(rec.PairSnapshots, "<nil>")
+					continue
+				}
+				b, err := pair.bm.MarshalBinary()
+				if err != nil {
+					rec.PairSnapshots = append(rec.PairSnapshots, "<marshal-error:"+err.Error()+">")
+				} else {
+					rec.PairSnapshots = append(rec.PairSnapshots, base64.StdEncoding.EncodeToString(b))
+				}
+			}
+		}
+
+		// record per-context last action (no global mutex required)
+		if c != nil {
+			c.lastAction = &rec
+		}
+
+		// catch panics inside action to dump a repro and stack before re-panicking
+		defer func() {
+			if r := recover(); r != nil {
+				// best-effort: write quick repro with lastAction from context
+				var lastAction *actionRecord
+				if c != nil {
+					lastAction = c.lastAction
+				}
+				ts := time.Now().UnixNano()
+				repro := "// Reproducer generated by smat (panic)\n"
+				repro += "package roaring\n\n"
+				repro += "import (\n\t\"encoding/base64\"\n\t\"testing\"\n)\n\n"
+				repro += fmt.Sprintf("func TestFuzzerPanicRepro_%d(t *testing.T) {\n", ts)
+				// similar to checkEquals repro
+				if lastAction != nil && len(lastAction.PairSnapshots) > 0 {
+					pairIndex := lastAction.X % len(lastAction.PairSnapshots)
+					if pairIndex < len(lastAction.PairSnapshots) {
+						snapshot := lastAction.PairSnapshots[pairIndex]
+						if snapshot != "<nil>" && !strings.HasPrefix(snapshot, "<") {
+							repro += fmt.Sprintf("\tb, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshot)
+							repro += "\tbm := NewBitmap()\n"
+							repro += "\tbm.UnmarshalBinary(b)\n"
+							// perform the action that caused panic
+							if strings.Contains(lastAction.Name, "setBit") {
+								repro += fmt.Sprintf("\tbm.AddInt(%d)\n", lastAction.Y)
+							} else if strings.Contains(lastAction.Name, "removeBit") {
+								repro += fmt.Sprintf("\tbm.Remove(%d)\n", lastAction.Y)
+							} else if strings.Contains(lastAction.Name, "flip") {
+								repro += fmt.Sprintf("\tbm.Flip(uint64(%d), uint64(%d)+1)\n", lastAction.Y, lastAction.Y)
+							} else if strings.Contains(lastAction.Name, "runOptimize") {
+								repro += "\tbm.RunOptimize()\n"
+							} else if strings.Contains(lastAction.Name, "clear") {
+								repro += "\tbm.Clear()\n"
+							} else if lastAction.Name == " or" {
+								pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+								if pairIndexY < len(lastAction.PairSnapshots) {
+									snapshotY := lastAction.PairSnapshots[pairIndexY]
+									if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+										repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+										repro += "\tbm2 := NewBitmap()\n"
+										repro += "\tbm2.UnmarshalBinary(b2)\n"
+										repro += "\tbm.Or(bm2)\n"
+									}
+								}
+							} else if lastAction.Name == " and" {
+								pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+								if pairIndexY < len(lastAction.PairSnapshots) {
+									snapshotY := lastAction.PairSnapshots[pairIndexY]
+									if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+										repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+										repro += "\tbm2 := NewBitmap()\n"
+										repro += "\tbm2.UnmarshalBinary(b2)\n"
+										repro += "\tbm.And(bm2)\n"
+									}
+								}
+							} else if lastAction.Name == " difference" {
+								pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+								if pairIndexY < len(lastAction.PairSnapshots) {
+									snapshotY := lastAction.PairSnapshots[pairIndexY]
+									if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+										repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+										repro += "\tbm2 := NewBitmap()\n"
+										repro += "\tbm2.UnmarshalBinary(b2)\n"
+										repro += "\tbm.AndNot(bm2)\n"
+									}
+								}
+							} else if lastAction.Name == " xor" {
+								pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+								if pairIndexY < len(lastAction.PairSnapshots) {
+									snapshotY := lastAction.PairSnapshots[pairIndexY]
+									if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+										repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+										repro += "\tbm2 := NewBitmap()\n"
+										repro += "\tbm2.UnmarshalBinary(b2)\n"
+										repro += "\tbm.Xor(bm2)\n"
+									}
+								}
+							} else {
+								repro += fmt.Sprintf("\t// Unhandled action: %s\n", lastAction.Name)
+							}
+						} else {
+							repro += "\t// invalid snapshot\n"
+						}
+					}
+				}
+				repro += "}\n"
+				if path, werr := saveReproFile("smat_panic_repro", ts, repro); werr == nil {
+					fmt.Printf("wrote panic repro to %s\n", path)
+				} else {
+					fmt.Printf("failed writing panic repro: %v\n", werr)
+				}
+				fmt.Printf("PANIC in action %s: %v\n", rec.Name, r)
+				fmt.Printf("stack:\n%s\n", debug.Stack())
+				panic(r)
+			}
+		}()
+
+		c.actions++
+		return f(ctx)
+	}
+}
+
+// saveReproFile writes the given repro content to workdir/<prefix>_<ts>_test.go
+// or falls back to the OS temp dir. Returns full path or error.
+func saveReproFile(prefix string, ts int64, content string) (string, error) {
+	// try workdir
+	if err := os.MkdirAll("workdir", 0o755); err == nil {
+		fname := fmt.Sprintf("workdir/%s_%d_test.go", prefix, ts)
+		if err := os.WriteFile(fname, []byte(content), 0o644); err == nil {
+			return fname, nil
+		}
+	}
+	// fallback to temp
+	tmp := os.TempDir()
+	fname := fmt.Sprintf("%s_%d_test.go", prefix, ts)
+	full := filepath.Join(tmp, fname)
+	if err := os.WriteFile(full, []byte(content), 0o644); err == nil {
+		return full, nil
+	} else {
+		return "", err
+	}
+}
+
+// Creates an smat action func based on a simple callback.
+func smatWrap(cb func(c *smatContext)) func(smat.Context) (next smat.State, err error) {
+	return func(ctx smat.Context) (next smat.State, err error) {
+		c := ctx.(*smatContext)
+		cb(c)
+		return smatRunning, nil
+	}
+}
+
+// Invokes a callback function with the input v bounded to len(c.pairs).
+func (c *smatContext) withPair(v int, cb func(*smatPair)) {
+	if len(c.pairs) > 0 {
+		if v < 0 {
+			v = -v
+		}
+		v = v % len(c.pairs)
+		cb(c.pairs[v])
+	}
+}
+
+// ------------------------------------------------------------------
+
+func smatSetupFunc(ctx smat.Context) (next smat.State, err error) {
+	return smatRunning, nil
+}
+
+func smatTeardownFunc(ctx smat.Context) (next smat.State, err error) {
+	return nil, err
+}
+
+// ------------------------------------------------------------------
+
+func smatPushPair(c *smatContext) {
+	if len(c.pairs) >= max_pairs {
+		return
+	}
+	p := &smatPair{
+		bm:  NewBitmap(),
+		bs:  bitset.New(100),
+		ctx: c,
+	}
+	c.pairs = append(c.pairs, p)
+}
+
+func smatPopPair(c *smatContext) {
+	if len(c.pairs) > 0 {
+		c.pairs = c.pairs[0 : len(c.pairs)-1]
+	}
+}
+
+func smatSetBit(c *smatContext) {
+	c.withPair(c.x, func(p *smatPair) {
+		p.Validate()
+		y := uint32(c.y)
+		p.bm.AddInt(int(y))
+		p.bs.Set(uint(y))
+		p.checkEquals()
+	})
+}
+
+func smatRemoveBit(c *smatContext) {
+	c.withPair(c.x, func(p *smatPair) {
+		p.Validate()
+		y := uint32(c.y)
+		p.bm.Remove(y)
+		p.bs.Clear(uint(y))
+		p.checkEquals()
+	})
+}
+
+func smatAnd(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			px.bm.And(py.bm)
+			px.bs = px.bs.Intersection(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatOr(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			px.bm.Or(py.bm)
+			px.bs = px.bs.Union(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatXor(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			px.bm.Xor(py.bm)
+			px.bs = px.bs.SymmetricDifference(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatAndCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			c0 := px.bm.AndCardinality(py.bm)
+			c1 := px.bs.IntersectionCardinality(py.bs)
+			if c0 != uint64(c1) {
+				panic("expected same add cardinality")
+			}
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatOrCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			c0 := px.bm.OrCardinality(py.bm)
+			c1 := px.bs.UnionCardinality(py.bs)
+			if c0 != uint64(c1) {
+				panic("expected same or cardinality")
+			}
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatXorCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			c0 := px.bm.OrCardinality(py.bm) - px.bm.AndCardinality(py.bm)
+			c1 := px.bs.SymmetricDifferenceCardinality(py.bs)
+			if c0 != uint64(c1) {
+				panic("expected same xor cardinality")
+			}
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatRunOptimize(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		px.Validate()
+		px.bm.RunOptimize()
+		px.checkEquals()
+	})
+}
+
+func smatClear(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		px.Validate()
+		px.bm.Clear()
+		px.bs = px.bs.ClearAll()
+		px.checkEquals()
+	})
+}
+
+func smatCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c0 := px.bm.GetCardinality()
+		c1 := px.bs.Count()
+		if c0 != uint64(c1) {
+			panic("expected same cardinality")
+		}
+	})
+}
+
+func smatIsEmpty(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c0 := px.bm.IsEmpty()
+		c1 := px.bs.None()
+		if c0 != c1 {
+			panic("expected same is empty")
+		}
+	})
+}
+
+func smatIntersects(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			v0 := px.bm.Intersects(py.bm)
+			v1 := px.bs.IntersectionCardinality(py.bs) > 0
+			if v0 != v1 {
+				panic("intersects not equal")
+			}
+
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatFlip(c *smatContext) {
+	c.withPair(c.x, func(p *smatPair) {
+		p.Validate()
+		y := uint32(c.y)
+		p.bm.Flip(uint64(y), uint64(y)+1)
+		p.bs = p.bs.Flip(uint(y))
+		p.checkEquals()
+	})
+}
+
+func smatDifference(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.Validate()
+			py.Validate()
+			px.bm.AndNot(py.bm)
+			px.bs = px.bs.Difference(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func (p *smatPair) checkEquals() {
+	valid := p.bm.Validate()
+	if valid != nil {
+		// marshal current bitmap
+		var curSnap string
+		if p != nil && p.bm != nil {
+			if b, err := p.bm.MarshalBinary(); err == nil {
+				curSnap = base64.StdEncoding.EncodeToString(b)
+			} else {
+				curSnap = "<marshal-error:" + err.Error() + ">"
+			}
+		} else {
+			curSnap = "<nil>"
+		}
+
+		// collect last action summary from context (per-worker)
+		last := "<none>"
+		if p != nil && p.ctx != nil {
+			c := p.ctx
+			if c.lastAction != nil {
+				last = fmt.Sprintf("action=%s x=%d y=%d pairs=%d", c.lastAction.Name, c.lastAction.X, c.lastAction.Y, len(c.lastAction.PairSnapshots))
+			}
+		}
+
+		// If debugging enabled, log extra info
+		smatLog("ERROR: ", "bitmap invalid: %v\n", valid)
+
+		// build a reproducible test snippet that reconstructs the bitmap and replays the failing action
+		ts := time.Now().UnixNano()
+		testName := fmt.Sprintf("TestFuzzerRepro_%d", ts)
+		repro := "// Reproducer generated by smat\n"
+		repro += "package roaring\n\n"
+		repro += "import (\n\t\"encoding/base64\"\n\t\"testing\"\n)\n\n"
+		repro += fmt.Sprintf("func %s(t *testing.T) {\n", testName)
+		var lastAction *actionRecord
+		if p != nil && p.ctx != nil {
+			lastAction = p.ctx.lastAction
+		}
+		// use the snapshot of the modified pair
+		if lastAction != nil && len(lastAction.PairSnapshots) > 0 {
+			// assume the modified pair is x % len(pairs), but since pairs are in order, and x is lastAction.X
+			pairIndex := lastAction.X % len(lastAction.PairSnapshots)
+			if pairIndex < len(lastAction.PairSnapshots) {
+				snapshot := lastAction.PairSnapshots[pairIndex]
+				if snapshot != "<nil>" && !strings.HasPrefix(snapshot, "<") {
+					repro += fmt.Sprintf("\tb, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshot)
+					repro += "\tbm := NewBitmap()\n"
+					repro += "\tbm.UnmarshalBinary(b)\n"
+					repro += "\tif err := bm.Validate(); err != nil {\n"
+					repro += "\t\tt.Errorf(\"Initial Validate failed: %v\", err)\n"
+					repro += "\t}\n"
+					// perform the action
+					if strings.Contains(lastAction.Name, "setBit") {
+						repro += fmt.Sprintf("\tbm.AddInt(%d)\n", lastAction.Y)
+					} else if strings.Contains(lastAction.Name, "removeBit") {
+						repro += fmt.Sprintf("\tbm.Remove(%d)\n", lastAction.Y)
+					} else if strings.Contains(lastAction.Name, "flip") {
+						repro += fmt.Sprintf("\tbm.Flip(uint64(%d), uint64(%d)+1)\n", lastAction.Y, lastAction.Y)
+					} else if strings.Contains(lastAction.Name, "runOptimize") {
+						repro += "\tbm.RunOptimize()\n"
+					} else if strings.Contains(lastAction.Name, "clear") {
+						repro += "\tbm.Clear()\n"
+					} else if lastAction.Name == " or" {
+						pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+						if pairIndexY < len(lastAction.PairSnapshots) {
+							snapshotY := lastAction.PairSnapshots[pairIndexY]
+							if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+								repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+								repro += "\tbm2 := NewBitmap()\n"
+								repro += "\tbm2.UnmarshalBinary(b2)\n"
+								repro += "\tbm.Or(bm2)\n"
+							}
+						}
+					} else if lastAction.Name == " and" {
+						pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+						if pairIndexY < len(lastAction.PairSnapshots) {
+							snapshotY := lastAction.PairSnapshots[pairIndexY]
+							if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+								repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+								repro += "\tbm2 := NewBitmap()\n"
+								repro += "\tbm2.UnmarshalBinary(b2)\n"
+								repro += "\tbm.And(bm2)\n"
+							}
+						}
+					} else if lastAction.Name == " difference" {
+						pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+						if pairIndexY < len(lastAction.PairSnapshots) {
+							snapshotY := lastAction.PairSnapshots[pairIndexY]
+							if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+								repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+								repro += "\tbm2 := NewBitmap()\n"
+								repro += "\tbm2.UnmarshalBinary(b2)\n"
+								repro += "\tbm.AndNot(bm2)\n"
+							}
+						}
+					} else if lastAction.Name == " xor" {
+						pairIndexY := lastAction.Y % len(lastAction.PairSnapshots)
+						if pairIndexY < len(lastAction.PairSnapshots) {
+							snapshotY := lastAction.PairSnapshots[pairIndexY]
+							if snapshotY != "<nil>" && !strings.HasPrefix(snapshotY, "<") {
+								repro += fmt.Sprintf("\tb2, _ := base64.StdEncoding.DecodeString(\"%s\")\n", snapshotY)
+								repro += "\tbm2 := NewBitmap()\n"
+								repro += "\tbm2.UnmarshalBinary(b2)\n"
+								repro += "\tbm.Xor(bm2)\n"
+							}
+						}
+					} else {
+						repro += fmt.Sprintf("\t// Unhandled action: %s\n", lastAction.Name)
+					}
+					repro += "\tif err := bm.Validate(); err != nil {\n"
+					repro += "\t\tt.Errorf(\"Validate failed: %v\", err)\n"
+					repro += "\t} else {\n"
+					repro += "\t\tt.Logf(\"Validate succeeded\")\n"
+					repro += "\t}\n"
+				} else {
+					repro += "\t// invalid snapshot\n"
+				}
+			}
+		}
+		repro += "}\n"
+
+		// print the repro snippet for the developer
+		fmt.Println()
+		fmt.Println("=== SMAT REPRODUCER SNIPPET ===")
+		if len(repro) > 10000 {
+			fmt.Println("// Reproducer too large, skipping full print")
+		} else {
+			fmt.Println(repro)
+		}
+
+		// also write the repro snippet to a timestamped file in workdir/
+		if len(repro) > 10000 {
+			repro = "// Reproducer too large, skipping\n"
+		}
+		if err := os.MkdirAll("workdir", 0o755); err == nil {
+			fname := fmt.Sprintf("workdir/smat_repro_%d_test.go", ts)
+			if werr := os.WriteFile(fname, []byte(repro), 0o644); werr == nil {
+				fmt.Printf("Wrote repro to %s\n", fname)
+			} else {
+				fmt.Printf("Failed writing repro file: %v\n", werr)
+			}
+		} else {
+			fmt.Printf("Failed creating workdir: %v\n", err)
+		}
+
+		panic(fmt.Sprintf("[checkEquals] bitmap invalid: %v\ncurrentBase64:%s\nlastAction:%s\n", valid, curSnap, last))
+	}
+	if !p.equalsBitSet(p.bs, p.bm) {
+		panic("bitset mismatch")
+	}
+}
+
+func (p *smatPair) Validate() {
+	valid := p.bm.Validate()
+	if valid != nil {
+		panic(fmt.Sprintf("[Validate] bitmap invalid: %v", valid))
+	}
+}
+
+func (p *smatPair) equalsBitSet(a *bitset.BitSet, b *Bitmap) bool {
+	for i, e := a.NextSet(0); e; i, e = a.NextSet(i + 1) {
+		if !b.ContainsInt(int(i)) {
+			fmt.Printf("in a bitset, not b bitmap, i: %d\n", i)
+			fmt.Printf("  a bitset: %s\n  b bitmap: %s\n",
+				a.String(), b.String())
+			return false
+		}
+	}
+
+	i := b.Iterator()
+	for i.HasNext() {
+		v := i.Next()
+		if !a.Test(uint(v)) {
+			fmt.Printf("in b bitmap, not a bitset, v: %d\n", v)
+			fmt.Printf("  a bitset: %s\n  b bitmap: %s\n",
+				a.String(), b.String())
+			return false
+		}
+	}
+
+	return true
+}

vendor/github.com/RoaringBitmap/roaring/v2/util.go

@@ -0,0 +1,309 @@
+package roaring
+
+import (
+	"cmp"
+	"math"
+	"math/bits"
+	"math/rand"
+	"slices"
+)
+
+const (
+	arrayDefaultMaxSize        = 4096 // containers with 4096 or fewer integers should be array containers.
+	arrayLazyLowerBound        = 1024
+	maxCapacity                = 1 << 16
+	serialCookieNoRunContainer = 12346 // only arrays and bitmaps
+	invalidCardinality         = -1
+	serialCookie               = 12347 // runs, arrays, and bitmaps
+	noOffsetThreshold          = 4
+
+	// MaxUint32 is the largest uint32 value.
+	MaxUint32 = math.MaxUint32
+
+	// MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper
+	// bound for ranges.
+	MaxRange uint64 = MaxUint32 + 1
+
+	// MaxUint16 is the largest 16 bit unsigned int.
+	// This is the largest value an interval16 can store.
+	MaxUint16 = math.MaxUint16
+
+	// Compute wordSizeInBytes, the size of a word in bytes.
+	_m              = ^uint64(0)
+	_logS           = _m>>8&1 + _m>>16&1 + _m>>32&1
+	wordSizeInBytes = 1 << _logS
+
+	// other constants used in ctz_generic.go
+	wordSizeInBits = wordSizeInBytes << 3 // word size in bits
+)
+
+const maxWord = 1<<wordSizeInBits - 1
+
+// doesn't apply to runContainers
+func getSizeInBytesFromCardinality(card int) int {
+	if card > arrayDefaultMaxSize {
+		// bitmapContainer
+		return maxCapacity / 8
+	}
+	// arrayContainer
+	return 2 * card
+}
+
+func fill(arr []uint64, val uint64) {
+	for i := range arr {
+		arr[i] = val
+	}
+}
+
+func fillRange(arr []uint64, start, end int, val uint64) {
+	for i := start; i < end; i++ {
+		arr[i] = val
+	}
+}
+
+func fillArrayAND(container []uint16, bitmap1, bitmap2 []uint64) {
+	if len(bitmap1) != len(bitmap2) {
+		panic("array lengths don't match")
+	}
+	// TODO: rewrite in assembly
+	pos := 0
+	for k := range bitmap1 {
+		bitset := bitmap1[k] & bitmap2[k]
+		for bitset != 0 {
+			t := bitset & -bitset
+			container[pos] = uint16((k*64 + bits.OnesCount64(t-1)))
+			pos = pos + 1
+			bitset ^= t
+		}
+	}
+}
+
+func fillArrayANDNOT(container []uint16, bitmap1, bitmap2 []uint64) {
+	if len(bitmap1) != len(bitmap2) {
+		panic("array lengths don't match")
+	}
+	// TODO: rewrite in assembly
+	pos := 0
+	for k := range bitmap1 {
+		bitset := bitmap1[k] &^ bitmap2[k]
+		for bitset != 0 {
+			t := bitset & -bitset
+			container[pos] = uint16((k*64 + bits.OnesCount64(t-1)))
+			pos = pos + 1
+			bitset ^= t
+		}
+	}
+}
+
+func fillArrayXOR(container []uint16, bitmap1, bitmap2 []uint64) {
+	if len(bitmap1) != len(bitmap2) {
+		panic("array lengths don't match")
+	}
+	// TODO: rewrite in assembly
+	pos := 0
+	for k := 0; k < len(bitmap1); k++ {
+		bitset := bitmap1[k] ^ bitmap2[k]
+		for bitset != 0 {
+			t := bitset & -bitset
+			container[pos] = uint16((k*64 + bits.OnesCount64(t-1)))
+			pos = pos + 1
+			bitset ^= t
+		}
+	}
+}
+
+func highbits(x uint32) uint16 {
+	return uint16(x >> 16)
+}
+
+func lowbits(x uint32) uint16 {
+	return uint16(x & maxLowBit)
+}
+
+func combineLoHi16(lob uint16, hob uint16) uint32 {
+	return combineLoHi32(uint32(lob), uint32(hob))
+}
+
+func combineLoHi32(lob uint32, hob uint32) uint32 {
+	return lob | (hob << 16)
+}
+
+const maxLowBit = 0xFFFF
+
+func flipBitmapRange(bitmap []uint64, start int, end int) {
+	if start >= end {
+		return
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	bitmap[firstword] ^= ^(^uint64(0) << uint(start%64))
+	for i := firstword; i < endword; i++ {
+		bitmap[i] = ^bitmap[i]
+	}
+	bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64)
+}
+
+func resetBitmapRange(bitmap []uint64, start int, end int) {
+	if start >= end {
+		return
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	if firstword == endword {
+		bitmap[firstword] &= ^((^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)))
+		return
+	}
+	bitmap[firstword] &= ^(^uint64(0) << uint(start%64))
+	for i := firstword + 1; i < endword; i++ {
+		bitmap[i] = 0
+	}
+	bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64))
+}
+
+func setBitmapRange(bitmap []uint64, start int, end int) {
+	if start >= end {
+		return
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	if firstword == endword {
+		bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64))
+		return
+	}
+	bitmap[firstword] |= ^uint64(0) << uint(start%64)
+	for i := firstword + 1; i < endword; i++ {
+		bitmap[i] = ^uint64(0)
+	}
+	bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64)
+}
+
+func flipBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
+	before := wordCardinalityForBitmapRange(bitmap, start, end)
+	flipBitmapRange(bitmap, start, end)
+	after := wordCardinalityForBitmapRange(bitmap, start, end)
+	return int(after - before)
+}
+
+func resetBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
+	before := wordCardinalityForBitmapRange(bitmap, start, end)
+	resetBitmapRange(bitmap, start, end)
+	after := wordCardinalityForBitmapRange(bitmap, start, end)
+	return int(after - before)
+}
+
+func setBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
+	before := wordCardinalityForBitmapRange(bitmap, start, end)
+	setBitmapRange(bitmap, start, end)
+	after := wordCardinalityForBitmapRange(bitmap, start, end)
+	return int(after - before)
+}
+
+func wordCardinalityForBitmapRange(bitmap []uint64, start int, end int) uint64 {
+	answer := uint64(0)
+	if start >= end {
+		return answer
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	for i := firstword; i <= endword; i++ {
+		answer += uint64(bits.OnesCount64(bitmap[i]))
+	}
+	return answer
+}
+
+func selectBitPosition(w uint64, j int) int {
+	seen := 0
+
+	// Divide 64bit
+	part := w & 0xFFFFFFFF
+	n := bits.OnesCount64(part)
+	if n <= j {
+		part = w >> 32
+		seen += 32
+		j -= n
+	}
+	w = part
+
+	// Divide 32bit
+	part = w & 0xFFFF
+	n = bits.OnesCount64(part)
+	if n <= j {
+		part = w >> 16
+		seen += 16
+		j -= n
+	}
+	w = part
+
+	// Divide 16bit
+	part = w & 0xFF
+	n = bits.OnesCount64(part)
+	if n <= j {
+		part = w >> 8
+		seen += 8
+		j -= n
+	}
+	w = part
+
+	// Lookup in final byte
+	var counter uint
+	for counter = 0; counter < 8; counter++ {
+		j -= int((w >> counter) & 1)
+		if j < 0 {
+			break
+		}
+	}
+	return seen + int(counter)
+}
+
+func panicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+type ph struct {
+	orig int
+	rand int
+}
+
+func getRandomPermutation(n int) []int {
+	r := make([]ph, n)
+	for i := 0; i < n; i++ {
+		r[i].orig = i
+		r[i].rand = rand.Intn(1 << 29)
+	}
+	slices.SortFunc(r, func(a, b ph) int { return cmp.Compare(a.rand, b.rand) })
+	m := make([]int, n)
+	for i := range m {
+		m[i] = r[i].orig
+	}
+	return m
+}
+
+func minOfInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxOfInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func maxOfUint16(a, b uint16) uint16 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func minOfUint16(a, b uint16) uint16 {
+	if a < b {
+		return a
+	}
+	return b
+}

vendor/github.com/bits-and-blooms/bitset/.gitignore

@@ -0,0 +1,26 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+target

vendor/github.com/bits-and-blooms/bitset/.travis.yml

@@ -0,0 +1,37 @@
+language: go
+
+sudo: false
+
+branches:
+  except:
+    - release
+
+branches:
+  only:
+    - master
+    - travis
+
+go:
+  - "1.11.x"
+  - tip
+
+matrix:
+  allow_failures:
+    - go: tip
+
+before_install:
+  - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi;
+  - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi;
+  - go get github.com/mattn/goveralls
+
+before_script:
+  - make deps
+
+script:
+  - make qa
+
+after_failure:
+  - cat ./target/test/report.xml
+
+after_success:
+  - if [ "$TRAVIS_GO_VERSION" = "1.11.1" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi;

vendor/github.com/bits-and-blooms/bitset/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 Will Fitzgerald. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/bits-and-blooms/bitset/README.md

@@ -0,0 +1,176 @@
+# bitset
+
+*Go language library to map between non-negative integers and boolean values*
+
+[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest)
+[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset)
+[![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc)
+
+
+This library is part of the [awesome go collection](https://github.com/avelino/awesome-go). It is used in production by several important systems:
+
+* [beego](https://github.com/beego/beego)
+* [CubeFS](https://github.com/cubefs/cubefs)
+* [Amazon EKS Distro](https://github.com/aws/eks-distro)
+* [sourcegraph](https://github.com/sourcegraph/sourcegraph-public-snapshot)
+* [torrent](https://github.com/anacrolix/torrent)
+
+
+## Description
+
+Package bitset implements bitsets, a mapping between non-negative integers and boolean values.
+It should be more efficient than map[uint] bool.
+
+It provides methods for setting, clearing, flipping, and testing individual integers.
+
+But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits.
+
+BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk automatically, but `Shrink` and `Compact` methods are available. On creation, a hint can be given for the number of bits that will be used.
+
+Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining.
+
+### Example use:
+
+```go
+package main
+
+import (
+	"fmt"
+	"math/rand"
+
+	"github.com/bits-and-blooms/bitset"
+)
+
+func main() {
+	fmt.Printf("Hello from BitSet!\n")
+	var b bitset.BitSet
+	// play some Go Fish
+	for i := 0; i < 100; i++ {
+		card1 := uint(rand.Intn(52))
+		card2 := uint(rand.Intn(52))
+		b.Set(card1)
+		if b.Test(card2) {
+			fmt.Println("Go Fish!")
+		}
+		b.Clear(card1)
+	}
+
+	// Chaining
+	b.Set(10).Set(11)
+
+	for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) {
+		fmt.Println("The following bit is set:", i)
+	}
+	if b.Intersection(bitset.New(100).Set(10)).Count() == 1 {
+		fmt.Println("Intersection works.")
+	} else {
+		fmt.Println("Intersection doesn't work???")
+	}
+}
+```
+
+If you have Go 1.23 or better, you can iterate over the set bits like so:
+
+```go
+for i := range b.EachSet() {}
+```
+
+
+
+Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc
+
+## Serialization
+
+
+You may serialize a bitset safely and portably to a stream
+of bytes as follows:
+```Go
+    const length = 9585
+	const oneEvery = 97
+	bs := bitset.New(length)
+	// Add some bits
+	for i := uint(0); i < length; i += oneEvery {
+		bs = bs.Set(i)
+	}
+
+	var buf bytes.Buffer
+	n, err := bs.WriteTo(&buf)
+	if err != nil {
+		// failure
+	}
+	// Here n == buf.Len()
+```
+You can later deserialize the result as follows:
+
+```Go
+	// Read back from buf
+	bs = bitset.New()
+	n, err = bs.ReadFrom(&buf)
+	if err != nil {
+		// error
+	}
+	// n is the number of bytes read
+```
+
+The `ReadFrom` function attempts to read the data into the existing
+BitSet instance, to minimize memory allocations.
+
+
+*Performance tip*: 
+When reading and writing to a file or a network connection, you may get better performance by 
+wrapping your streams with `bufio` instances.
+
+E.g., 
+```Go
+	f, err := os.Create("myfile")
+	w := bufio.NewWriter(f)
+```
+```Go
+	f, err := os.Open("myfile")
+	r := bufio.NewReader(f)
+```
+
+## Memory Usage
+
+The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](https://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring).
+
+The `roaring` library allows you to go back and forth between compressed Roaring bitmaps and the conventional bitset instances:
+```Go
+			mybitset := roaringbitmap.ToBitSet()
+			newroaringbitmap := roaring.FromBitSet(mybitset)
+```
+
+
+### Goroutine safety
+
+In general, it's not safe to access the same BitSet using different goroutines--they are unsynchronized for performance.
+
+Should you want to access a BitSet from more than one goroutine, you should provide synchronization. Typically this is done by using channels to pass the *BitSet around (in Go style; so there is only ever one owner), or by using `sync.Mutex` to serialize operations on BitSets.
+
+## Installation
+
+```bash
+go get github.com/bits-and-blooms/bitset
+```
+
+## Contributing
+
+If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)")
+
+## Running all tests
+
+Before committing the code, please check if it passes tests, has adequate coverage, etc.
+```bash
+go test
+go test -cover
+```
+
+## Stars
+
+
+[![Star History Chart](https://api.star-history.com/svg?repos=bits-and-blooms/bitset&type=Date)](https://www.star-history.com/#bits-and-blooms/bitset&Date)
+
+## Further reading
+
+<p>Mastering Programming: From Testing to Performance in Go</p>
+<div><a href="https://www.amazon.com/dp/B0FMPGSWR5"><img style="margin-left: auto; margin-right: auto;" src="https://m.media-amazon.com/images/I/61feneHS7kL._SL1499_.jpg" alt="" width="250px" /></a></div>

vendor/github.com/bits-and-blooms/bitset/SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+You can report privately a vulnerability by email at daniel@lemire.me (current maintainer).

vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml

@@ -0,0 +1,39 @@
+# Go
+# Build your Go project.
+# Add steps that test, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/go
+
+trigger:
+- master
+
+pool:
+  vmImage: 'Ubuntu-16.04'
+
+variables:
+  GOBIN:  '$(GOPATH)/bin' # Go binaries path
+  GOROOT: '/usr/local/go1.11' # Go installation path
+  GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path
+  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code
+
+steps:
+- script: |
+    mkdir -p '$(GOBIN)'
+    mkdir -p '$(GOPATH)/pkg'
+    mkdir -p '$(modulePath)'
+    shopt -s extglob
+    shopt -s dotglob
+    mv !(gopath) '$(modulePath)'
+    echo '##vso[task.prependpath]$(GOBIN)'
+    echo '##vso[task.prependpath]$(GOROOT)/bin'
+  displayName: 'Set up the Go workspace'
+
+- script: |
+    go version
+    go get -v -t -d ./...
+    if [ -f Gopkg.toml ]; then
+        curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+        dep ensure
+    fi
+    go build -v .
+  workingDirectory: '$(modulePath)'
+  displayName: 'Get dependencies, then build'

vendor/github.com/bits-and-blooms/bitset/bitset_iter.go

@@ -0,0 +1,23 @@
+//go:build go1.23
+// +build go1.23
+
+package bitset
+
+import (
+	"iter"
+	"math/bits"
+)
+
+func (b *BitSet) EachSet() iter.Seq[uint] {
+	return func(yield func(uint) bool) {
+		for wordIndex, word := range b.set {
+			idx := 0
+			for trail := bits.TrailingZeros64(word); trail != 64; trail = bits.TrailingZeros64(word >> idx) {
+				if !yield(uint(wordIndex<<log2WordSize + idx + trail)) {
+					return
+				}
+				idx += trail + 1
+			}
+		}
+	}
+}

vendor/github.com/bits-and-blooms/bitset/popcnt.go

@@ -0,0 +1,52 @@
+package bitset
+
+import "math/bits"
+
+func popcntSlice(s []uint64) (cnt uint64) {
+	for _, x := range s {
+		cnt += uint64(bits.OnesCount64(x))
+	}
+	return
+}
+
+func popcntMaskSlice(s, m []uint64) (cnt uint64) {
+	// The next line is to help the bounds checker, it matters!
+	_ = m[len(s)-1] // BCE
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] &^ m[i]))
+	}
+	return
+}
+
+// popcntAndSlice computes the population count of the AND of two slices.
+// It assumes that len(m) >= len(s) > 0.
+func popcntAndSlice(s, m []uint64) (cnt uint64) {
+	// The next line is to help the bounds checker, it matters!
+	_ = m[len(s)-1] // BCE
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] & m[i]))
+	}
+	return
+}
+
+// popcntOrSlice computes the population count of the OR of two slices.
+// It assumes that len(m) >= len(s) > 0.
+func popcntOrSlice(s, m []uint64) (cnt uint64) {
+	// The next line is to help the bounds checker, it matters!
+	_ = m[len(s)-1] // BCE
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] | m[i]))
+	}
+	return
+}
+
+// popcntXorSlice computes the population count of the XOR of two slices.
+// It assumes that len(m) >= len(s) > 0.
+func popcntXorSlice(s, m []uint64) (cnt uint64) {
+	// The next line is to help the bounds checker, it matters!
+	_ = m[len(s)-1] // BCE
+	for i := range s {
+		cnt += uint64(bits.OnesCount64(s[i] ^ m[i]))
+	}
+	return
+}

vendor/github.com/bits-and-blooms/bitset/select.go

@@ -0,0 +1,47 @@
+package bitset
+
+import "math/bits"
+
+func select64(w uint64, j uint) uint {
+	seen := 0
+	// Divide 64bit
+	part := w & 0xFFFFFFFF
+	n := uint(bits.OnesCount64(part))
+	if n <= j {
+		part = w >> 32
+		seen += 32
+		j -= n
+	}
+	ww := part
+
+	// Divide 32bit
+	part = ww & 0xFFFF
+
+	n = uint(bits.OnesCount64(part))
+	if n <= j {
+		part = ww >> 16
+		seen += 16
+		j -= n
+	}
+	ww = part
+
+	// Divide 16bit
+	part = ww & 0xFF
+	n = uint(bits.OnesCount64(part))
+	if n <= j {
+		part = ww >> 8
+		seen += 8
+		j -= n
+	}
+	ww = part
+
+	// Lookup in final byte
+	counter := 0
+	for ; counter < 8; counter++ {
+		j -= uint((ww >> counter) & 1)
+		if j+1 == 0 {
+			break
+		}
+	}
+	return uint(seen + counter)
+}

vendor/github.com/mschoch/smat/.gitignore

@@ -0,0 +1,14 @@
+#*
+*.sublime-*
+*~
+.#*
+.project
+.settings
+**/.idea/
+**/*.iml
+/examples/bolt/boltsmat-fuzz.zip
+/examples/bolt/workdir/
+.DS_Store
+coverage.out
+*.test
+tags

vendor/github.com/mschoch/smat/.travis.yml

@@ -0,0 +1,16 @@
+sudo: false
+language: go
+go:
+- 1.6
+script:
+- go get golang.org/x/tools/cmd/cover
+- go get github.com/mattn/goveralls
+- go get github.com/kisielk/errcheck
+- go test -v -race
+- go vet
+- errcheck ./...
+- go test -coverprofile=profile.out -covermode=count
+- goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS
+notifications:
+  email:
+  - marty.schoch@gmail.com

vendor/github.com/mschoch/smat/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/mschoch/smat/README.md

@@ -0,0 +1,166 @@
+# smat – State Machine Assisted Testing
+
+The concept is simple, describe valid uses of your library as states and actions.  States describe which actions are possible, and with what probability they should occur.  Actions mutate the context and transition to another state.
+
+By doing this, two things are possible:
+
+1.  Use [go-fuzz](https://github.com/dvyukov/go-fuzz) to find/test interesting sequences of operations on your library.
+
+2.  Automate longevity testing of your application by performing long sequences of valid operations.
+
+**NOTE**: both of these can also incorporate validation logic (not just failure detection by building validation into the state machine)
+
+## Status
+
+The API is still not stable.  This is brand new and we'll probably change things we don't like...
+
+[![Build Status](https://travis-ci.org/mschoch/smat.svg?branch=master)](https://travis-ci.org/mschoch/smat)
+[![Coverage Status](https://coveralls.io/repos/github/mschoch/smat/badge.svg?branch=master)](https://coveralls.io/github/mschoch/smat?branch=master)
+[![GoDoc](https://godoc.org/github.com/mschoch/smat?status.svg)](https://godoc.org/github.com/mschoch/smat)
+[![codebeat badge](https://codebeat.co/badges/c3ff6180-a241-4128-97f0-fa6bf6f48752)](https://codebeat.co/projects/github-com-mschoch-smat)
+[![Go Report Card](https://goreportcard.com/badge/github.com/mschoch/smat)](https://goreportcard.com/report/github.com/mschoch/smat)
+
+## License
+
+Apache 2.0
+
+## How do I use it?
+
+### smat.Context
+
+Choose a structure to keep track of any state.  You pass in an instance of this when you start, and it will be passed to every action when it executes.  The actions may mutate this context.
+
+For example, consider a database library, once you open a database handle, you need to use it inside of the other actions.  So you might use a structure like:
+
+```
+type context struct {
+  db *DB
+}
+```
+
+### smat.State
+
+A state represents a state that your application/library can be in, and the probabilities thats certain actions should be taken.
+
+For example, consider a database library, in a state where the database is open, there many things you can do.  Let's consider just two right now, you can set a value, or you can delete a value.
+
+```
+func dbOpen(next byte) smat.ActionID {
+	return smat.PercentExecute(next,
+		smat.PercentAction{50, setValue},
+		smat.PercentAction{50, deleteValue},
+	)
+}
+```
+
+This says that in the open state, there are two valid actions, 50% of the time you should set a value and 50% of the time you should delete a value.  **NOTE**: these percentages are just for characterizing the test workload.
+
+### smat.Action
+
+Actions are functions that do some work, optionally mutate the context, and indicate the next state to transition to.  Below we see an example action to set value in a database.
+
+```
+func setValueFunc(ctx smat.Context) (next smat.State, err error) {
+  // type assert to our custom context type
+	context := ctx.(*context)
+  // perform the operation
+  err = context.db.Set("k", "v")
+  if err != nil {
+    return nil, err
+  }
+  // return the new state
+  return dbOpen, nil
+}
+```
+
+### smat.ActionID and smat.ActionMap
+
+Actions are just functions, and since we can't compare functions in Go, we need to introduce an external identifier for them.  This allows us to build a bi-directional mapping which we'll take advantage of later.
+
+```
+const (
+  setup smat.ActionID = iota
+  teardown
+  setValue
+  deleteValue
+)
+
+var actionMap = smat.ActionMap{
+  setup:       setupFunc,
+  teardown:    teardownFunc,
+	setValue:    setValueFunc,
+	deleteValue: deleteValueFunc,
+}
+```
+
+### smat.ActionSeq
+
+A common way that many users think about a library is as a sequence of actions to be performed.  Using the ActionID's that we've already seen we can build up sequences of operations.
+
+```
+  actionSeq := smat.ActionSeq{
+		open,
+		setValue,
+		setValue,
+		setValue,
+	}
+```
+
+Notice that we build these actions using the constants we defined above, and because of this we can have a bi-directional mapping between a stream of bytes (driving the state machine) and a sequence of actions to be performed.
+
+## Fuzzing
+
+We've built a lot of pieces, lets wire it up to go-fuzz.
+
+```
+func Fuzz(data []byte) int {
+	return smat.Fuzz(&context{}, setup, teardown, actionMap, data)
+}
+```
+
+* The first argument is an instance of context structure.
+* The second argument is the ActionID of our setup function.  The setup function does not consume any of the input stream and is used to initialize the context and determine the start state.
+* The third argument is the teardown function.  This will be called unconditionally to clean up any resources associated with the test.
+* The fourth argument is the actionMap which maps all ActionIDs to Actions.
+* The fifth argument is the data passed in from the go-fuzz application.
+
+### Generating Initial go-fuzz Corpus
+
+Earlier we mentioned the bi-directional mapping between Actions and the byte stream driving the state machine.  We can now leverage this to build the inital go-fuzz corpus.
+
+Using the `ActinSeq`s we learned about earlier we can build up a list of them as:
+
+    var actionSeqs = []smat.ActionSeq{...}
+
+Then, we can write them out to disk using:
+
+```
+for i, actionSeq := range actionSeqs {
+  byteSequence, err := actionSeq.ByteEncoding(&context{}, setup, teardown, actionMap)
+  if err != nil {
+    // handle error
+  }
+  os.MkdirAll("workdir/corpus", 0700)
+  ioutil.WriteFile(fmt.Sprintf("workdir/corpus/%d", i), byteSequence, 0600)
+}
+```
+
+You can then either put this into a test case or a main application depending on your needs.
+
+## Longevity Testing
+
+Fuzzing is great, but most of your corpus is likely to be shorter meaningful sequences.  And go-fuzz works to find shortest sequences that cause problems, but sometimes you actually want to explore longer sequences that appear to go-fuzz as not triggering additional code coverage.
+
+For these cases we have another helper you can use:
+
+```
+  Longevity(ctx, setup, teardown, actionMap, 0, closeChan)
+```
+
+The first four arguments are the same, the last two are:
+* random seed used to ensure repeatable tests
+* closeChan (chan struct{}) - close this channel if you want the function to stop and return ErrClosed, otherwise it will run forever
+
+## Examples
+
+See the examples directory for a working example that tests some BoltDB functionality.

vendor/github.com/mschoch/smat/actionseq.go

@@ -0,0 +1,61 @@
+//  Copyright (c) 2016 Marty Schoch
+
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+package smat
+
+// ActionSeq represents a sequence of actions, used for populating a corpus
+// of byte sequences for the corresponding fuzz tests
+type ActionSeq []ActionID
+
+// ByteEncoding runs the FSM to produce a byte sequence to trigger the
+// desired action
+func (a ActionSeq) ByteEncoding(ctx Context, setup, teardown ActionID, actionMap ActionMap) ([]byte, error) {
+	setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown)
+	if err != nil {
+		return nil, err
+	}
+	state, err := setupFunc(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_, _ = teardownFunc(ctx)
+	}()
+
+	var rv []byte
+	for _, actionID := range a {
+		b, err := probeStateForAction(state, actionID)
+		if err != nil {
+			return nil, err
+		}
+		rv = append(rv, b)
+		action, ok := actionMap[actionID]
+		if !ok {
+			continue
+		}
+		state, err = action(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return rv, nil
+}
+
+func probeStateForAction(state State, actionID ActionID) (byte, error) {
+	for i := 0; i < 256; i++ {
+		nextActionID := state(byte(i))
+		if nextActionID == actionID {
+			return byte(i), nil
+		}
+	}
+	return 0, ErrActionNotPossible
+}

vendor/github.com/mschoch/smat/smat.go

@@ -0,0 +1,161 @@
+//  Copyright (c) 2016 Marty Schoch
+
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+package smat
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+)
+
+// Logger is a configurable logger used by this package
+// by default output is discarded
+var Logger = log.New(ioutil.Discard, "smat ", log.LstdFlags)
+
+// Context is a container for any user state
+type Context interface{}
+
+// State is a function which describes which action to perform in the event
+// that a particular byte is seen
+type State func(next byte) ActionID
+
+// PercentAction describes the frequency with which an action should occur
+// for example: Action{Percent:10, Action:DonateMoney} means that 10% of
+// the time you should donate money.
+type PercentAction struct {
+	Percent int
+	Action  ActionID
+}
+
+// Action is any function which returns the next state to transition to
+// it can optionally mutate the provided context object
+// if any error occurs, it may return an error which will abort execution
+type Action func(Context) (State, error)
+
+// ActionID is a unique identifier for an action
+type ActionID int
+
+// NopAction does nothing and simply continues to the next input
+var NopAction ActionID = -1
+
+// ActionMap is a mapping form ActionID to Action
+type ActionMap map[ActionID]Action
+
+func (a ActionMap) findSetupTeardown(setup, teardown ActionID) (Action, Action, error) {
+	setupFunc, ok := a[setup]
+	if !ok {
+		return nil, nil, ErrSetupMissing
+	}
+	teardownFunc, ok := a[teardown]
+	if !ok {
+		return nil, nil, ErrTeardownMissing
+	}
+	return setupFunc, teardownFunc, nil
+}
+
+// Fuzz runs the fuzzing state machine with the provided context
+// first, the setup action is executed unconditionally
+// the start state is determined by this action
+// actionMap is a lookup table for all actions
+// the data byte slice determines all future state transitions
+// finally, the teardown action is executed unconditionally for cleanup
+func Fuzz(ctx Context, setup, teardown ActionID, actionMap ActionMap, data []byte) int {
+	reader := bytes.NewReader(data)
+	err := runReader(ctx, setup, teardown, actionMap, reader, nil)
+	if err != nil {
+		panic(err)
+	}
+	return 1
+}
+
+// Longevity runs the state machine with the provided context
+// first, the setup action is executed unconditionally
+// the start state is determined by this action
+// actionMap is a lookup table for all actions
+// random bytes are generated to determine all future state transitions
+// finally, the teardown action is executed unconditionally for cleanup
+func Longevity(ctx Context, setup, teardown ActionID, actionMap ActionMap, seed int64, closeChan chan struct{}) error {
+	source := rand.NewSource(seed)
+	return runReader(ctx, setup, teardown, actionMap, rand.New(source), closeChan)
+}
+
+var (
+	// ErrSetupMissing is returned when the setup action cannot be found
+	ErrSetupMissing = fmt.Errorf("setup action missing")
+	// ErrTeardownMissing is returned when the teardown action cannot be found
+	ErrTeardownMissing = fmt.Errorf("teardown action missing")
+	// ErrClosed is returned when the closeChan was closed to cancel the op
+	ErrClosed = fmt.Errorf("closed")
+	// ErrActionNotPossible is returned when an action is encountered in a
+	// FuzzCase that is not possible in the current state
+	ErrActionNotPossible = fmt.Errorf("action not possible in state")
+)
+
+func runReader(ctx Context, setup, teardown ActionID, actionMap ActionMap, r io.Reader, closeChan chan struct{}) error {
+	setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown)
+	if err != nil {
+		return err
+	}
+	Logger.Printf("invoking setup action")
+	state, err := setupFunc(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		Logger.Printf("invoking teardown action")
+		_, _ = teardownFunc(ctx)
+	}()
+
+	reader := bufio.NewReader(r)
+	for next, err := reader.ReadByte(); err == nil; next, err = reader.ReadByte() {
+		select {
+		case <-closeChan:
+			return ErrClosed
+		default:
+			actionID := state(next)
+			action, ok := actionMap[actionID]
+			if !ok {
+				Logger.Printf("no such action defined, continuing")
+				continue
+			}
+			Logger.Printf("invoking action - %d", actionID)
+			state, err = action(ctx)
+			if err != nil {
+				Logger.Printf("it was action %d that returned err %v", actionID, err)
+				return err
+			}
+		}
+	}
+	return err
+}
+
+// PercentExecute interprets the next byte as a random value and normalizes it
+// to values 0-99, it then looks to see which action should be execued based
+// on the action distributions
+func PercentExecute(next byte, pas ...PercentAction) ActionID {
+	percent := int(99 * int(next) / 255)
+
+	sofar := 0
+	for _, pa := range pas {
+		sofar = sofar + pa.Percent
+		if percent < sofar {
+			return pa.Action
+		}
+
+	}
+	return NopAction
+}

vendor/modules.txt

@@ -133,6 +133,11 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric
 # github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0
 ## explicit; go 1.25.0
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping
+# github.com/RoaringBitmap/roaring/v2 v2.25.0
+## explicit; go 1.24.0
+github.com/RoaringBitmap/roaring/v2
+github.com/RoaringBitmap/roaring/v2/internal
+github.com/RoaringBitmap/roaring/v2/roaring64
 # github.com/VictoriaMetrics/VictoriaLogs v1.51.1-0.20260624061259-dc94972a8708
 ## explicit; go 1.26.4
 github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage
@@ -298,6 +303,9 @@ github.com/bboreham/go-loser
 # github.com/beorn7/perks v1.0.1
 ## explicit; go 1.11
 github.com/beorn7/perks/quantile
+# github.com/bits-and-blooms/bitset v1.24.4
+## explicit; go 1.16
+github.com/bits-and-blooms/bitset
 # github.com/bmatcuk/doublestar/v4 v4.10.0
 ## explicit; go 1.16
 github.com/bmatcuk/doublestar/v4
@@ -547,6 +555,9 @@ github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee
 ## explicit; go 1.12
 github.com/modern-go/reflect2
+# github.com/mschoch/smat v0.2.0
+## explicit; go 1.13
+github.com/mschoch/smat
 # github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 ## explicit
 github.com/munnerz/goautoneg