Vendor dependencies

This commit is contained in:
2026-08-01 16:11:49 +03:00
parent 7f139a0241
commit 6b5e7f0f8b
29706 changed files with 9575646 additions and 0 deletions
+174
View File
@@ -0,0 +1,174 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::{Criterion, Throughput};
use rand::distr::{Distribution, StandardUniform};
extern crate arrow;
use arrow::compute::kernels::aggregate::*;
use arrow::util::bench_util::*;
use arrow::{array::*, datatypes::Float32Type};
use arrow_array::types::{Float64Type, Int8Type, Int16Type, Int32Type, Int64Type};
const BATCH_SIZE: usize = 64 * 1024;
fn primitive_benchmark<T: ArrowNumericType>(c: &mut Criterion, name: &str)
where
StandardUniform: Distribution<T::Native>,
{
let nonnull_array = create_primitive_array::<T>(BATCH_SIZE, 0.0);
let nullable_array = create_primitive_array::<T>(BATCH_SIZE, 0.5);
c.benchmark_group(name)
.throughput(Throughput::Bytes(
(std::mem::size_of::<T::Native>() * BATCH_SIZE) as u64,
))
.bench_function("sum nonnull", |b| b.iter(|| sum(&nonnull_array)))
.bench_function("min nonnull", |b| b.iter(|| min(&nonnull_array)))
.bench_function("max nonnull", |b| b.iter(|| max(&nonnull_array)))
.bench_function("sum nullable", |b| b.iter(|| sum(&nullable_array)))
.bench_function("min nullable", |b| b.iter(|| min(&nullable_array)))
.bench_function("max nullable", |b| b.iter(|| max(&nullable_array)));
}
fn add_benchmark(c: &mut Criterion) {
primitive_benchmark::<Float32Type>(c, "float32");
primitive_benchmark::<Float64Type>(c, "float64");
primitive_benchmark::<Int8Type>(c, "int8");
primitive_benchmark::<Int16Type>(c, "int16");
primitive_benchmark::<Int32Type>(c, "int32");
primitive_benchmark::<Int64Type>(c, "int64");
{
let nonnull_strings = create_string_array_with_len::<i32>(BATCH_SIZE, 0.0, 16);
let nullable_strings = create_string_array_with_len::<i32>(BATCH_SIZE, 0.5, 16);
c.benchmark_group("string")
.throughput(Throughput::Elements(BATCH_SIZE as u64))
.bench_function("min nonnull", |b| b.iter(|| min_string(&nonnull_strings)))
.bench_function("max nonnull", |b| b.iter(|| max_string(&nonnull_strings)))
.bench_function("min nullable", |b| b.iter(|| min_string(&nullable_strings)))
.bench_function("max nullable", |b| b.iter(|| max_string(&nullable_strings)));
}
{
let nonnull_strings = create_string_view_array_with_len(BATCH_SIZE, 0.0, 16, false);
let nullable_strings = create_string_view_array_with_len(BATCH_SIZE, 0.5, 16, false);
c.benchmark_group("string view")
.throughput(Throughput::Elements(BATCH_SIZE as u64))
.bench_function("min nonnull", |b| {
b.iter(|| min_string_view(&nonnull_strings))
})
.bench_function("max nonnull", |b| {
b.iter(|| max_string_view(&nonnull_strings))
})
.bench_function("min nullable", |b| {
b.iter(|| min_string_view(&nullable_strings))
})
.bench_function("max nullable", |b| {
b.iter(|| max_string_view(&nullable_strings))
});
}
{
let nonnull_bools_mixed = create_boolean_array(BATCH_SIZE, 0.0, 0.5);
let nonnull_bools_all_false = create_boolean_array(BATCH_SIZE, 0.0, 0.0);
let nonnull_bools_all_true = create_boolean_array(BATCH_SIZE, 0.0, 1.0);
let nullable_bool_mixed = create_boolean_array(BATCH_SIZE, 0.5, 0.5);
let nullable_bool_all_false = create_boolean_array(BATCH_SIZE, 0.5, 0.0);
let nullable_bool_all_true = create_boolean_array(BATCH_SIZE, 0.5, 1.0);
c.benchmark_group("bool")
.throughput(Throughput::Elements(BATCH_SIZE as u64))
.bench_function("min nonnull mixed", |b| {
b.iter(|| min_boolean(&nonnull_bools_mixed))
})
.bench_function("max nonnull mixed", |b| {
b.iter(|| max_boolean(&nonnull_bools_mixed))
})
.bench_function("or nonnull mixed", |b| {
b.iter(|| bool_or(&nonnull_bools_mixed))
})
.bench_function("and nonnull mixed", |b| {
b.iter(|| bool_and(&nonnull_bools_mixed))
})
.bench_function("min nonnull false", |b| {
b.iter(|| min_boolean(&nonnull_bools_all_false))
})
.bench_function("max nonnull false", |b| {
b.iter(|| max_boolean(&nonnull_bools_all_false))
})
.bench_function("or nonnull false", |b| {
b.iter(|| bool_or(&nonnull_bools_all_false))
})
.bench_function("and nonnull false", |b| {
b.iter(|| bool_and(&nonnull_bools_all_false))
})
.bench_function("min nonnull true", |b| {
b.iter(|| min_boolean(&nonnull_bools_all_true))
})
.bench_function("max nonnull true", |b| {
b.iter(|| max_boolean(&nonnull_bools_all_true))
})
.bench_function("or nonnull true", |b| {
b.iter(|| bool_or(&nonnull_bools_all_true))
})
.bench_function("and nonnull true", |b| {
b.iter(|| bool_and(&nonnull_bools_all_true))
})
.bench_function("min nullable mixed", |b| {
b.iter(|| min_boolean(&nullable_bool_mixed))
})
.bench_function("max nullable mixed", |b| {
b.iter(|| max_boolean(&nullable_bool_mixed))
})
.bench_function("or nullable mixed", |b| {
b.iter(|| bool_or(&nullable_bool_mixed))
})
.bench_function("and nullable mixed", |b| {
b.iter(|| bool_and(&nullable_bool_mixed))
})
.bench_function("min nullable false", |b| {
b.iter(|| min_boolean(&nullable_bool_all_false))
})
.bench_function("max nullable false", |b| {
b.iter(|| max_boolean(&nullable_bool_all_false))
})
.bench_function("or nullable false", |b| {
b.iter(|| bool_or(&nullable_bool_all_false))
})
.bench_function("and nullable false", |b| {
b.iter(|| bool_and(&nullable_bool_all_false))
})
.bench_function("min nullable true", |b| {
b.iter(|| min_boolean(&nullable_bool_all_true))
})
.bench_function("max nullable true", |b| {
b.iter(|| max_boolean(&nullable_bool_all_true))
})
.bench_function("or nullable true", |b| {
b.iter(|| bool_or(&nullable_bool_all_true))
})
.bench_function("and nullable true", |b| {
b.iter(|| bool_and(&nullable_bool_all_true))
});
}
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
@@ -0,0 +1,79 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use criterion::*;
extern crate arrow;
use arrow::compute::kernels::numeric::*;
use arrow::datatypes::Float32Type;
use arrow::util::bench_util::*;
use arrow_array::Scalar;
use std::hint;
fn add_benchmark(c: &mut Criterion) {
const BATCH_SIZE: usize = 64 * 1024;
for null_density in [0., 0.1, 0.5, 0.9, 1.0] {
let arr_a = create_primitive_array::<Float32Type>(BATCH_SIZE, null_density);
let arr_b = create_primitive_array::<Float32Type>(BATCH_SIZE, null_density);
let scalar_a = create_primitive_array::<Float32Type>(1, 0.);
let scalar = Scalar::new(&scalar_a);
c.bench_function(&format!("add({null_density})"), |b| {
b.iter(|| hint::black_box(add_wrapping(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("add_checked({null_density})"), |b| {
b.iter(|| hint::black_box(add(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("add_scalar({null_density})"), |b| {
b.iter(|| hint::black_box(add_wrapping(&arr_a, &scalar).unwrap()))
});
c.bench_function(&format!("subtract({null_density})"), |b| {
b.iter(|| hint::black_box(sub_wrapping(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("subtract_checked({null_density})"), |b| {
b.iter(|| hint::black_box(sub(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("subtract_scalar({null_density})"), |b| {
b.iter(|| hint::black_box(sub_wrapping(&arr_a, &scalar).unwrap()))
});
c.bench_function(&format!("multiply({null_density})"), |b| {
b.iter(|| hint::black_box(mul_wrapping(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("multiply_checked({null_density})"), |b| {
b.iter(|| hint::black_box(mul(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("multiply_scalar({null_density})"), |b| {
b.iter(|| hint::black_box(mul_wrapping(&arr_a, &scalar).unwrap()))
});
c.bench_function(&format!("divide({null_density})"), |b| {
b.iter(|| hint::black_box(div(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("divide_scalar({null_density})"), |b| {
b.iter(|| hint::black_box(div(&arr_a, &scalar).unwrap()))
});
c.bench_function(&format!("modulo({null_density})"), |b| {
b.iter(|| hint::black_box(rem(&arr_a, &arr_b).unwrap()))
});
c.bench_function(&format!("modulo_scalar({null_density})"), |b| {
b.iter(|| hint::black_box(rem(&arr_a, &scalar).unwrap()))
});
}
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
@@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::{array::*, buffer::Buffer, datatypes::DataType};
fn create_binary_array_data(length: i32) -> ArrayData {
let value_buffer = Buffer::from_iter(0_i32..length);
let offsets_buffer = Buffer::from_iter(0_i32..length + 1);
ArrayData::try_new(
DataType::Binary,
length as usize,
None,
0,
vec![offsets_buffer, value_buffer],
vec![],
)
.unwrap()
}
fn validate_utf8_array(arr: &ArrayData) {
arr.validate_values().unwrap();
}
fn validate_benchmark(c: &mut Criterion) {
//Binary Array
c.bench_function("validate_binary_array_data 20000", |b| {
b.iter(|| create_binary_array_data(20000))
});
//Utf8 Array
let str_arr = StringArray::from(vec!["test"; 20000]).to_data();
c.bench_function("validate_utf8_array_data 20000", |b| {
b.iter(|| validate_utf8_array(&str_arr))
});
let byte_array = BinaryArray::from_iter_values(std::iter::repeat_n(b"test", 20000));
c.bench_function("byte_array_to_string_array 20000", |b| {
b.iter(|| StringArray::from(BinaryArray::from(byte_array.to_data())))
});
}
criterion_group!(benches, validate_benchmark);
criterion_main!(benches);
+253
View File
@@ -0,0 +1,253 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use arrow::array::*;
use arrow_buffer::i256;
use rand::Rng;
use std::iter::repeat_n;
use std::{hint, sync::Arc};
fn array_from_vec(n: usize) {
let v: Vec<i32> = (0..n as i32).collect();
hint::black_box(Int32Array::from(v));
}
fn array_string_from_vec(n: usize) {
let mut v: Vec<Option<&str>> = Vec::with_capacity(n);
for i in 0..n {
if i % 2 == 0 {
v.push(Some("hello world"));
} else {
v.push(None);
}
}
hint::black_box(StringArray::from(v));
}
fn struct_array_values(
n: usize,
) -> (
&'static str,
Vec<Option<&'static str>>,
&'static str,
Vec<Option<i32>>,
) {
let mut strings: Vec<Option<&str>> = Vec::with_capacity(n);
let mut ints: Vec<Option<i32>> = Vec::with_capacity(n);
for _ in 0..n / 4 {
strings.extend_from_slice(&[Some("joe"), None, None, Some("mark")]);
ints.extend_from_slice(&[Some(1), Some(2), None, Some(4)]);
}
("f1", strings, "f2", ints)
}
fn struct_array_from_vec(
field1: &str,
strings: &[Option<&str>],
field2: &str,
ints: &[Option<i32>],
) {
let strings: ArrayRef = Arc::new(StringArray::from(strings.to_owned()));
let ints: ArrayRef = Arc::new(Int32Array::from(ints.to_owned()));
hint::black_box(StructArray::try_from(vec![(field1, strings), (field2, ints)]).unwrap());
}
fn decimal32_array_from_vec(array: &[Option<i32>]) {
hint::black_box(
array
.iter()
.copied()
.collect::<Decimal32Array>()
.with_precision_and_scale(9, 2)
.unwrap(),
);
}
fn decimal64_array_from_vec(array: &[Option<i64>]) {
hint::black_box(
array
.iter()
.copied()
.collect::<Decimal64Array>()
.with_precision_and_scale(17, 2)
.unwrap(),
);
}
fn decimal128_array_from_vec(array: &[Option<i128>]) {
hint::black_box(
array
.iter()
.copied()
.collect::<Decimal128Array>()
.with_precision_and_scale(34, 2)
.unwrap(),
);
}
fn decimal256_array_from_vec(array: &[Option<i256>]) {
hint::black_box(
array
.iter()
.copied()
.collect::<Decimal256Array>()
.with_precision_and_scale(70, 2)
.unwrap(),
);
}
fn array_from_vec_decimal_benchmark(c: &mut Criterion) {
// bench decimal32 array
// create option<i32> array
let size: usize = 1 << 15;
let mut rng = rand::rng();
let mut array = vec![];
for _ in 0..size {
array.push(Some(rng.random_range::<i32, _>(0..99999999)));
}
c.bench_function("decimal32_array_from_vec 32768", |b| {
b.iter(|| decimal32_array_from_vec(array.as_slice()))
});
// bench decimal64 array
// create option<i64> array
let size: usize = 1 << 15;
let mut rng = rand::rng();
let mut array = vec![];
for _ in 0..size {
array.push(Some(rng.random_range::<i64, _>(0..9999999999)));
}
c.bench_function("decimal64_array_from_vec 32768", |b| {
b.iter(|| decimal64_array_from_vec(array.as_slice()))
});
// bench decimal128 array
// create option<i128> array
let size: usize = 1 << 15;
let mut rng = rand::rng();
let mut array = vec![];
for _ in 0..size {
array.push(Some(rng.random_range::<i128, _>(0..9999999999)));
}
c.bench_function("decimal128_array_from_vec 32768", |b| {
b.iter(|| decimal128_array_from_vec(array.as_slice()))
});
// bench decimal256array
// create option<into<decimal256>> array
let size = 1 << 10;
let mut array = vec![];
let mut rng = rand::rng();
for _ in 0..size {
let decimal = i256::from_i128(rng.random_range::<i128, _>(0..9999999999999));
array.push(Some(decimal));
}
// bench decimal256 array
c.bench_function("decimal256_array_from_vec 32768", |b| {
b.iter(|| decimal256_array_from_vec(array.as_slice()))
});
}
fn array_from_vec_benchmark(c: &mut Criterion) {
c.bench_function("array_from_vec 128", |b| b.iter(|| array_from_vec(128)));
c.bench_function("array_from_vec 256", |b| b.iter(|| array_from_vec(256)));
c.bench_function("array_from_vec 512", |b| b.iter(|| array_from_vec(512)));
c.bench_function("array_string_from_vec 128", |b| {
b.iter(|| array_string_from_vec(128))
});
c.bench_function("array_string_from_vec 256", |b| {
b.iter(|| array_string_from_vec(256))
});
c.bench_function("array_string_from_vec 512", |b| {
b.iter(|| array_string_from_vec(512))
});
let (field1, strings, field2, ints) = struct_array_values(128);
c.bench_function("struct_array_from_vec 128", |b| {
b.iter(|| struct_array_from_vec(field1, &strings, field2, &ints))
});
let (field1, strings, field2, ints) = struct_array_values(256);
c.bench_function("struct_array_from_vec 256", |b| {
b.iter(|| struct_array_from_vec(field1, &strings, field2, &ints))
});
let (field1, strings, field2, ints) = struct_array_values(512);
c.bench_function("struct_array_from_vec 512", |b| {
b.iter(|| struct_array_from_vec(field1, &strings, field2, &ints))
});
let (field1, strings, field2, ints) = struct_array_values(1024);
c.bench_function("struct_array_from_vec 1024", |b| {
b.iter(|| struct_array_from_vec(field1, &strings, field2, &ints))
});
}
fn gen_option_vector<TItem: Copy>(item: TItem, len: usize) -> Vec<Option<TItem>> {
hint::black_box(
repeat_n(item, len)
.enumerate()
.map(|(idx, item)| if idx % 3 == 0 { None } else { Some(item) })
.collect(),
)
}
fn from_iter_benchmark(c: &mut Criterion) {
const ITER_LEN: usize = 16_384;
// All ArrowPrimitiveType use the same implementation
c.bench_function("Int64Array::from_iter", |b| {
let values = gen_option_vector(1, ITER_LEN);
b.iter(|| hint::black_box(Int64Array::from_iter(values.iter())));
});
c.bench_function("Int64Array::from_trusted_len_iter", |b| {
let values = gen_option_vector(1, ITER_LEN);
b.iter(|| unsafe {
// SAFETY: values.iter() is a TrustedLenIterator
hint::black_box(Int64Array::from_trusted_len_iter(values.iter()))
});
});
c.bench_function("BooleanArray::from_iter", |b| {
let values = gen_option_vector(true, ITER_LEN);
b.iter(|| hint::black_box(BooleanArray::from_iter(values.iter())));
});
c.bench_function("BooleanArray::from_trusted_len_iter", |b| {
let values = gen_option_vector(true, ITER_LEN);
b.iter(|| unsafe {
// SAFETY: values.iter() is a TrustedLenIterator
hint::black_box(BooleanArray::from_trusted_len_iter(values.iter()))
});
});
}
criterion_group!(
benches,
array_from_vec_benchmark,
array_from_vec_decimal_benchmark,
from_iter_benchmark
);
criterion_main!(benches);
+305
View File
@@ -0,0 +1,305 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
#[macro_use]
extern crate criterion;
use criterion::{Criterion, Throughput};
use std::hint;
use arrow::array::*;
use arrow::util::bench_util::*;
use arrow_array::types::{Int8Type, Int16Type, Int32Type, Int64Type};
const BATCH_SIZE: usize = 64 * 1024;
/// Run [`ArrayIter::fold`] while using black_box on each item and the result of the cb to prevent compiler optimizations.
fn fold_black_box_item_and_cb_res<ArrayAcc, F, B>(array: ArrayAcc, init: B, mut f: F)
where
ArrayAcc: ArrayAccessor,
F: FnMut(B, Option<ArrayAcc::Item>) -> B,
{
let result = ArrayIter::new(array).fold(hint::black_box(init), |acc, item| {
let res = f(acc, hint::black_box(item));
hint::black_box(res)
});
hint::black_box(result);
}
/// Run [`ArrayIter::fold`] while using black_box on each item to prevent compiler optimizations.
fn fold_black_box_item<ArrayAcc, F, B>(array: ArrayAcc, init: B, mut f: F)
where
ArrayAcc: ArrayAccessor,
F: FnMut(B, Option<ArrayAcc::Item>) -> B,
{
let result = ArrayIter::new(array).fold(hint::black_box(init), |acc, item| {
f(acc, hint::black_box(item))
});
hint::black_box(result);
}
/// Run [`ArrayIter::fold`] without using black_box on each item, but only on the result
/// to see if the compiler can do more optimizations.
fn fold_black_box_result<ArrayAcc, F, B>(array: ArrayAcc, init: B, f: F)
where
ArrayAcc: ArrayAccessor,
F: FnMut(B, Option<ArrayAcc::Item>) -> B,
{
let result = ArrayIter::new(array).fold(hint::black_box(init), f);
hint::black_box(result);
}
/// Run [`ArrayIter::any`] while using black_box on each item and the predicate return value to prevent compiler optimizations.
fn any_black_box_item_and_predicate<ArrayAcc>(
array: ArrayAcc,
mut any_predicate: impl FnMut(Option<ArrayAcc::Item>) -> bool,
) where
ArrayAcc: ArrayAccessor,
{
let any_res = ArrayIter::new(array).any(|item| {
let item = hint::black_box(item);
let res = any_predicate(item);
hint::black_box(res)
});
hint::black_box(any_res);
}
/// Run [`ArrayIter::any`] without using black_box in the loop, but only on the result
/// to see if the compiler can do more optimizations.
fn any_black_box_result<ArrayAcc>(
array: ArrayAcc,
any_predicate: impl FnMut(Option<ArrayAcc::Item>) -> bool,
) where
ArrayAcc: ArrayAccessor,
{
let any_res = ArrayIter::new(array).any(any_predicate);
hint::black_box(any_res);
}
/// Benchmark [`ArrayIter`] functions,
///
/// The passed `predicate_that_will_always_evaluate_to_false` function should be a predicate
/// that always returns `false` to ensure that the full array is always iterated over.
///
/// The predicate function should:
/// 1. always return false
/// 2. be impossible for the compiler to optimize away
/// 3. not use `hint::black_box` internally (unless impossible) to allow for more compiler optimizations
///
/// the way to achieve this is to make the predicate check for a value that is not presented in the array.
///
/// The reason for these requirements is that we want to iterate over the entire array while
/// letting the compiler have room for optimizations so it will be more representative of real world usage.
fn benchmark_array_iter<ArrayAcc, FoldFn, FoldInit>(
c: &mut Criterion,
name: &str,
nonnull_array: ArrayAcc,
nullable_array: ArrayAcc,
fold_init: FoldInit,
fold_fn: FoldFn,
predicate_that_will_always_evaluate_to_false: impl Fn(Option<ArrayAcc::Item>) -> bool,
) where
ArrayAcc: ArrayAccessor + Copy,
FoldInit: Copy,
FoldFn: Fn(FoldInit, Option<ArrayAcc::Item>) -> FoldInit,
{
let predicate_that_will_always_evaluate_to_false =
&predicate_that_will_always_evaluate_to_false;
let fold_fn = &fold_fn;
// Assert always false return false
{
let found = ArrayIter::new(nonnull_array).any(predicate_that_will_always_evaluate_to_false);
assert!(!found, "The predicate must always evaluate to false");
}
{
let found =
ArrayIter::new(nullable_array).any(predicate_that_will_always_evaluate_to_false);
assert!(!found, "The predicate must always evaluate to false");
}
c.benchmark_group(name)
.throughput(Throughput::Elements(BATCH_SIZE as u64))
// Most of the Rust default iterator functions are implemented on top of 2 functions:
// `fold` and `try_fold`
// so we are benchmarking `fold` first
.bench_function("nonnull fold black box item and fold result", |b| {
b.iter(|| fold_black_box_item_and_cb_res(nonnull_array, fold_init, fold_fn))
})
.bench_function("nonnull fold black box item", |b| {
b.iter(|| fold_black_box_item(nonnull_array, fold_init, fold_fn))
})
.bench_function("nonnull fold black box only result", |b| {
b.iter(|| fold_black_box_result(nonnull_array, fold_init, fold_fn))
})
.bench_function("null fold black box item and fold result", |b| {
b.iter(|| fold_black_box_item_and_cb_res(nullable_array, fold_init, fold_fn))
})
.bench_function("null fold black box item", |b| {
b.iter(|| fold_black_box_item(nullable_array, fold_init, fold_fn))
})
.bench_function("null fold black box only result", |b| {
b.iter(|| fold_black_box_result(nullable_array, fold_init, fold_fn))
})
// Due to `try_fold` not being available in stable Rust,
// we are benchmarking `any` instead which the default Rust implementation
// uses `try_fold` under the hood.
.bench_function("nonnull any black box item and predicate", |b| {
b.iter(|| {
any_black_box_item_and_predicate(
nonnull_array,
predicate_that_will_always_evaluate_to_false,
)
})
})
.bench_function("nonnull any black box only result", |b| {
b.iter(|| {
any_black_box_result(nonnull_array, predicate_that_will_always_evaluate_to_false)
})
})
.bench_function("null any black box item and predicate", |b| {
b.iter(|| {
any_black_box_item_and_predicate(
nullable_array,
predicate_that_will_always_evaluate_to_false,
)
})
})
.bench_function("null any black box only result", |b| {
b.iter(|| {
any_black_box_result(nullable_array, predicate_that_will_always_evaluate_to_false)
})
});
}
/// Replace all occurrences of `item_to_replace` with `replace_with` in the given `PrimitiveArray`.
/// will make it so we can filter by missing value
fn replace_primitive_value<T>(
array: PrimitiveArray<T>,
item_to_replace: T::Native,
replace_with: T::Native,
) -> PrimitiveArray<T>
where
T: ArrowPrimitiveType,
<T as ArrowPrimitiveType>::Native: Eq,
{
array.unary(|item| {
if item == item_to_replace {
replace_with
} else {
item
}
})
}
fn add_benchmark(c: &mut Criterion) {
benchmark_array_iter(
c,
"int8",
&replace_primitive_value(create_primitive_array::<Int8Type>(BATCH_SIZE, 0.0), 42, 1),
&replace_primitive_value(create_primitive_array::<Int8Type>(BATCH_SIZE, 0.5), 42, 1),
// fold init
0i8,
// fold function
|acc, item| acc.wrapping_add(item.unwrap_or_default()),
// predicate that will always evaluate to false while allowing us to avoid using hint::black_box and let the compiler optimize more
|item| item == Some(42),
);
benchmark_array_iter(
c,
"int16",
&replace_primitive_value(create_primitive_array::<Int16Type>(BATCH_SIZE, 0.0), 42, 1),
&replace_primitive_value(create_primitive_array::<Int16Type>(BATCH_SIZE, 0.5), 42, 1),
// fold init
0i16,
// fold function
|acc, item| acc.wrapping_add(item.unwrap_or_default()),
// predicate that will always evaluate to false while allowing us to avoid using hint::black_box and let the compiler optimize more
|item| item == Some(42),
);
benchmark_array_iter(
c,
"int32",
&replace_primitive_value(create_primitive_array::<Int32Type>(BATCH_SIZE, 0.0), 42, 1),
&replace_primitive_value(create_primitive_array::<Int32Type>(BATCH_SIZE, 0.5), 42, 1),
// fold init
0i32,
// fold function
|acc, item| acc.wrapping_add(item.unwrap_or_default()),
// predicate that will always evaluate to false while allowing us to avoid using hint::black_box and let the compiler optimize more
|item| item == Some(42),
);
benchmark_array_iter(
c,
"int64",
&replace_primitive_value(create_primitive_array::<Int64Type>(BATCH_SIZE, 0.0), 42, 1),
&replace_primitive_value(create_primitive_array::<Int64Type>(BATCH_SIZE, 0.5), 42, 1),
// fold init
0i64,
// fold function
|acc, item| acc.wrapping_add(item.unwrap_or_default()),
// predicate that will always evaluate to false while allowing us to avoid using hint::black_box and let the compiler optimize more
|item| item == Some(42),
);
benchmark_array_iter(
c,
"string with len 16",
&create_string_array_with_len::<i32>(BATCH_SIZE, 0.0, 16),
&create_string_array_with_len::<i32>(BATCH_SIZE, 0.5, 16),
// fold init
0_usize,
// fold function
|acc, item| acc.wrapping_add(item.map(|item| item.len()).unwrap_or_default()),
// predicate that will always evaluate to false while allowing us to avoid using hint::black_box and let the compiler optimize more
|item| item.is_some_and(|item| item.is_empty()),
);
benchmark_array_iter(
c,
"string view with len 16",
&create_string_view_array_with_len(BATCH_SIZE, 0.0, 16, false),
&create_string_view_array_with_len(BATCH_SIZE, 0.5, 16, false),
// fold init
0_usize,
// fold function
|acc, item| acc.wrapping_add(item.map(|item| item.len()).unwrap_or_default()),
// predicate that will always evaluate to false while allowing us to avoid using hint::black_box and let the compiler optimize more
|item| item.is_some_and(|item| item.is_empty()),
);
benchmark_array_iter(
c,
"boolean mixed true and false",
&create_boolean_array(BATCH_SIZE, 0.0, 0.5),
&create_boolean_array(BATCH_SIZE, 0.5, 0.5),
// fold init
0_usize,
// fold function
|acc, item| acc.wrapping_add(item.unwrap_or_default() as usize),
// Must use black_box here as this can be optimized away
|_item| hint::black_box(false),
);
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+52
View File
@@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::array::*;
use std::sync::Arc;
fn create_array_slice(array: &ArrayRef, length: usize) -> ArrayRef {
array.slice(0, length)
}
fn create_array_with_nulls(size: usize) -> ArrayRef {
let array: Float64Array = (0..size)
.map(|i| if i % 2 == 0 { Some(1.0) } else { None })
.collect();
Arc::new(array)
}
fn array_slice_benchmark(c: &mut Criterion) {
let array = create_array_with_nulls(4096);
c.bench_function("array_slice 128", |b| {
b.iter(|| create_array_slice(&array, 128))
});
c.bench_function("array_slice 512", |b| {
b.iter(|| create_array_slice(&array, 512))
});
c.bench_function("array_slice 2048", |b| {
b.iter(|| create_array_slice(&array, 2048))
});
}
criterion_group!(benches, array_slice_benchmark);
criterion_main!(benches);
+47
View File
@@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::{array::*, compute::kernels::length::bit_length};
use std::hint;
fn bench_bit_length(array: &StringArray) {
hint::black_box(bit_length(array).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
fn double_vec<T: Clone>(v: Vec<T>) -> Vec<T> {
[&v[..], &v[..]].concat()
}
// double ["hello", " ", "world", "!"] 10 times
let mut values = vec!["one", "on", "o", ""];
for _ in 0..10 {
values = double_vec(values);
}
let array = StringArray::from(values);
c.bench_function("bit_length", |b| b.iter(|| bench_bit_length(&array)));
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+118
View File
@@ -0,0 +1,118 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use arrow::compute::kernels::bitwise::{
bitwise_and, bitwise_and_scalar, bitwise_not, bitwise_or, bitwise_or_scalar, bitwise_xor,
bitwise_xor_scalar,
};
use arrow::datatypes::Int64Type;
use criterion::Criterion;
use rand::RngCore;
use std::hint;
extern crate arrow;
use arrow::util::bench_util::create_primitive_array;
use arrow::util::test_util::seedable_rng;
fn bitwise_array_benchmark(c: &mut Criterion) {
let size = 64 * 1024_usize;
let left_without_null = create_primitive_array::<Int64Type>(size, 0 as f32);
let right_without_null = create_primitive_array::<Int64Type>(size, 0 as f32);
let left_with_null = create_primitive_array::<Int64Type>(size, 0.2_f32);
let right_with_null = create_primitive_array::<Int64Type>(size, 0.2_f32);
// array and
let mut group = c.benchmark_group("bench bitwise array: and");
group.bench_function("bitwise array and, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_and(&left_without_null, &right_without_null).unwrap()))
});
group.bench_function("bitwise array and, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_and(&left_with_null, &right_with_null).unwrap()))
});
group.finish();
// array or
let mut group = c.benchmark_group("bench bitwise: or");
group.bench_function("bitwise array or, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_or(&left_without_null, &right_without_null).unwrap()))
});
group.bench_function("bitwise array or, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_or(&left_with_null, &right_with_null).unwrap()))
});
group.finish();
// xor
let mut group = c.benchmark_group("bench bitwise: xor");
group.bench_function("bitwise array xor, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_xor(&left_without_null, &right_without_null).unwrap()))
});
group.bench_function("bitwise array xor, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_xor(&left_with_null, &right_with_null).unwrap()))
});
group.finish();
// not
let mut group = c.benchmark_group("bench bitwise: not");
group.bench_function("bitwise array not, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_not(&left_without_null).unwrap()))
});
group.bench_function("bitwise array not, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_not(&left_with_null).unwrap()))
});
group.finish();
}
fn bitwise_array_scalar_benchmark(c: &mut Criterion) {
let size = 64 * 1024_usize;
let array_without_null = create_primitive_array::<Int64Type>(size, 0 as f32);
let array_with_null = create_primitive_array::<Int64Type>(size, 0.2_f32);
let scalar = seedable_rng().next_u64() as i64;
// array scalar and
let mut group = c.benchmark_group("bench bitwise array scalar: and");
group.bench_function("bitwise array scalar and, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_and_scalar(&array_without_null, scalar).unwrap()))
});
group.bench_function("bitwise array and, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_and_scalar(&array_with_null, scalar).unwrap()))
});
group.finish();
// array scalar or
let mut group = c.benchmark_group("bench bitwise array scalar: or");
group.bench_function("bitwise array scalar or, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_or_scalar(&array_without_null, scalar).unwrap()))
});
group.bench_function("bitwise array scalar or, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_or_scalar(&array_with_null, scalar).unwrap()))
});
group.finish();
// array scalar xor
let mut group = c.benchmark_group("bench bitwise array scalar: xor");
group.bench_function("bitwise array scalar xor, no nulls", |b| {
b.iter(|| hint::black_box(bitwise_xor_scalar(&array_without_null, scalar).unwrap()))
});
group.bench_function("bitwise array scalar xor, 20% nulls", |b| {
b.iter(|| hint::black_box(bitwise_xor_scalar(&array_with_null, scalar).unwrap()))
});
group.finish();
}
criterion_group!(
benches,
bitwise_array_benchmark,
bitwise_array_scalar_benchmark
);
criterion_main!(benches);
@@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::array::BooleanBufferBuilder;
use criterion::{Criterion, criterion_group, criterion_main};
use rand::{Rng, rng};
fn rand_bytes(len: usize) -> Vec<u8> {
let mut rng = rng();
let mut buf = vec![0_u8; len];
rng.fill(buf.as_mut_slice());
buf
}
fn boolean_append_packed(c: &mut Criterion) {
let mut rng = rng();
let source = rand_bytes(1024);
let ranges: Vec<_> = (0..100)
.map(|_| {
let start: usize = rng.random_range(0..1024 * 8);
let end: usize = rng.random_range(start..1024 * 8);
start..end
})
.collect();
let total_bits: usize = ranges.iter().map(|x| x.end - x.start).sum();
c.bench_function("boolean_append_packed", |b| {
b.iter(|| {
let mut buffer = BooleanBufferBuilder::new(total_bits);
for range in &ranges {
buffer.append_packed_range(range.clone(), &source);
}
assert_eq!(buffer.len(), total_bits);
})
});
}
criterion_group!(benches, boolean_append_packed);
criterion_main!(benches);
+83
View File
@@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use arrow::util::bench_util::create_boolean_array;
extern crate arrow;
use arrow::array::*;
use arrow::compute::kernels::boolean as boolean_kernels;
use std::hint;
fn bench_and(lhs: &BooleanArray, rhs: &BooleanArray) {
hint::black_box(boolean_kernels::and(lhs, rhs).unwrap());
}
fn bench_or(lhs: &BooleanArray, rhs: &BooleanArray) {
hint::black_box(boolean_kernels::or(lhs, rhs).unwrap());
}
fn bench_not(array: &BooleanArray) {
hint::black_box(boolean_kernels::not(array).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
// allocate arrays of 32K elements
let size = 2usize.pow(15);
// Note we allocate all arrays before the benchmark to ensure the allocation of the arrays
// is not affected by allocations that happen during the benchmarked operation.
let array1 = create_boolean_array(size, 0.0, 0.5);
let array2 = create_boolean_array(size, 0.0, 0.5);
// Slice by 1 (not aligned to byte (8 bit) or word (64 bit) boundaries)
let offset = 1;
let array1_sliced_1 = array1.slice(offset, size - offset);
let array2_sliced_1 = array2.slice(offset, size - offset);
// Slice by 24 (aligned on byte (8 bit) but not word (64 bit) boundaries)
let offset = 24;
let array1_sliced_24 = array1.slice(offset, size - offset);
let array2_sliced_24 = array2.slice(offset, size - offset);
c.bench_function("and", |b| b.iter(|| bench_and(&array1, &array2)));
c.bench_function("or", |b| b.iter(|| bench_or(&array1, &array2)));
c.bench_function("not", |b| b.iter(|| bench_not(&array1)));
c.bench_function("and_sliced_1", |b| {
b.iter(|| bench_and(&array1_sliced_1, &array2_sliced_1))
});
c.bench_function("or_sliced_1", |b| {
b.iter(|| bench_or(&array1_sliced_1, &array2_sliced_1))
});
c.bench_function("not_sliced_1", |b| b.iter(|| bench_not(&array1_sliced_1)));
c.bench_function("and_sliced_24", |b| {
b.iter(|| bench_and(&array1_sliced_24, &array2_sliced_24))
});
c.bench_function("or_sliced_24", |b| {
b.iter(|| bench_or(&array1_sliced_24, &array2_sliced_24))
});
c.bench_function("not_slice_24", |b| b.iter(|| bench_not(&array1_sliced_24)));
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+99
View File
@@ -0,0 +1,99 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::{Criterion, Throughput};
extern crate arrow;
use arrow::buffer::{Buffer, MutableBuffer, buffer_bin_and, buffer_bin_or, buffer_unary_not};
use std::hint;
/// Helper function to create arrays
fn create_buffer(size: usize) -> Buffer {
let mut result = MutableBuffer::new(size).with_bitset(size, false);
for i in 0..size {
result.as_slice_mut()[i] = 0b01010101 << i << (i % 4);
}
result.into()
}
fn bench_buffer_and(left: &Buffer, right: &Buffer) {
hint::black_box(buffer_bin_and(left, 0, right, 0, left.len() * 8));
}
fn bench_buffer_or(left: &Buffer, right: &Buffer) {
hint::black_box(buffer_bin_or(left, 0, right, 0, left.len() * 8));
}
fn bench_buffer_not(buffer: &Buffer) {
hint::black_box(buffer_unary_not(buffer, 0, buffer.len() * 8));
}
fn bench_buffer_and_with_offsets(
left: &Buffer,
left_offset: usize,
right: &Buffer,
right_offset: usize,
len: usize,
) {
hint::black_box(buffer_bin_and(left, left_offset, right, right_offset, len));
}
fn bench_buffer_or_with_offsets(
left: &Buffer,
left_offset: usize,
right: &Buffer,
right_offset: usize,
len: usize,
) {
hint::black_box(buffer_bin_or(left, left_offset, right, right_offset, len));
}
fn bench_buffer_not_with_offsets(buffer: &Buffer, offset: usize, len: usize) {
hint::black_box(buffer_unary_not(buffer, offset, len));
}
fn bit_ops_benchmark(c: &mut Criterion) {
let left = create_buffer(512 * 10);
let right = create_buffer(512 * 10);
c.benchmark_group("buffer_binary_ops")
.throughput(Throughput::Bytes(3 * left.len() as u64))
.bench_function("and", |b| b.iter(|| bench_buffer_and(&left, &right)))
.bench_function("or", |b| b.iter(|| bench_buffer_or(&left, &right)))
.bench_function("and_with_offset", |b| {
b.iter(|| bench_buffer_and_with_offsets(&left, 1, &right, 2, left.len() * 8 - 5))
})
.bench_function("or_with_offset", |b| {
b.iter(|| bench_buffer_or_with_offsets(&left, 1, &right, 2, left.len() * 8 - 5))
});
c.benchmark_group("buffer_unary_ops")
.throughput(Throughput::Bytes(2 * left.len() as u64))
.bench_function("not", |b| b.iter(|| bench_buffer_not(&left)))
.bench_function("not_with_offset", |b| {
b.iter(|| bench_buffer_not_with_offsets(&left, 1, left.len() * 8 - 5))
});
}
criterion_group!(benches, bit_ops_benchmark);
criterion_main!(benches);
+184
View File
@@ -0,0 +1,184 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use arrow::util::test_util::seedable_rng;
use criterion::Criterion;
use rand::Rng;
use rand::distr::Uniform;
extern crate arrow;
use arrow::{
buffer::{Buffer, MutableBuffer},
datatypes::ToByteSlice,
};
use std::hint;
fn mutable_buffer_from_iter(data: &[Vec<bool>]) -> Vec<Buffer> {
hint::black_box(
data.iter()
.map(|vec| vec.iter().copied().collect::<MutableBuffer>().into())
.collect::<Vec<_>>(),
)
}
fn buffer_from_iter(data: &[Vec<bool>]) -> Vec<Buffer> {
hint::black_box(
data.iter()
.map(|vec| vec.iter().copied().collect::<Buffer>())
.collect::<Vec<_>>(),
)
}
fn mutable_buffer_iter_bitset(data: &[Vec<bool>]) -> Vec<Buffer> {
hint::black_box({
data.iter()
.map(|datum| {
let mut result =
MutableBuffer::new(data.len().div_ceil(8)).with_bitset(datum.len(), false);
for (i, value) in datum.iter().enumerate() {
if *value {
unsafe {
arrow::util::bit_util::set_bit_raw(result.as_mut_ptr(), i);
}
}
}
result.into()
})
.collect::<Vec<_>>()
})
}
fn mutable_iter_extend_from_slice(data: &[Vec<u32>], capacity: usize) -> Buffer {
hint::black_box({
let mut result = MutableBuffer::new(capacity);
data.iter().for_each(|vec| {
vec.iter()
.for_each(|elem| result.extend_from_slice(elem.to_byte_slice()))
});
result.into()
})
}
fn mutable_buffer(data: &[Vec<u32>], capacity: usize) -> Buffer {
hint::black_box({
let mut result = MutableBuffer::new(capacity);
data.iter().for_each(|vec| result.extend_from_slice(vec));
result.into()
})
}
fn mutable_buffer_extend(data: &[Vec<u32>], capacity: usize) -> Buffer {
hint::black_box({
let mut result = MutableBuffer::new(capacity);
data.iter()
.for_each(|vec| result.extend(vec.iter().copied()));
result.into()
})
}
fn from_slice(data: &[Vec<u32>], capacity: usize) -> Buffer {
hint::black_box({
let mut a = Vec::<u32>::with_capacity(capacity);
data.iter().for_each(|vec| a.extend(vec));
Buffer::from(a.to_byte_slice())
})
}
fn create_data(size: usize) -> Vec<Vec<u32>> {
let rng = &mut seedable_rng();
let range = Uniform::new(0, 33).unwrap();
(0..size)
.map(|_| {
let size = rng.sample(range);
seedable_rng()
.sample_iter(&range)
.take(size as usize)
.collect()
})
.collect()
}
fn create_data_bool(size: usize) -> Vec<Vec<bool>> {
let rng = &mut seedable_rng();
let range = Uniform::new(0, 33).unwrap();
(0..size)
.map(|_| {
let size = rng.sample(range);
seedable_rng()
.sample_iter(&range)
.take(size as usize)
.map(|x| x > 15)
.collect()
})
.collect()
}
fn benchmark(c: &mut Criterion) {
let size = 2usize.pow(15);
let data = create_data(size);
let bool_data = create_data_bool(size);
let cap = data.iter().map(|i| i.len()).sum();
let byte_cap = cap * std::mem::size_of::<u32>();
c.bench_function("mutable iter extend_from_slice", |b| {
b.iter(|| mutable_iter_extend_from_slice(hint::black_box(&data), hint::black_box(0)))
});
c.bench_function("mutable", |b| {
b.iter(|| mutable_buffer(hint::black_box(&data), hint::black_box(0)))
});
c.bench_function("mutable extend", |b| {
b.iter(|| mutable_buffer_extend(&data, 0))
});
c.bench_function("mutable prepared", |b| {
b.iter(|| mutable_buffer(hint::black_box(&data), hint::black_box(byte_cap)))
});
c.bench_function("from_slice", |b| {
b.iter(|| from_slice(hint::black_box(&data), hint::black_box(0)))
});
c.bench_function("from_slice prepared", |b| {
b.iter(|| from_slice(hint::black_box(&data), hint::black_box(cap)))
});
c.bench_function("MutableBuffer iter bitset", |b| {
b.iter(|| mutable_buffer_iter_bitset(hint::black_box(&bool_data)))
});
c.bench_function("MutableBuffer::from_iter bool", |b| {
b.iter(|| mutable_buffer_from_iter(hint::black_box(&bool_data)))
});
c.bench_function("Buffer::from_iter bool", |b| {
b.iter(|| buffer_from_iter(hint::black_box(&bool_data)))
});
}
criterion_group!(benches, benchmark);
criterion_main!(benches);
+195
View File
@@ -0,0 +1,195 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
extern crate criterion;
extern crate rand;
use std::mem::size_of;
use criterion::*;
use rand::distr::StandardUniform;
use arrow::array::*;
use arrow::util::test_util::seedable_rng;
use arrow_buffer::i256;
use rand::Rng;
use std::hint;
// Build arrays with 512k elements.
const BATCH_SIZE: usize = 8 << 10;
const NUM_BATCHES: usize = 64;
fn bench_primitive(c: &mut Criterion) {
let data: [i64; BATCH_SIZE] = [100; BATCH_SIZE];
let mut group = c.benchmark_group("bench_primitive");
group.throughput(Throughput::Bytes(
((data.len() * NUM_BATCHES * size_of::<i64>()) as u32).into(),
));
group.bench_function("bench_primitive", |b| {
b.iter(|| {
let mut builder = Int64Builder::with_capacity(64);
for _ in 0..NUM_BATCHES {
builder.append_slice(&data[..]);
}
hint::black_box(builder.finish());
})
});
group.finish();
}
fn bench_primitive_nulls(c: &mut Criterion) {
let mut group = c.benchmark_group("bench_primitive_nulls");
group.bench_function("bench_primitive_nulls", |b| {
b.iter(|| {
let mut builder = UInt8Builder::with_capacity(64);
for _ in 0..NUM_BATCHES * BATCH_SIZE {
builder.append_null();
}
hint::black_box(builder.finish());
})
});
group.finish();
}
fn bench_bool(c: &mut Criterion) {
let data: Vec<bool> = seedable_rng()
.sample_iter(&StandardUniform)
.take(BATCH_SIZE)
.collect();
let data_len = data.len();
let mut group = c.benchmark_group("bench_bool");
group.throughput(Throughput::Bytes(
((data_len * NUM_BATCHES * size_of::<bool>()) as u32).into(),
));
group.bench_function("bench_bool", |b| {
b.iter(|| {
let mut builder = BooleanBuilder::with_capacity(64);
for _ in 0..NUM_BATCHES {
builder.append_slice(&data[..]);
}
hint::black_box(builder.finish());
})
});
group.finish();
}
fn bench_string(c: &mut Criterion) {
const SAMPLE_STRING: &str = "sample string";
let mut group = c.benchmark_group("bench_primitive");
group.throughput(Throughput::Bytes(
((BATCH_SIZE * NUM_BATCHES * SAMPLE_STRING.len()) as u32).into(),
));
group.bench_function("bench_string", |b| {
b.iter(|| {
let mut builder = StringBuilder::new();
for _ in 0..NUM_BATCHES * BATCH_SIZE {
builder.append_value(SAMPLE_STRING);
}
hint::black_box(builder.finish());
})
});
group.finish();
}
fn bench_decimal32(c: &mut Criterion) {
c.bench_function("bench_decimal32_builder", |b| {
b.iter(|| {
let mut rng = rand::rng();
let mut decimal_builder = Decimal32Builder::with_capacity(BATCH_SIZE);
for _ in 0..BATCH_SIZE {
decimal_builder.append_value(rng.random_range::<i32, _>(0..999999999));
}
hint::black_box(
decimal_builder
.finish()
.with_precision_and_scale(9, 0)
.unwrap(),
);
})
});
}
fn bench_decimal64(c: &mut Criterion) {
c.bench_function("bench_decimal64_builder", |b| {
b.iter(|| {
let mut rng = rand::rng();
let mut decimal_builder = Decimal64Builder::with_capacity(BATCH_SIZE);
for _ in 0..BATCH_SIZE {
decimal_builder.append_value(rng.random_range::<i64, _>(0..9999999999));
}
hint::black_box(
decimal_builder
.finish()
.with_precision_and_scale(18, 0)
.unwrap(),
);
})
});
}
fn bench_decimal128(c: &mut Criterion) {
c.bench_function("bench_decimal128_builder", |b| {
b.iter(|| {
let mut rng = rand::rng();
let mut decimal_builder = Decimal128Builder::with_capacity(BATCH_SIZE);
for _ in 0..BATCH_SIZE {
decimal_builder.append_value(rng.random_range::<i128, _>(0..9999999999));
}
hint::black_box(
decimal_builder
.finish()
.with_precision_and_scale(38, 0)
.unwrap(),
);
})
});
}
fn bench_decimal256(c: &mut Criterion) {
c.bench_function("bench_decimal256_builder", |b| {
b.iter(|| {
let mut rng = rand::rng();
let mut decimal_builder = Decimal256Builder::with_capacity(BATCH_SIZE);
for _ in 0..BATCH_SIZE {
decimal_builder
.append_value(i256::from_i128(rng.random_range::<i128, _>(0..99999999999)));
}
hint::black_box(
decimal_builder
.finish()
.with_precision_and_scale(76, 10)
.unwrap(),
);
})
});
}
criterion_group!(
benches,
bench_primitive,
bench_primitive_nulls,
bench_bool,
bench_string,
bench_decimal32,
bench_decimal64,
bench_decimal128,
bench_decimal256,
);
criterion_main!(benches);
+405
View File
@@ -0,0 +1,405 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use rand::Rng;
use rand::distr::{Distribution, StandardUniform, Uniform};
use std::hint;
use chrono::DateTime;
use std::sync::Arc;
extern crate arrow;
use arrow::array::*;
use arrow::compute::cast;
use arrow::datatypes::*;
use arrow::util::bench_util::*;
use arrow::util::test_util::seedable_rng;
fn build_array<T: ArrowPrimitiveType>(size: usize) -> ArrayRef
where
StandardUniform: Distribution<T::Native>,
{
let array = create_primitive_array::<T>(size, 0.1);
Arc::new(array)
}
fn build_utf8_date_array(size: usize, with_nulls: bool) -> ArrayRef {
use chrono::NaiveDate;
// use random numbers to avoid spurious compiler optimizations wrt to branching
let mut rng = seedable_rng();
let mut builder = StringBuilder::new();
let range = Uniform::new(0, 737776).unwrap();
for _ in 0..size {
if with_nulls && rng.random::<f32>() > 0.8 {
builder.append_null();
} else {
let string = NaiveDate::from_num_days_from_ce_opt(rng.sample(range))
.unwrap()
.format("%Y-%m-%d")
.to_string();
builder.append_value(&string);
}
}
Arc::new(builder.finish())
}
fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef {
// use random numbers to avoid spurious compiler optimizations wrt to branching
let mut rng = seedable_rng();
let mut builder = StringBuilder::new();
let range = Uniform::new(0, 1608071414123).unwrap();
for _ in 0..size {
if with_nulls && rng.random::<f32>() > 0.8 {
builder.append_null();
} else {
let string = DateTime::from_timestamp(rng.sample(range), 0)
.unwrap()
.format("%Y-%m-%dT%H:%M:%S")
.to_string();
builder.append_value(&string);
}
}
Arc::new(builder.finish())
}
fn build_decimal32_array(size: usize, precision: u8, scale: i8) -> ArrayRef {
let mut rng = seedable_rng();
let mut builder = Decimal32Builder::with_capacity(size);
for _ in 0..size {
builder.append_value(rng.random_range::<i32, _>(0..1000000));
}
Arc::new(
builder
.finish()
.with_precision_and_scale(precision, scale)
.unwrap(),
)
}
fn build_decimal64_array(size: usize, precision: u8, scale: i8) -> ArrayRef {
let mut rng = seedable_rng();
let mut builder = Decimal64Builder::with_capacity(size);
for _ in 0..size {
builder.append_value(rng.random_range::<i64, _>(0..1000000000));
}
Arc::new(
builder
.finish()
.with_precision_and_scale(precision, scale)
.unwrap(),
)
}
fn build_decimal128_array(size: usize, precision: u8, scale: i8) -> ArrayRef {
let mut rng = seedable_rng();
let mut builder = Decimal128Builder::with_capacity(size);
for _ in 0..size {
builder.append_value(rng.random_range::<i128, _>(0..1000000000));
}
Arc::new(
builder
.finish()
.with_precision_and_scale(precision, scale)
.unwrap(),
)
}
fn build_decimal256_array(size: usize, precision: u8, scale: i8) -> ArrayRef {
let mut rng = seedable_rng();
let mut builder = Decimal256Builder::with_capacity(size);
let mut bytes = [0; 32];
for _ in 0..size {
let num = rng.random_range::<i128, _>(0..1000000000);
bytes[0..16].clone_from_slice(&num.to_le_bytes());
builder.append_value(i256::from_le_bytes(bytes));
}
Arc::new(
builder
.finish()
.with_precision_and_scale(precision, scale)
.unwrap(),
)
}
fn build_string_array(size: usize) -> ArrayRef {
let mut builder = StringBuilder::new();
for v in 0..size {
match v % 3 {
0 => builder.append_value("small"),
1 => builder.append_value("larger string more than 12 bytes"),
_ => builder.append_null(),
}
}
Arc::new(builder.finish())
}
fn build_dict_array(size: usize) -> ArrayRef {
let values = StringArray::from_iter([
Some("small"),
Some("larger string more than 12 bytes"),
None,
]);
let keys = UInt64Array::from_iter((0..size as u64).map(|v| v % 3));
Arc::new(DictionaryArray::new(keys, Arc::new(values)))
}
// cast array from specified primitive array type to desired data type
fn cast_array(array: &ArrayRef, to_type: DataType) {
hint::black_box(cast(array, &to_type).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
let i32_array = build_array::<Int32Type>(512);
let i64_array = build_array::<Int64Type>(512);
let f32_array = build_array::<Float32Type>(512);
let f32_utf8_array = cast(&build_array::<Float32Type>(512), &DataType::Utf8).unwrap();
let f64_array = build_array::<Float64Type>(512);
let date64_array = build_array::<Date64Type>(512);
let date32_array = build_array::<Date32Type>(512);
let time32s_array = build_array::<Time32SecondType>(512);
let time64ns_array = build_array::<Time64NanosecondType>(512);
let time_ns_array = build_array::<TimestampNanosecondType>(512);
let time_ms_array = build_array::<TimestampMillisecondType>(512);
let utf8_date_array = build_utf8_date_array(512, true);
let utf8_date_time_array = build_utf8_date_time_array(512, true);
let decimal32_array = build_decimal32_array(512, 9, 3);
let decimal64_array = build_decimal64_array(512, 10, 3);
let decimal128_array = build_decimal128_array(512, 10, 3);
let decimal256_array = build_decimal256_array(512, 50, 3);
let string_array = build_string_array(512);
let wide_string_array = cast(&string_array, &DataType::LargeUtf8).unwrap();
let dict_array = build_dict_array(10_000);
let string_view_array = cast(&dict_array, &DataType::Utf8View).unwrap();
let binary_view_array = cast(&string_view_array, &DataType::BinaryView).unwrap();
c.bench_function("cast int32 to int32 512", |b| {
b.iter(|| cast_array(&i32_array, DataType::Int32))
});
c.bench_function("cast int32 to uint32 512", |b| {
b.iter(|| cast_array(&i32_array, DataType::UInt32))
});
c.bench_function("cast int32 to float32 512", |b| {
b.iter(|| cast_array(&i32_array, DataType::Float32))
});
c.bench_function("cast int32 to float64 512", |b| {
b.iter(|| cast_array(&i32_array, DataType::Float64))
});
c.bench_function("cast int32 to int64 512", |b| {
b.iter(|| cast_array(&i32_array, DataType::Int64))
});
c.bench_function("cast float32 to int32 512", |b| {
b.iter(|| cast_array(&f32_array, DataType::Int32))
});
c.bench_function("cast float64 to float32 512", |b| {
b.iter(|| cast_array(&f64_array, DataType::Float32))
});
c.bench_function("cast float64 to uint64 512", |b| {
b.iter(|| cast_array(&f64_array, DataType::UInt64))
});
c.bench_function("cast int64 to int32 512", |b| {
b.iter(|| cast_array(&i64_array, DataType::Int32))
});
c.bench_function("cast date64 to date32 512", |b| {
b.iter(|| cast_array(&date64_array, DataType::Date32))
});
c.bench_function("cast date32 to date64 512", |b| {
b.iter(|| cast_array(&date32_array, DataType::Date64))
});
c.bench_function("cast time32s to time32ms 512", |b| {
b.iter(|| cast_array(&time32s_array, DataType::Time32(TimeUnit::Millisecond)))
});
c.bench_function("cast time32s to time64us 512", |b| {
b.iter(|| cast_array(&time32s_array, DataType::Time64(TimeUnit::Microsecond)))
});
c.bench_function("cast time64ns to time32s 512", |b| {
b.iter(|| cast_array(&time64ns_array, DataType::Time32(TimeUnit::Second)))
});
c.bench_function("cast timestamp_ns to timestamp_s 512", |b| {
b.iter(|| {
cast_array(
&time_ns_array,
DataType::Timestamp(TimeUnit::Nanosecond, None),
)
})
});
c.bench_function("cast timestamp_ms to timestamp_ns 512", |b| {
b.iter(|| {
cast_array(
&time_ms_array,
DataType::Timestamp(TimeUnit::Nanosecond, None),
)
})
});
c.bench_function("cast utf8 to f32", |b| {
b.iter(|| cast_array(&f32_utf8_array, DataType::Float32))
});
c.bench_function("cast i64 to string 512", |b| {
b.iter(|| cast_array(&i64_array, DataType::Utf8))
});
c.bench_function("cast f32 to string 512", |b| {
b.iter(|| cast_array(&f32_array, DataType::Utf8))
});
c.bench_function("cast f64 to string 512", |b| {
b.iter(|| cast_array(&f64_array, DataType::Utf8))
});
c.bench_function("cast timestamp_ms to i64 512", |b| {
b.iter(|| cast_array(&time_ms_array, DataType::Int64))
});
c.bench_function("cast utf8 to date32 512", |b| {
b.iter(|| cast_array(&utf8_date_array, DataType::Date32))
});
c.bench_function("cast utf8 to date64 512", |b| {
b.iter(|| cast_array(&utf8_date_time_array, DataType::Date64))
});
c.bench_function("cast decimal32 to decimal32 512", |b| {
b.iter(|| cast_array(&decimal32_array, DataType::Decimal32(9, 4)))
});
c.bench_function("cast decimal32 to decimal32 512 lower precision", |b| {
b.iter(|| cast_array(&decimal32_array, DataType::Decimal32(6, 5)))
});
c.bench_function("cast decimal32 to decimal64 512", |b| {
b.iter(|| cast_array(&decimal32_array, DataType::Decimal64(11, 5)))
});
c.bench_function("cast decimal64 to decimal32 512", |b| {
b.iter(|| cast_array(&decimal64_array, DataType::Decimal32(9, 2)))
});
c.bench_function("cast decimal64 to decimal64 512", |b| {
b.iter(|| cast_array(&decimal64_array, DataType::Decimal64(12, 4)))
});
c.bench_function("cast decimal128 to decimal128 512", |b| {
b.iter(|| cast_array(&decimal128_array, DataType::Decimal128(30, 5)))
});
c.bench_function("cast decimal128 to decimal128 512 lower precision", |b| {
b.iter(|| cast_array(&decimal128_array, DataType::Decimal128(6, 5)))
});
c.bench_function("cast decimal128 to decimal256 512", |b| {
b.iter(|| cast_array(&decimal128_array, DataType::Decimal256(50, 5)))
});
c.bench_function("cast decimal256 to decimal128 512", |b| {
b.iter(|| cast_array(&decimal256_array, DataType::Decimal128(38, 2)))
});
c.bench_function("cast decimal256 to decimal256 512", |b| {
b.iter(|| cast_array(&decimal256_array, DataType::Decimal256(50, 5)))
});
c.bench_function("cast decimal128 to decimal128 512 with same scale", |b| {
b.iter(|| cast_array(&decimal128_array, DataType::Decimal128(30, 3)))
});
c.bench_function(
"cast decimal128 to decimal128 512 with lower scale (infallible)",
|b| b.iter(|| cast_array(&decimal128_array, DataType::Decimal128(7, -1))),
);
c.bench_function("cast decimal256 to decimal256 512 with same scale", |b| {
b.iter(|| cast_array(&decimal256_array, DataType::Decimal256(60, 3)))
});
c.bench_function("cast dict to string view", |b| {
b.iter(|| cast_array(&dict_array, DataType::Utf8View))
});
c.bench_function("cast string view to dict", |b| {
b.iter(|| {
cast_array(
&string_view_array,
DataType::Dictionary(Box::new(DataType::UInt64), Box::new(DataType::Utf8)),
)
})
});
c.bench_function("cast string view to string", |b| {
b.iter(|| cast_array(&string_view_array, DataType::Utf8))
});
c.bench_function("cast string view to wide string", |b| {
b.iter(|| cast_array(&string_view_array, DataType::LargeUtf8))
});
c.bench_function("cast binary view to string", |b| {
b.iter(|| cast_array(&binary_view_array, DataType::Utf8))
});
c.bench_function("cast binary view to wide string", |b| {
b.iter(|| cast_array(&binary_view_array, DataType::LargeUtf8))
});
c.bench_function("cast string to binary view 512", |b| {
b.iter(|| cast_array(&string_array, DataType::BinaryView))
});
c.bench_function("cast wide string to binary view 512", |b| {
b.iter(|| cast_array(&wide_string_array, DataType::BinaryView))
});
c.bench_function("cast string view to binary view", |b| {
b.iter(|| cast_array(&string_view_array, DataType::BinaryView))
});
c.bench_function("cast binary view to string view", |b| {
b.iter(|| cast_array(&binary_view_array, DataType::Utf8View))
});
c.bench_function("cast string single run to ree<int32>", |b| {
let source_array = StringArray::from(vec!["a"; 8192]);
let array_ref = Arc::new(source_array) as ArrayRef;
let target_type = DataType::RunEndEncoded(
Arc::new(Field::new("run_ends", DataType::Int32, false)),
Arc::new(Field::new("values", DataType::Utf8, true)),
);
b.iter(|| cast(&array_ref, &target_type).unwrap());
});
c.bench_function("cast runs of 10 string to ree<int32>", |b| {
let source_array: Int32Array = (0..8192).map(|i| i / 10).collect();
let array_ref = Arc::new(source_array) as ArrayRef;
let target_type = DataType::RunEndEncoded(
Arc::new(Field::new("run_ends", DataType::Int32, false)),
Arc::new(Field::new("values", DataType::Int32, true)),
);
b.iter(|| cast(&array_ref, &target_type).unwrap());
});
c.bench_function("cast runs of 1000 int32s to ree<int32>", |b| {
let source_array: Int32Array = (0..8192).map(|i| i / 1000).collect();
let array_ref = Arc::new(source_array) as ArrayRef;
let target_type = DataType::RunEndEncoded(
Arc::new(Field::new("run_ends", DataType::Int32, false)),
Arc::new(Field::new("values", DataType::Int32, true)),
);
b.iter(|| cast(&array_ref, &target_type).unwrap());
});
c.bench_function("cast no runs of int32s to ree<int32>", |b| {
let source_array: Int32Array = (0..8192).collect();
let array_ref = Arc::new(source_array) as ArrayRef;
let target_type = DataType::RunEndEncoded(
Arc::new(Field::new("run_ends", DataType::Int32, false)),
Arc::new(Field::new("values", DataType::Int32, true)),
);
b.iter(|| cast(&array_ref, &target_type).unwrap());
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+471
View File
@@ -0,0 +1,471 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Benchmarks for the `coalesce` kernels in Arrow.
use arrow::util::bench_util::*;
use std::sync::Arc;
use arrow::array::*;
use arrow_array::types::{Float64Type, Int32Type, TimestampNanosecondType};
use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
use arrow_select::coalesce::BatchCoalescer;
use criterion::{Criterion, criterion_group, criterion_main};
/// Benchmarks for generating evently sized output RecordBatches
/// from a sequence of filtered source batches
///
fn add_all_filter_benchmarks(c: &mut Criterion) {
let batch_size = 8192; // 8K rows is a commonly used size for batches
// Multiple primitive types
let primitive_schema = SchemaRef::new(Schema::new(vec![
Field::new("int32_val", DataType::Int32, true),
Field::new("float_val", DataType::Float64, true),
Field::new(
"timestamp_val",
DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
true,
),
]));
// Single StringViewArray
let single_schema = SchemaRef::new(Schema::new(vec![Field::new(
"value",
DataType::Utf8View,
true,
)]));
// Mixed primitive, StringViewArray
let mixed_utf8view_schema = SchemaRef::new(Schema::new(vec![
Field::new("int32_val", DataType::Int32, true),
Field::new("float_val", DataType::Float64, true),
Field::new("utf8view_val", DataType::Utf8View, true),
]));
// Mixed primitive, StringArray
let mixed_utf8_schema = SchemaRef::new(Schema::new(vec![
Field::new("int32_val", DataType::Int32, true),
Field::new("float_val", DataType::Float64, true),
Field::new("utf8", DataType::Utf8, true),
]));
// dictionary types
//
let mixed_dict_schema = SchemaRef::new(Schema::new(vec![
Field::new(
"string_dict",
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
true,
),
Field::new("float_val1", DataType::Float64, true),
Field::new("float_val2", DataType::Float64, true),
// TODO model other dictionary types here (FixedSizeBinary for example)
]));
// Null density: 0, 10%
for null_density in [0.0, 0.1] {
// Selectivity: 0.1%, 1%, 10%, 80%
for selectivity in [0.001, 0.01, 0.1, 0.8] {
FilterBenchmarkBuilder {
c,
name: "primitive",
batch_size,
num_output_batches: 50,
null_density,
selectivity,
max_string_len: 30,
schema: &primitive_schema,
}
.build();
FilterBenchmarkBuilder {
c,
name: "single_utf8view",
batch_size,
num_output_batches: 50,
null_density,
selectivity,
max_string_len: 30,
schema: &single_schema,
}
.build();
// Model mostly short strings, but some longer ones
FilterBenchmarkBuilder {
c,
name: "mixed_utf8view (max_string_len=20)",
batch_size,
num_output_batches: 20,
null_density,
selectivity,
max_string_len: 20,
schema: &mixed_utf8view_schema,
}
.build();
// Model mostly longer strings
FilterBenchmarkBuilder {
c,
name: "mixed_utf8view (max_string_len=128)",
batch_size,
num_output_batches: 20,
null_density,
selectivity,
max_string_len: 128,
schema: &mixed_utf8view_schema,
}
.build();
FilterBenchmarkBuilder {
c,
name: "mixed_utf8",
batch_size,
num_output_batches: 20,
null_density,
selectivity,
max_string_len: 30,
schema: &mixed_utf8_schema,
}
.build();
FilterBenchmarkBuilder {
c,
name: "mixed_dict",
batch_size,
num_output_batches: 10,
null_density,
selectivity,
max_string_len: 30,
schema: &mixed_dict_schema,
}
.build();
}
}
}
criterion_group!(benches, add_all_filter_benchmarks);
criterion_main!(benches);
/// Run the filters with a batch_size, null_density, selectivity, and schema
struct FilterBenchmarkBuilder<'a> {
/// Benchmark criterion instance
c: &'a mut Criterion,
/// Name of the benchmark
name: &'a str,
/// Size of the input and output batches
batch_size: usize,
/// Number of output batches to collect (tuned to keep benchmark time reasonable)
num_output_batches: usize,
/// between 0.0 .. 1.0, percent of data rows (not filter rows) that should be null
null_density: f32,
/// between 0.0 .. 1.0, percent of rows that should be kept by the filter
selectivity: f32,
/// The maximum length of strings in the data stream
///
/// For StringViewArray, strings <= 12 bytes are stored inline, longer
/// strings are stored in a separate buffer so it is important to vary to
/// mix the relative paths
max_string_len: usize,
/// Schema of the data stream
schema: &'a SchemaRef,
}
impl FilterBenchmarkBuilder<'_> {
fn build(self) {
let Self {
c,
name,
batch_size,
num_output_batches,
null_density,
selectivity,
max_string_len,
schema,
} = self;
let filters = FilterStreamBuilder::new()
.with_batch_size(batch_size)
.with_true_density(selectivity)
.with_null_density(0.0) // no nulls in the filter
.build();
let data = DataStreamBuilder::new(Arc::clone(schema))
.with_batch_size(batch_size)
.with_null_density(null_density)
.with_max_string_len(max_string_len)
.build();
// Keep feeding the filter stream into the coalescer until we hit a total number of output batches
let id = format!(
"filter: {name}, {batch_size}, nulls: {null_density}, selectivity: {selectivity}"
);
c.bench_function(&id, |b| {
b.iter(|| {
filter_streams(num_output_batches, filters.clone(), data.clone());
})
});
}
}
/// Pull RecordBatches from a data stream and apply a sequence of
/// filters from a filter stream until we have a specified number of output
/// batches.
fn filter_streams(
mut num_output_batches: usize,
mut filter_stream: FilterStream,
mut data_stream: DataStream,
) {
let schema = data_stream.schema();
let batch_size = data_stream.batch_size();
let mut coalescer = BatchCoalescer::new(Arc::clone(schema), batch_size);
while num_output_batches > 0 {
let filter = filter_stream.next_filter();
let batch = data_stream.next_batch();
coalescer
.push_batch_with_filter(batch.clone(), filter)
.unwrap();
// consume (but discard) the output batch
if coalescer.next_completed_batch().is_some() {
num_output_batches -= 1;
}
}
}
/// Stream of filters to apply to a sequence of input RecordBatches
///
/// This pre-computes a sequence of filters and then repeats it forever.
#[derive(Debug, Clone)]
struct FilterStream {
index: usize,
// arc'd so it is cheaply cloned
batches: Arc<[BooleanArray]>,
}
impl FilterStream {
pub fn next_filter(&mut self) -> &BooleanArray {
let current_index = self.index;
self.index += 1;
if self.index >= self.batches.len() {
self.index = 0; // loop back to the start
}
self.batches
.get(current_index)
.expect("No more filters available")
}
}
#[derive(Debug)]
struct FilterStreamBuilder {
batch_size: usize,
num_batches: usize, // number of unique batches to create
null_density: f32,
true_density: f32,
}
impl FilterStreamBuilder {
fn new() -> Self {
FilterStreamBuilder {
batch_size: 8192, // default batch size
num_batches: 11, // default number of unique batches (different than data stream)
null_density: 0.0, // default null density
true_density: 0.5, // default true density
}
}
/// set the batch size for the filter stream
fn with_batch_size(mut self, batch_size: usize) -> Self {
self.batch_size = batch_size;
self
}
/// set the null density for the filter stream
fn with_null_density(mut self, null_density: f32) -> Self {
assert!((0.0..=1.0).contains(&null_density));
self.null_density = null_density;
self
}
/// set the true density for the filter stream
fn with_true_density(mut self, true_density: f32) -> Self {
assert!((0.0..=1.0).contains(&true_density));
self.true_density = true_density;
self
}
fn build(self) -> FilterStream {
let Self {
batch_size,
num_batches,
null_density,
true_density,
} = self;
let batches = (0..num_batches)
.map(|_| create_boolean_array(batch_size, null_density, true_density))
.collect::<Vec<_>>();
FilterStream {
index: 0,
batches: Arc::from(batches),
}
}
}
#[derive(Debug, Clone)]
struct DataStream {
schema: SchemaRef,
index: usize,
batch_size: usize,
// arc'd so it is cheaply cloned
batches: Arc<[RecordBatch]>,
}
impl DataStream {
/// Returns the schema for this data stream
pub fn schema(&self) -> &SchemaRef {
&self.schema
}
/// Returns the batch size
pub fn batch_size(&self) -> usize {
self.batch_size
}
fn next_batch(&mut self) -> &RecordBatch {
let current_index = self.index;
self.index += 1;
if self.index >= self.batches.len() {
self.index = 0; // loop back to the start
}
self.batches
.get(current_index)
.expect("No more batches available")
}
}
#[derive(Debug, Clone)]
struct DataStreamBuilder {
schema: SchemaRef,
batch_size: usize,
null_density: f32,
num_batches: usize, // number of unique batches to create
max_string_len: usize, // maximum length of strings in the data stream
}
impl DataStreamBuilder {
fn new(schema: SchemaRef) -> Self {
DataStreamBuilder {
schema,
batch_size: 8192,
null_density: 0.0,
num_batches: 10,
max_string_len: 30,
}
}
/// set the batch size for the data stream
fn with_batch_size(mut self, batch_size: usize) -> Self {
self.batch_size = batch_size;
self
}
/// set the null density for the data stream
fn with_null_density(mut self, null_density: f32) -> Self {
assert!((0.0..=1.0).contains(&null_density));
self.null_density = null_density;
self
}
fn with_max_string_len(mut self, max_string_len: usize) -> Self {
self.max_string_len = max_string_len;
self
}
/// build the data stream (not implemented yet)
fn build(self) -> DataStream {
let batches = (0..self.num_batches)
.map(|seed| {
let columns = self
.schema
.fields()
.iter()
.map(|field| self.create_input_array(field, seed as u64))
.collect::<Vec<_>>();
RecordBatch::try_new(self.schema.clone(), columns).unwrap()
})
.collect::<Vec<_>>();
let Self {
schema,
batch_size,
null_density: _,
num_batches: _,
max_string_len: _,
} = self;
DataStream {
schema,
index: 0,
batch_size,
batches: Arc::from(batches),
}
}
fn create_input_array(&self, field: &Field, seed: u64) -> ArrayRef {
match field.data_type() {
DataType::Int32 => Arc::new(create_primitive_array_with_seed::<Int32Type>(
self.batch_size,
self.null_density,
seed,
)),
DataType::Float64 => Arc::new(create_primitive_array_with_seed::<Float64Type>(
self.batch_size,
self.null_density,
seed,
)),
DataType::Timestamp(TimeUnit::Nanosecond, Some(tz)) => Arc::new(
create_primitive_array_with_seed::<TimestampNanosecondType>(
self.batch_size,
self.null_density,
seed,
)
.with_timezone(Arc::clone(tz)),
),
DataType::Utf8 => Arc::new(create_string_array::<i32>(
self.batch_size,
self.null_density,
)), // TODO seed
DataType::Utf8View => {
Arc::new(create_string_view_array_with_max_len(
self.batch_size,
self.null_density,
self.max_string_len,
)) // TODO seed
}
DataType::Dictionary(key_type, value_type)
if key_type.as_ref() == &DataType::Int32
&& value_type.as_ref() == &DataType::Utf8 =>
{
Arc::new(create_string_dict_array::<Int32Type>(
self.batch_size,
self.null_density,
self.max_string_len,
)) // TODO seed
}
_ => panic!("Unsupported data type: {field:?}"),
}
}
}
+536
View File
@@ -0,0 +1,536 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
#[macro_use]
extern crate criterion;
use arrow::compute::kernels::cmp::*;
use arrow::util::bench_util::*;
use arrow::util::test_util::seedable_rng;
use arrow::{array::*, datatypes::Float32Type, datatypes::Int32Type};
use arrow_buffer::IntervalMonthDayNano;
use arrow_string::like::*;
use arrow_string::regexp::regexp_is_match_scalar;
use criterion::Criterion;
use rand::Rng;
use rand::rngs::StdRng;
use std::hint;
const SIZE: usize = 65536;
fn bench_like_utf8_scalar(arr_a: &StringArray, value_b: &str) {
like(arr_a, &StringArray::new_scalar(value_b)).unwrap();
}
fn bench_like_utf8view_scalar(arr_a: &StringViewArray, value_b: &str) {
like(arr_a, &StringViewArray::new_scalar(value_b)).unwrap();
}
fn bench_nlike_utf8_scalar(arr_a: &StringArray, value_b: &str) {
nlike(arr_a, &StringArray::new_scalar(value_b)).unwrap();
}
fn bench_ilike_utf8_scalar(arr_a: &StringArray, value_b: &str) {
ilike(arr_a, &StringArray::new_scalar(value_b)).unwrap();
}
fn bench_nilike_utf8_scalar(arr_a: &StringArray, value_b: &str) {
nilike(arr_a, &StringArray::new_scalar(value_b)).unwrap();
}
fn bench_stringview_regexp_is_match_scalar(arr_a: &StringViewArray, value_b: &str) {
regexp_is_match_scalar(hint::black_box(arr_a), hint::black_box(value_b), None).unwrap();
}
fn bench_string_regexp_is_match_scalar(arr_a: &StringArray, value_b: &str) {
regexp_is_match_scalar(hint::black_box(arr_a), hint::black_box(value_b), None).unwrap();
}
fn make_string_array(size: usize, rng: &mut StdRng) -> impl Iterator<Item = Option<String>> + '_ {
(0..size).map(|_| {
let len = rng.random_range(0..64);
let bytes = (0..len).map(|_| rng.random_range(0..128)).collect();
Some(String::from_utf8(bytes).unwrap())
})
}
fn make_inlined_string_array(
size: usize,
rng: &mut StdRng,
) -> impl Iterator<Item = Option<String>> + '_ {
(0..size).map(|_| {
let len = rng.random_range(0..12);
let bytes = (0..len).map(|_| rng.random_range(0..128)).collect();
Some(String::from_utf8(bytes).unwrap())
})
}
fn add_benchmark(c: &mut Criterion) {
let arr_a = create_primitive_array_with_seed::<Float32Type>(SIZE, 0.0, 42);
let arr_b = create_primitive_array_with_seed::<Float32Type>(SIZE, 0.0, 43);
let arr_month_day_nano_a = create_month_day_nano_array_with_seed(SIZE, 0.0, 43);
let arr_month_day_nano_b = create_month_day_nano_array_with_seed(SIZE, 0.0, 43);
let arr_string = create_string_array::<i32>(SIZE, 0.0);
let arr_string_view = create_string_view_array(SIZE, 0.0);
// create long string arrays with the same prefix
let arr_long_string = create_longer_string_array_with_same_prefix::<i32>(SIZE, 0.0);
let arr_long_string_view = create_longer_string_view_array_with_same_prefix(SIZE, 0.0);
let left_arr_long_string = create_longer_string_array_with_same_prefix::<i32>(SIZE, 0.0);
let right_arr_long_string = create_longer_string_array_with_same_prefix::<i32>(SIZE, 0.0);
let left_arr_long_string_view = create_longer_string_view_array_with_same_prefix(SIZE, 0.0);
let right_arr_long_string_view = create_longer_string_view_array_with_same_prefix(SIZE, 0.0);
let scalar = Float32Array::from(vec![1.0]);
// eq benchmarks
c.bench_function("eq Float32", |b| b.iter(|| eq(&arr_a, &arr_b)));
c.bench_function("eq scalar Float32", |b| {
b.iter(|| eq(&arr_a, &Scalar::new(&scalar)).unwrap())
});
c.bench_function("neq Float32", |b| b.iter(|| neq(&arr_a, &arr_b)));
c.bench_function("neq scalar Float32", |b| {
b.iter(|| neq(&arr_a, &Scalar::new(&scalar)).unwrap())
});
c.bench_function("lt Float32", |b| b.iter(|| lt(&arr_a, &arr_b)));
c.bench_function("lt scalar Float32", |b| {
b.iter(|| lt(&arr_a, &Scalar::new(&scalar)).unwrap())
});
c.bench_function("lt_eq Float32", |b| b.iter(|| lt_eq(&arr_a, &arr_b)));
c.bench_function("lt_eq scalar Float32", |b| {
b.iter(|| lt_eq(&arr_a, &Scalar::new(&scalar)).unwrap())
});
c.bench_function("gt Float32", |b| b.iter(|| gt(&arr_a, &arr_b)));
c.bench_function("gt scalar Float32", |b| {
b.iter(|| gt(&arr_a, &Scalar::new(&scalar)).unwrap())
});
c.bench_function("gt_eq Float32", |b| b.iter(|| gt_eq(&arr_a, &arr_b)));
c.bench_function("gt_eq scalar Float32", |b| {
b.iter(|| gt_eq(&arr_a, &Scalar::new(&scalar)).unwrap())
});
let arr_a = create_primitive_array_with_seed::<Int32Type>(SIZE, 0.0, 42);
let arr_b = create_primitive_array_with_seed::<Int32Type>(SIZE, 0.0, 43);
let scalar = Int32Array::new_scalar(1);
c.bench_function("eq Int32", |b| b.iter(|| eq(&arr_a, &arr_b)));
c.bench_function("eq scalar Int32", |b| {
b.iter(|| eq(&arr_a, &scalar).unwrap())
});
c.bench_function("neq Int32", |b| b.iter(|| neq(&arr_a, &arr_b)));
c.bench_function("neq scalar Int32", |b| {
b.iter(|| neq(&arr_a, &scalar).unwrap())
});
c.bench_function("lt Int32", |b| b.iter(|| lt(&arr_a, &arr_b)));
c.bench_function("lt scalar Int32", |b| {
b.iter(|| lt(&arr_a, &scalar).unwrap())
});
c.bench_function("lt_eq Int32", |b| b.iter(|| lt_eq(&arr_a, &arr_b)));
c.bench_function("lt_eq scalar Int32", |b| {
b.iter(|| lt_eq(&arr_a, &scalar).unwrap())
});
c.bench_function("gt Int32", |b| b.iter(|| gt(&arr_a, &arr_b)));
c.bench_function("gt scalar Int32", |b| {
b.iter(|| gt(&arr_a, &scalar).unwrap())
});
c.bench_function("gt_eq Int32", |b| b.iter(|| gt_eq(&arr_a, &arr_b)));
c.bench_function("gt_eq scalar Int32", |b| {
b.iter(|| gt_eq(&arr_a, &scalar).unwrap())
});
c.bench_function("eq MonthDayNano", |b| {
b.iter(|| eq(&arr_month_day_nano_a, &arr_month_day_nano_b))
});
let scalar = IntervalMonthDayNanoArray::new_scalar(IntervalMonthDayNano::new(123, 0, 0));
c.bench_function("eq scalar MonthDayNano", |b| {
b.iter(|| eq(&arr_month_day_nano_b, &scalar).unwrap())
});
let mut rng = seedable_rng();
let mut array_gen = make_string_array(1024 * 1024 * 8, &mut rng);
let string_left = StringArray::from_iter(array_gen);
let string_view_left = StringViewArray::from_iter(string_left.iter());
// reference to the same rng to make sure we generate **different** array data,
// ow. the left and right will be identical
array_gen = make_string_array(1024 * 1024 * 8, &mut rng);
let string_right = StringArray::from_iter(array_gen);
let string_view_right = StringViewArray::from_iter(string_right.iter());
let string_scalar = StringArray::new_scalar("xxxx");
c.bench_function("eq scalar StringArray", |b| {
b.iter(|| eq(&string_scalar, &string_left).unwrap())
});
c.bench_function("lt scalar StringViewArray", |b| {
b.iter(|| {
lt(
&Scalar::new(StringViewArray::from_iter_values(["xxxx"])),
&string_view_left,
)
.unwrap()
})
});
c.bench_function("lt scalar StringArray", |b| {
b.iter(|| {
lt(
&Scalar::new(StringArray::from_iter_values(["xxxx"])),
&string_left,
)
.unwrap()
})
});
// StringViewArray has special handling for strings with length <= 12 and length <= 4
let string_view_scalar = StringViewArray::new_scalar("xxxx");
c.bench_function("eq scalar StringViewArray 4 bytes", |b| {
b.iter(|| eq(&string_view_scalar, &string_view_left).unwrap())
});
let string_view_scalar = StringViewArray::new_scalar("xxxxxx");
c.bench_function("eq scalar StringViewArray 6 bytes", |b| {
b.iter(|| eq(&string_view_scalar, &string_view_left).unwrap())
});
let string_view_scalar = StringViewArray::new_scalar("xxxxxxxxxxxxx");
c.bench_function("eq scalar StringViewArray 13 bytes", |b| {
b.iter(|| eq(&string_view_scalar, &string_view_left).unwrap())
});
c.bench_function("eq StringArray StringArray", |b| {
b.iter(|| eq(&string_left, &string_right).unwrap())
});
c.bench_function("eq StringViewArray StringViewArray", |b| {
b.iter(|| eq(&string_view_left, &string_view_right).unwrap())
});
let array_gen = make_inlined_string_array(1024 * 1024 * 8, &mut rng);
let string_left = StringArray::from_iter(array_gen);
let string_view_inlined_left = StringViewArray::from_iter(string_left.iter());
let array_gen = make_inlined_string_array(1024 * 1024 * 8, &mut rng);
let string_right = StringArray::from_iter(array_gen);
let string_view_inlined_right = StringViewArray::from_iter(string_right.iter());
// Add fast path benchmarks for StringViewArray, both side are inlined views < 12 bytes
c.bench_function("eq StringViewArray StringViewArray inlined bytes", |b| {
b.iter(|| eq(&string_view_inlined_left, &string_view_inlined_right).unwrap())
});
c.bench_function("lt StringViewArray StringViewArray inlined bytes", |b| {
b.iter(|| lt(&string_view_inlined_left, &string_view_inlined_right).unwrap())
});
// eq benchmarks for long strings with the same prefix
c.bench_function("eq long same prefix strings StringArray", |b| {
b.iter(|| eq(&left_arr_long_string, &right_arr_long_string).unwrap())
});
c.bench_function("neq long same prefix strings StringArray", |b| {
b.iter(|| neq(&left_arr_long_string, &right_arr_long_string).unwrap())
});
c.bench_function("lt long same prefix strings StringArray", |b| {
b.iter(|| lt(&left_arr_long_string, &right_arr_long_string).unwrap())
});
c.bench_function("eq long same prefix strings StringViewArray", |b| {
b.iter(|| eq(&left_arr_long_string_view, &right_arr_long_string_view).unwrap())
});
c.bench_function("neq long same prefix strings StringViewArray", |b| {
b.iter(|| neq(&left_arr_long_string_view, &right_arr_long_string_view).unwrap())
});
c.bench_function("lt long same prefix strings StringViewArray", |b| {
b.iter(|| lt(&left_arr_long_string_view, &right_arr_long_string_view).unwrap())
});
// StringArray: LIKE benchmarks
c.bench_function("like_utf8 scalar equals", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_string, "xxxx"))
});
c.bench_function("like_utf8 scalar contains", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_string, "%xxxx%"))
});
c.bench_function("like_utf8 scalar ends with", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_string, "%xxxx"))
});
c.bench_function("like_utf8 scalar starts with", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_string, "xxxx%"))
});
c.bench_function("like_utf8 scalar complex", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_string, "%xx_xx%xxx"))
});
// StringArray: LIKE benchmarks with long strings 4 bytes prefix
// Note:
// long strings mean strings start with same 4 bytes prefix such as "test",
// followed by a tail, ensuring the total length is greater than 12 bytes.
c.bench_function("long same prefix strings like_utf8 scalar equals", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_long_string, "prefix_1234"))
});
c.bench_function("long same prefix strings like_utf8 scalar contains", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_long_string, "%prefix_1234%"))
});
c.bench_function("long same prefix strings like_utf8 scalar ends with", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_long_string, "%prefix_1234"))
});
c.bench_function(
"long same prefix strings like_utf8 scalar starts with",
|b| b.iter(|| bench_like_utf8_scalar(&arr_long_string, "prefix_1234%")),
);
c.bench_function("long same prefix strings like_utf8 scalar complex", |b| {
b.iter(|| bench_like_utf8_scalar(&arr_long_string, "%prefix_1234%xxx"))
});
// StringViewArray: LIKE benchmarks with long strings 4 bytes prefix
// Note:
// long strings mean strings start with same 4 bytes prefix such as "test",
// followed by a tail, ensuring the total length is greater than 12 bytes.
c.bench_function(
"long same prefix strings like_utf8view scalar equals",
|b| b.iter(|| bench_like_utf8view_scalar(&arr_long_string_view, "prefix_1234")),
);
c.bench_function(
"long same prefix strings like_utf8view scalar contains",
|b| b.iter(|| bench_like_utf8view_scalar(&arr_long_string_view, "%prefix_1234%")),
);
c.bench_function(
"long same prefix strings like_utf8view scalar ends with",
|b| b.iter(|| bench_like_utf8view_scalar(&arr_long_string_view, "%prefix_1234")),
);
c.bench_function(
"long same prefix strings like_utf8view scalar starts with",
|b| b.iter(|| bench_like_utf8view_scalar(&arr_long_string_view, "prefix_1234%")),
);
c.bench_function(
"long same prefix strings like_utf8view scalar complex",
|b| b.iter(|| bench_like_utf8view_scalar(&arr_long_string_view, "%prefix_1234%xxx")),
);
// StringViewArray: LIKE benchmarks
// Note: since like/nlike share the same implementation, we only benchmark one
c.bench_function("like_utf8view scalar equals", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "xxxx"))
});
c.bench_function("like_utf8view scalar contains", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "%xxxx%"))
});
// StringView has special handling for strings with length <= 12 and length <= 4
c.bench_function("like_utf8view scalar ends with 4 bytes", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "%xxxx"))
});
c.bench_function("like_utf8view scalar ends with 6 bytes", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "%xxxxxx"))
});
c.bench_function("like_utf8view scalar ends with 13 bytes", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "%xxxxxxxxxxxxx"))
});
c.bench_function("like_utf8view scalar starts with 4 bytes", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "xxxx%"))
});
c.bench_function("like_utf8view scalar starts with 6 bytes", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "xxxxxx%"))
});
c.bench_function("like_utf8view scalar starts with 13 bytes", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "xxxxxxxxxxxxx%"))
});
c.bench_function("like_utf8view scalar complex", |b| {
b.iter(|| bench_like_utf8view_scalar(&string_view_left, "%xx_xx%xxx"))
});
// StringArray: NOT LIKE benchmarks
c.bench_function("nlike_utf8 scalar equals", |b| {
b.iter(|| bench_nlike_utf8_scalar(&arr_string, "xxxx"))
});
c.bench_function("nlike_utf8 scalar contains", |b| {
b.iter(|| bench_nlike_utf8_scalar(&arr_string, "%xxxx%"))
});
c.bench_function("nlike_utf8 scalar ends with", |b| {
b.iter(|| bench_nlike_utf8_scalar(&arr_string, "%xxxx"))
});
c.bench_function("nlike_utf8 scalar starts with", |b| {
b.iter(|| bench_nlike_utf8_scalar(&arr_string, "xxxx%"))
});
c.bench_function("nlike_utf8 scalar complex", |b| {
b.iter(|| bench_nlike_utf8_scalar(&arr_string, "%xx_xx%xxx"))
});
// StringArray: ILIKE benchmarks
c.bench_function("ilike_utf8 scalar equals", |b| {
b.iter(|| bench_ilike_utf8_scalar(&arr_string, "xxXX"))
});
c.bench_function("ilike_utf8 scalar contains", |b| {
b.iter(|| bench_ilike_utf8_scalar(&arr_string, "%xxXX%"))
});
c.bench_function("ilike_utf8 scalar ends with", |b| {
b.iter(|| bench_ilike_utf8_scalar(&arr_string, "%xXXx"))
});
c.bench_function("ilike_utf8 scalar starts with", |b| {
b.iter(|| bench_ilike_utf8_scalar(&arr_string, "XXXx%"))
});
c.bench_function("ilike_utf8 scalar complex", |b| {
b.iter(|| bench_ilike_utf8_scalar(&arr_string, "%xx_xX%xXX"))
});
// StringArray: NOT ILIKE benchmarks
c.bench_function("nilike_utf8 scalar equals", |b| {
b.iter(|| bench_nilike_utf8_scalar(&arr_string, "xxXX"))
});
c.bench_function("nilike_utf8 scalar contains", |b| {
b.iter(|| bench_nilike_utf8_scalar(&arr_string, "%xxXX%"))
});
c.bench_function("nilike_utf8 scalar ends with", |b| {
b.iter(|| bench_nilike_utf8_scalar(&arr_string, "%xXXx"))
});
c.bench_function("nilike_utf8 scalar starts with", |b| {
b.iter(|| bench_nilike_utf8_scalar(&arr_string, "XXXx%"))
});
c.bench_function("nilike_utf8 scalar complex", |b| {
b.iter(|| bench_nilike_utf8_scalar(&arr_string, "%xx_xX%xXX"))
});
// StringArray: regexp_matches_utf8 scalar benchmarks
let mut group =
c.benchmark_group("StringArray: regexp_matches_utf8 scalar benchmarks".to_string());
group
.bench_function("regexp_matches_utf8 scalar starts with", |b| {
b.iter(|| bench_string_regexp_is_match_scalar(&arr_string, "^xx"))
})
.bench_function("regexp_matches_utf8 scalar contains", |b| {
b.iter(|| bench_string_regexp_is_match_scalar(&arr_string, ".*xxXX.*"))
})
.bench_function("regexp_matches_utf8 scalar ends with", |b| {
b.iter(|| bench_string_regexp_is_match_scalar(&arr_string, "xx$"))
})
.bench_function("regexp_matches_utf8 scalar complex", |b| {
b.iter(|| bench_string_regexp_is_match_scalar(&arr_string, ".*x{2}.xX.*xXX"))
});
group.finish();
// StringViewArray: regexp_matches_utf8view scalar benchmarks
group =
c.benchmark_group("StringViewArray: regexp_matches_utf8view scalar benchmarks".to_string());
group
.bench_function("regexp_matches_utf8view scalar starts with", |b| {
b.iter(|| bench_stringview_regexp_is_match_scalar(&arr_string_view, "^xx"))
})
.bench_function("regexp_matches_utf8view scalar contains", |b| {
b.iter(|| bench_stringview_regexp_is_match_scalar(&arr_string_view, ".*xxXX.*"))
})
.bench_function("regexp_matches_utf8view scalar ends with", |b| {
b.iter(|| bench_stringview_regexp_is_match_scalar(&arr_string_view, "xx$"))
})
.bench_function("regexp_matches_utf8view scalar complex", |b| {
b.iter(|| bench_stringview_regexp_is_match_scalar(&arr_string_view, ".*x{2}.xX.*xXX"))
});
group.finish();
// DictionaryArray benchmarks
let strings = create_string_array::<i32>(20, 0.);
let dict_arr_a = create_dict_from_values::<Int32Type>(SIZE, 0., &strings);
let scalar = StringArray::from(vec!["test"]);
c.bench_function("eq_dyn_utf8_scalar dictionary[10] string[4])", |b| {
b.iter(|| eq(&dict_arr_a, &Scalar::new(&scalar)))
});
c.bench_function(
"gt_eq_dyn_utf8_scalar scalar dictionary[10] string[4])",
|b| b.iter(|| gt_eq(&dict_arr_a, &Scalar::new(&scalar))),
);
c.bench_function("like_utf8_scalar_dyn dictionary[10] string[4])", |b| {
b.iter(|| like(&dict_arr_a, &StringArray::new_scalar("test")))
});
c.bench_function("ilike_utf8_scalar_dyn dictionary[10] string[4])", |b| {
b.iter(|| ilike(&dict_arr_a, &StringArray::new_scalar("test")))
});
let strings = create_string_array::<i32>(20, 0.);
let dict_arr_a = create_dict_from_values::<Int32Type>(SIZE, 0., &strings);
let dict_arr_b = create_dict_from_values::<Int32Type>(SIZE, 0., &strings);
c.bench_function("eq dictionary[10] string[4])", |b| {
b.iter(|| eq(&dict_arr_a, &dict_arr_b).unwrap())
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+242
View File
@@ -0,0 +1,242 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
#[macro_use]
extern crate criterion;
use std::sync::Arc;
use criterion::Criterion;
use arrow::array::*;
use arrow::compute::concat;
use arrow::datatypes::*;
use arrow::util::bench_util::*;
use std::hint;
fn bench_concat(v1: &dyn Array, v2: &dyn Array) {
hint::black_box(concat(&[v1, v2]).unwrap());
}
fn bench_concat_arrays(arrays: &[&dyn Array]) {
hint::black_box(concat(arrays).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
let v1 = create_primitive_array::<Int32Type>(1024, 0.0);
let v2 = create_primitive_array::<Int32Type>(1024, 0.0);
c.bench_function("concat i32 1024", |b| b.iter(|| bench_concat(&v1, &v2)));
let v1 = create_primitive_array::<Int32Type>(1024, 0.5);
let v2 = create_primitive_array::<Int32Type>(1024, 0.5);
c.bench_function("concat i32 nulls 1024", |b| {
b.iter(|| bench_concat(&v1, &v2))
});
let small_array = create_primitive_array::<Int32Type>(4, 0.0);
let arrays: Vec<_> = (0..1024).map(|_| &small_array as &dyn Array).collect();
c.bench_function("concat 1024 arrays i32 4", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
{
let input = (0..100)
.map(|_| create_primitive_array::<Int32Type>(8192, 0.0))
.collect::<Vec<_>>();
let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn Array).collect();
c.bench_function("concat i32 8192 over 100 arrays", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
}
{
let input = (0..100)
.map(|_| create_primitive_array::<Int32Type>(8192, 0.5))
.collect::<Vec<_>>();
let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn Array).collect();
c.bench_function("concat i32 nulls 8192 over 100 arrays", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
}
let v1 = create_boolean_array(1024, 0.0, 0.5);
let v2 = create_boolean_array(1024, 0.0, 0.5);
c.bench_function("concat boolean 1024", |b| b.iter(|| bench_concat(&v1, &v2)));
let v1 = create_boolean_array(1024, 0.5, 0.5);
let v2 = create_boolean_array(1024, 0.5, 0.5);
c.bench_function("concat boolean nulls 1024", |b| {
b.iter(|| bench_concat(&v1, &v2))
});
let small_array = create_boolean_array(4, 0.0, 0.5);
let arrays: Vec<_> = (0..1024).map(|_| &small_array as &dyn Array).collect();
c.bench_function("concat 1024 arrays boolean 4", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
{
let input = (0..100)
.map(|_| create_boolean_array(8192, 0.0, 0.5))
.collect::<Vec<_>>();
let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn Array).collect();
c.bench_function("concat boolean 8192 over 100 arrays", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
}
{
let input = (0..100)
.map(|_| create_boolean_array(8192, 0.5, 0.5))
.collect::<Vec<_>>();
let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn Array).collect();
c.bench_function("concat boolean nulls 8192 over 100 arrays", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
}
let v1 = create_string_array::<i32>(1024, 0.0);
let v2 = create_string_array::<i32>(1024, 0.0);
c.bench_function("concat str 1024", |b| b.iter(|| bench_concat(&v1, &v2)));
let v1 = create_string_array::<i32>(1024, 0.5);
let v2 = create_string_array::<i32>(1024, 0.5);
c.bench_function("concat str nulls 1024", |b| {
b.iter(|| bench_concat(&v1, &v2))
});
let small_array = create_string_array::<i32>(4, 0.0);
let arrays: Vec<_> = (0..1024).map(|_| &small_array as &dyn Array).collect();
c.bench_function("concat 1024 arrays str 4", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
{
let input = (0..100)
.map(|_| create_string_array::<i32>(8192, 0.0))
.collect::<Vec<_>>();
let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn Array).collect();
c.bench_function("concat str 8192 over 100 arrays", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
}
{
let input = (0..100)
.map(|_| create_string_array::<i32>(8192, 0.5))
.collect::<Vec<_>>();
let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn Array).collect();
c.bench_function("concat str nulls 8192 over 100 arrays", |b| {
b.iter(|| bench_concat_arrays(&arrays))
});
}
// String view arrays
for null_density in [0.0, 0.2] {
// Any strings less than 12 characters are stored as prefix only, so specially
// benchmark cases that have different mixes of lengths.
for (name, str_len) in [("all_inline", 12), ("", 20), ("", 128)] {
let array = create_string_view_array_with_len(8192, null_density, str_len, false);
let arrays = (0..10).map(|_| &array as &dyn Array).collect::<Vec<_>>();
let id = format!(
"concat utf8_view {name} max_str_len={str_len} null_density={null_density}"
);
c.bench_function(&id, |b| b.iter(|| bench_concat_arrays(&arrays)));
}
}
let v1 = create_string_array_with_len::<i32>(10, 0.0, 20);
let v1 = create_dict_from_values::<Int32Type>(1024, 0.0, &v1);
let v2 = create_string_array_with_len::<i32>(10, 0.0, 20);
let v2 = create_dict_from_values::<Int32Type>(1024, 0.0, &v2);
c.bench_function("concat str_dict 1024", |b| {
b.iter(|| bench_concat(&v1, &v2))
});
let v1 = create_string_array_with_len::<i32>(1024, 0.0, 20);
let v1 = create_sparse_dict_from_values::<Int32Type>(1024, 0.0, &v1, 10..20);
let v2 = create_string_array_with_len::<i32>(1024, 0.0, 20);
let v2 = create_sparse_dict_from_values::<Int32Type>(1024, 0.0, &v2, 30..40);
c.bench_function("concat str_dict_sparse 1024", |b| {
b.iter(|| bench_concat(&v1, &v2))
});
let v1 = FixedSizeListArray::try_new(
Arc::new(Field::new_list_field(DataType::Int32, true)),
1024,
Arc::new(create_primitive_array::<Int32Type>(1024 * 1024, 0.0)),
None,
)
.unwrap();
let v2 = FixedSizeListArray::try_new(
Arc::new(Field::new_list_field(DataType::Int32, true)),
1024,
Arc::new(create_primitive_array::<Int32Type>(1024 * 1024, 0.0)),
None,
)
.unwrap();
c.bench_function("concat fixed size lists", |b| {
b.iter(|| bench_concat(&v1, &v2))
});
{
let batch_size = 1024;
let batch_count = 2;
let struct_arrays = (0..batch_count)
.map(|_| {
let ints = create_primitive_array::<Int32Type>(batch_size, 0.0);
let string_dict = create_sparse_dict_from_values::<Int32Type>(
batch_size,
0.0,
&create_string_array_with_len::<i32>(20, 0.0, 10),
0..10,
);
let int_dict = create_sparse_dict_from_values::<UInt16Type>(
batch_size,
0.0,
&create_primitive_array::<Int64Type>(20, 0.0),
0..10,
);
let fields = vec![
Field::new("int_field", ints.data_type().clone(), false),
Field::new("strings_dict_field", string_dict.data_type().clone(), false),
Field::new("int_dict_field", int_dict.data_type().clone(), false),
];
StructArray::try_new(
fields.clone().into(),
vec![Arc::new(ints), Arc::new(string_dict), Arc::new(int_dict)],
None,
)
.unwrap()
})
.collect::<Vec<_>>();
let array_refs = struct_arrays
.iter()
.map(|a| a as &dyn Array)
.collect::<Vec<_>>();
c.bench_function(
&format!("concat struct with int32 and dicts size={batch_size} count={batch_count}"),
|b| b.iter(|| bench_concat_arrays(&array_refs)),
);
}
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+184
View File
@@ -0,0 +1,184 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
extern crate criterion;
use std::io::Cursor;
use std::sync::Arc;
use arrow::util::bench_util::create_string_view_array_with_len;
use criterion::*;
use rand::Rng;
use arrow::array::*;
use arrow::csv;
use arrow::datatypes::*;
use arrow::util::bench_util::{create_primitive_array, create_string_array_with_len};
use arrow::util::test_util::seedable_rng;
fn do_bench(c: &mut Criterion, name: &str, cols: Vec<ArrayRef>) {
let batch = RecordBatch::try_from_iter(cols.into_iter().map(|a| ("col", a))).unwrap();
let mut buf = Vec::with_capacity(1024);
let mut csv = csv::Writer::new(&mut buf);
csv.write(&batch).unwrap();
drop(csv);
for batch_size in [128, 1024, 4096] {
c.bench_function(&format!("{name} - {batch_size}"), |b| {
b.iter(|| {
let cursor = Cursor::new(buf.as_slice());
let reader = csv::ReaderBuilder::new(batch.schema())
.with_batch_size(batch_size)
.with_header(true)
.build_buffered(cursor)
.unwrap();
for next in reader {
next.unwrap();
}
});
});
}
}
fn criterion_benchmark(c: &mut Criterion) {
let mut rng = seedable_rng();
// Single Primitive Column tests
let values = Int32Array::from_iter_values((0..4096).map(|_| rng.random_range(0..1024)));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 i32_small(0)", cols);
let values = Int32Array::from_iter_values((0..4096).map(|_| rng.random()));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 i32(0)", cols);
let values = UInt64Array::from_iter_values((0..4096).map(|_| rng.random_range(0..1024)));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 u64_small(0)", cols);
let values = UInt64Array::from_iter_values((0..4096).map(|_| rng.random()));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 u64(0)", cols);
let values = Int64Array::from_iter_values((0..4096).map(|_| rng.random_range(0..1024) - 512));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 i64_small(0)", cols);
let values = Int64Array::from_iter_values((0..4096).map(|_| rng.random()));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 i64(0)", cols);
let cols = vec![Arc::new(Float32Array::from_iter_values(
(0..4096).map(|_| rng.random_range(0..1024000) as f32 / 1000.),
)) as _];
do_bench(c, "4096 f32_small(0)", cols);
let values = Float32Array::from_iter_values((0..4096).map(|_| rng.random()));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 f32(0)", cols);
let cols = vec![Arc::new(Float64Array::from_iter_values(
(0..4096).map(|_| rng.random_range(0..1024000) as f64 / 1000.),
)) as _];
do_bench(c, "4096 f64_small(0)", cols);
let values = Float64Array::from_iter_values((0..4096).map(|_| rng.random()));
let cols = vec![Arc::new(values) as ArrayRef];
do_bench(c, "4096 f64(0)", cols);
// Single String Column tests
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0., 10)) as ArrayRef];
do_bench(c, "4096 string(10, 0)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0., 30)) as ArrayRef];
do_bench(c, "4096 string(30, 0)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0., 100)) as ArrayRef];
do_bench(c, "4096 string(100, 0)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0.5, 100)) as ArrayRef];
do_bench(c, "4096 string(100, 0.5)", cols);
// Single StringView Column tests
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0., 10, false)) as ArrayRef];
do_bench(c, "4096 StringView(10, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0., 30, false)) as ArrayRef];
do_bench(c, "4096 StringView(30, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0., 100, false)) as ArrayRef];
do_bench(c, "4096 StringView(100, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0.5, 100, false)) as ArrayRef];
do_bench(c, "4096 StringView(100, 0.5)", cols);
// Multi-Column(with String) tests
let cols = vec![
Arc::new(create_string_array_with_len::<i32>(4096, 0.5, 20)) as ArrayRef,
Arc::new(create_string_array_with_len::<i32>(4096, 0., 30)) as ArrayRef,
Arc::new(create_string_array_with_len::<i32>(4096, 0., 100)) as ArrayRef,
Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef,
];
do_bench(
c,
"4096 string(20, 0.5), string(30, 0), string(100, 0), i64(0)",
cols,
);
let cols = vec![
Arc::new(create_string_array_with_len::<i32>(4096, 0.5, 20)) as ArrayRef,
Arc::new(create_string_array_with_len::<i32>(4096, 0., 30)) as ArrayRef,
Arc::new(create_primitive_array::<Float64Type>(4096, 0.)) as ArrayRef,
Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef,
];
do_bench(
c,
"4096 string(20, 0.5), string(30, 0), f64(0), i64(0)",
cols,
);
// Multi-Column(with StringView) tests
let cols = vec![
Arc::new(create_string_view_array_with_len(4096, 0.5, 20, false)) as ArrayRef,
Arc::new(create_string_view_array_with_len(4096, 0., 30, false)) as ArrayRef,
Arc::new(create_string_view_array_with_len(4096, 0., 100, false)) as ArrayRef,
Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef,
];
do_bench(
c,
"4096 StringView(20, 0.5), StringView(30, 0), StringView(100, 0), i64(0)",
cols,
);
let cols = vec![
Arc::new(create_string_view_array_with_len(4096, 0.5, 20, false)) as ArrayRef,
Arc::new(create_string_view_array_with_len(4096, 0., 30, false)) as ArrayRef,
Arc::new(create_primitive_array::<Float64Type>(4096, 0.)) as ArrayRef,
Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef,
];
do_bench(
c,
"4096 StringView(20, 0.5), StringView(30, 0), f64(0), i64(0)",
cols,
);
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
+69
View File
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
extern crate criterion;
use criterion::*;
use arrow::array::*;
use arrow::csv;
use arrow::datatypes::*;
use std::env;
use std::fs::File;
use std::hint;
use std::sync::Arc;
fn criterion_benchmark(c: &mut Criterion) {
let schema = Schema::new(vec![
Field::new("c1", DataType::Utf8, false),
Field::new("c2", DataType::Float64, true),
Field::new("c3", DataType::UInt32, false),
Field::new("c4", DataType::Boolean, true),
]);
let c1 = StringArray::from(vec![
"Lorem ipsum dolor sit amet",
"consectetur adipiscing elit",
"sed do eiusmod tempor",
]);
let c2 = PrimitiveArray::<Float64Type>::from(vec![Some(123.564532), None, Some(-556132.25)]);
let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
let b = RecordBatch::try_new(
Arc::new(schema),
vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
)
.unwrap();
let path = env::temp_dir().join("bench_write_csv.csv");
let file = File::create(path).unwrap();
let mut writer = csv::Writer::new(file);
let batches = vec![&b, &b, &b, &b, &b, &b, &b, &b, &b, &b, &b];
c.bench_function("record_batches_to_csv", |b| {
b.iter(|| {
#[allow(clippy::unit_arg)]
hint::black_box(for batch in &batches {
writer.write(batch).unwrap()
});
});
});
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
+137
View File
@@ -0,0 +1,137 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use arrow::array::{
Array, Decimal32Array, Decimal32Builder, Decimal64Array, Decimal64Builder, Decimal128Array,
Decimal128Builder, Decimal256Array, Decimal256Builder,
};
use criterion::Criterion;
use rand::Rng;
extern crate arrow;
use arrow_buffer::i256;
fn validate_decimal32_array(array: Decimal32Array) {
array.with_precision_and_scale(8, 0).unwrap();
}
fn validate_decimal64_array(array: Decimal64Array) {
array.with_precision_and_scale(16, 0).unwrap();
}
fn validate_decimal128_array(array: Decimal128Array) {
array.with_precision_and_scale(35, 0).unwrap();
}
fn validate_decimal256_array(array: Decimal256Array) {
array.with_precision_and_scale(35, 0).unwrap();
}
fn validate_decimal32_benchmark(c: &mut Criterion) {
let mut rng = rand::rng();
let size: i32 = 20000;
let mut decimal_builder = Decimal32Builder::with_capacity(size as usize);
for _ in 0..size {
decimal_builder.append_value(rng.random_range::<i32, _>(0..99999999));
}
let decimal_array = decimal_builder
.finish()
.with_precision_and_scale(9, 0)
.unwrap();
let data = decimal_array.into_data();
c.bench_function("validate_decimal32_array 20000", |b| {
b.iter(|| {
let array = Decimal32Array::from(data.clone());
validate_decimal32_array(array);
})
});
}
fn validate_decimal64_benchmark(c: &mut Criterion) {
let mut rng = rand::rng();
let size: i64 = 20000;
let mut decimal_builder = Decimal64Builder::with_capacity(size as usize);
for _ in 0..size {
decimal_builder.append_value(rng.random_range::<i64, _>(0..999999999999));
}
let decimal_array = decimal_builder
.finish()
.with_precision_and_scale(18, 0)
.unwrap();
let data = decimal_array.into_data();
c.bench_function("validate_decimal64_array 20000", |b| {
b.iter(|| {
let array = Decimal64Array::from(data.clone());
validate_decimal64_array(array);
})
});
}
fn validate_decimal128_benchmark(c: &mut Criterion) {
let mut rng = rand::rng();
let size: i128 = 20000;
let mut decimal_builder = Decimal128Builder::with_capacity(size as usize);
for _ in 0..size {
decimal_builder.append_value(rng.random_range::<i128, _>(0..999999999999));
}
let decimal_array = decimal_builder
.finish()
.with_precision_and_scale(38, 0)
.unwrap();
let data = decimal_array.into_data();
c.bench_function("validate_decimal128_array 20000", |b| {
b.iter(|| {
let array = Decimal128Array::from(data.clone());
validate_decimal128_array(array);
})
});
}
fn validate_decimal256_benchmark(c: &mut Criterion) {
let mut rng = rand::rng();
let size: i128 = 20000;
let mut decimal_builder = Decimal256Builder::with_capacity(size as usize);
for _ in 0..size {
let v = rng.random_range::<i128, _>(0..999999999999999);
let decimal = i256::from_i128(v);
decimal_builder.append_value(decimal);
}
let decimal_array256_data = decimal_builder
.finish()
.with_precision_and_scale(76, 0)
.unwrap();
let data = decimal_array256_data.into_data();
c.bench_function("validate_decimal256_array 20000", |b| {
b.iter(|| {
let array = Decimal256Array::from(data.clone());
validate_decimal256_array(array);
})
});
}
criterion_group!(
benches,
validate_decimal32_benchmark,
validate_decimal64_benchmark,
validate_decimal128_benchmark,
validate_decimal256_benchmark,
);
criterion_main!(benches);
+61
View File
@@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// Allowed because we use `arr == arr` in benchmarks
#![allow(clippy::eq_op)]
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::util::bench_util::*;
use arrow::{array::*, datatypes::Float32Type};
use std::hint;
fn bench_equal<A: Array + PartialEq<A>>(arr_a: &A) {
hint::black_box(arr_a == arr_a);
}
fn add_benchmark(c: &mut Criterion) {
let arr_a = create_primitive_array::<Float32Type>(512, 0.0);
c.bench_function("equal_512", |b| b.iter(|| bench_equal(&arr_a)));
let arr_a_nulls = create_primitive_array::<Float32Type>(512, 0.5);
c.bench_function("equal_nulls_512", |b| b.iter(|| bench_equal(&arr_a_nulls)));
let arr_a = create_primitive_array::<Float32Type>(51200, 0.1);
c.bench_function("equal_51200", |b| b.iter(|| bench_equal(&arr_a)));
let arr_a = create_string_array::<i32>(512, 0.0);
c.bench_function("equal_string_512", |b| b.iter(|| bench_equal(&arr_a)));
let arr_a_nulls = create_string_array::<i32>(512, 0.5);
c.bench_function("equal_string_nulls_512", |b| {
b.iter(|| bench_equal(&arr_a_nulls))
});
let arr_a = create_boolean_array(512, 0.0, 0.5);
c.bench_function("equal_bool_512", |b| b.iter(|| bench_equal(&arr_a)));
let arr_a = create_boolean_array(513, 0.0, 0.5);
c.bench_function("equal_bool_513", |b| b.iter(|| bench_equal(&arr_a)));
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+301
View File
@@ -0,0 +1,301 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
extern crate arrow;
use std::sync::Arc;
use arrow::compute::{FilterBuilder, FilterPredicate, filter_record_batch};
use arrow::util::bench_util::*;
use arrow::array::*;
use arrow::compute::filter;
use arrow::datatypes::{Field, Float32Type, Int32Type, Int64Type, Schema, UInt8Type};
use arrow_array::types::Decimal128Type;
use criterion::{Criterion, criterion_group, criterion_main};
use std::hint;
fn bench_filter(data_array: &dyn Array, filter_array: &BooleanArray) {
hint::black_box(filter(data_array, filter_array).unwrap());
}
fn bench_built_filter(filter: &FilterPredicate, array: &dyn Array) {
hint::black_box(filter.filter(array).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
let size = 65536;
let filter_array = create_boolean_array(size, 0.0, 0.5);
let dense_filter_array = create_boolean_array(size, 0.0, 1.0 - 1.0 / 1024.0);
let sparse_filter_array = create_boolean_array(size, 0.0, 1.0 / 1024.0);
let filter = FilterBuilder::new(&filter_array).optimize().build();
let dense_filter = FilterBuilder::new(&dense_filter_array).optimize().build();
let sparse_filter = FilterBuilder::new(&sparse_filter_array).optimize().build();
let data_array = create_primitive_array::<UInt8Type>(size, 0.0);
c.bench_function("filter optimize (kept 1/2)", |b| {
b.iter(|| FilterBuilder::new(&filter_array).optimize().build())
});
c.bench_function("filter optimize high selectivity (kept 1023/1024)", |b| {
b.iter(|| FilterBuilder::new(&dense_filter_array).optimize().build())
});
c.bench_function("filter optimize low selectivity (kept 1/1024)", |b| {
b.iter(|| FilterBuilder::new(&sparse_filter_array).optimize().build())
});
c.bench_function("filter u8 (kept 1/2)", |b| {
b.iter(|| bench_filter(&data_array, &filter_array))
});
c.bench_function("filter u8 high selectivity (kept 1023/1024)", |b| {
b.iter(|| bench_filter(&data_array, &dense_filter_array))
});
c.bench_function("filter u8 low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_filter(&data_array, &sparse_filter_array))
});
c.bench_function("filter context u8 (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function("filter context u8 high selectivity (kept 1023/1024)", |b| {
b.iter(|| bench_built_filter(&dense_filter, &data_array))
});
c.bench_function("filter context u8 low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_built_filter(&sparse_filter, &data_array))
});
let data_array = create_primitive_array::<Int32Type>(size, 0.0);
c.bench_function("filter i32 (kept 1/2)", |b| {
b.iter(|| bench_filter(&data_array, &filter_array))
});
c.bench_function("filter i32 high selectivity (kept 1023/1024)", |b| {
b.iter(|| bench_filter(&data_array, &dense_filter_array))
});
c.bench_function("filter i32 low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_filter(&data_array, &sparse_filter_array))
});
c.bench_function("filter context i32 (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context i32 high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function("filter context i32 low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_built_filter(&sparse_filter, &data_array))
});
let data_array = create_primitive_array::<Int32Type>(size, 0.5);
c.bench_function("filter context i32 w NULLs (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context i32 w NULLs high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context i32 w NULLs low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let data_array = create_primitive_array::<UInt8Type>(size, 0.5);
c.bench_function("filter context u8 w NULLs (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context u8 w NULLs high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context u8 w NULLs low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let data_array = create_primitive_array::<Float32Type>(size, 0.5);
c.bench_function("filter f32 (kept 1/2)", |b| {
b.iter(|| bench_filter(&data_array, &filter_array))
});
c.bench_function("filter context f32 (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context f32 high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function("filter context f32 low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_built_filter(&sparse_filter, &data_array))
});
let data_array = create_primitive_array::<Decimal128Type>(size, 0.0);
c.bench_function("filter decimal128 (kept 1/2)", |b| {
b.iter(|| bench_filter(&data_array, &filter_array))
});
c.bench_function("filter decimal128 high selectivity (kept 1023/1024)", |b| {
b.iter(|| bench_filter(&data_array, &dense_filter_array))
});
c.bench_function("filter decimal128 low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_filter(&data_array, &sparse_filter_array))
});
c.bench_function("filter context decimal128 (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context decimal128 high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context decimal128 low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let data_array = create_string_array::<i32>(size, 0.5);
c.bench_function("filter context string (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context string high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function("filter context string low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_built_filter(&sparse_filter, &data_array))
});
let data_array = create_string_dict_array::<Int32Type>(size, 0.0, 4);
c.bench_function("filter context string dictionary (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context string dictionary high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context string dictionary low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let data_array = create_string_dict_array::<Int32Type>(size, 0.5, 4);
c.bench_function("filter context string dictionary w NULLs (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context string dictionary w NULLs high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context string dictionary w NULLs low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let mut add_benchmark_for_fsb_with_length = |value_length: usize| {
let data_array = create_fsb_array(size, 0.0, value_length);
c.bench_function(
format!("filter fsb with value length {value_length} (kept 1/2)").as_str(),
|b| b.iter(|| bench_filter(&data_array, &filter_array)),
);
c.bench_function(
format!(
"filter fsb with value length {value_length} high selectivity (kept 1023/1024)"
)
.as_str(),
|b| b.iter(|| bench_filter(&data_array, &dense_filter_array)),
);
c.bench_function(
format!("filter fsb with value length {value_length} low selectivity (kept 1/1024)")
.as_str(),
|b| b.iter(|| bench_filter(&data_array, &sparse_filter_array)),
);
c.bench_function(
format!("filter context fsb with value length {value_length} (kept 1/2)").as_str(),
|b| b.iter(|| bench_built_filter(&filter, &filter_array)),
);
c.bench_function(
format!(
"filter context fsb with value length {value_length} high selectivity (kept 1023/1024)"
)
.as_str(),
|b| b.iter(|| bench_built_filter(&filter, &dense_filter_array)),
);
c.bench_function(
format!(
"filter context fsb with value length {value_length} low selectivity (kept 1/1024)"
)
.as_str(),
|b| b.iter(|| bench_built_filter(&filter, &sparse_filter_array)),
);
};
add_benchmark_for_fsb_with_length(5);
add_benchmark_for_fsb_with_length(20);
add_benchmark_for_fsb_with_length(50);
let data_array = create_primitive_array::<Float32Type>(size, 0.0);
let field = Field::new("c1", data_array.data_type().clone(), true);
let schema = Schema::new(vec![field]);
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(data_array)]).unwrap();
c.bench_function("filter single record batch", |b| {
b.iter(|| filter_record_batch(&batch, &filter_array))
});
let data_array = create_string_view_array_with_len(size, 0.5, 4, false);
c.bench_function("filter context short string view (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context short string view high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context short string view low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let data_array = create_string_view_array_with_len(size, 0.5, 4, true);
c.bench_function("filter context mixed string view (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function(
"filter context mixed string view high selectivity (kept 1023/1024)",
|b| b.iter(|| bench_built_filter(&dense_filter, &data_array)),
);
c.bench_function(
"filter context mixed string view low selectivity (kept 1/1024)",
|b| b.iter(|| bench_built_filter(&sparse_filter, &data_array)),
);
let data_array = create_primitive_run_array::<Int32Type, Int64Type>(size, size);
c.bench_function("filter run array (kept 1/2)", |b| {
b.iter(|| bench_built_filter(&filter, &data_array))
});
c.bench_function("filter run array high selectivity (kept 1023/1024)", |b| {
b.iter(|| bench_built_filter(&dense_filter, &data_array))
});
c.bench_function("filter run array low selectivity (kept 1/1024)", |b| {
b.iter(|| bench_built_filter(&sparse_filter, &data_array))
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+172
View File
@@ -0,0 +1,172 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use std::ops::Range;
use rand::Rng;
extern crate arrow;
use arrow::datatypes::*;
use arrow::util::test_util::seedable_rng;
use arrow::{array::*, util::bench_util::*};
use arrow_select::interleave::interleave;
use std::hint;
use std::sync::Arc;
fn do_bench(
c: &mut Criterion,
prefix: &str,
len: usize,
base: &dyn Array,
slices: &[Range<usize>],
) {
let arrays: Vec<_> = slices
.iter()
.map(|r| base.slice(r.start, r.end - r.start))
.collect();
let values: Vec<_> = arrays.iter().map(|x| x.as_ref()).collect();
bench_values(
c,
&format!("interleave {prefix} {len} {slices:?}"),
len,
&values,
);
}
fn bench_values(c: &mut Criterion, name: &str, len: usize, values: &[&dyn Array]) {
let mut rng = seedable_rng();
let indices: Vec<_> = (0..len)
.map(|_| {
let array_idx = rng.random_range(0..values.len());
let value_idx = rng.random_range(0..values[array_idx].len());
(array_idx, value_idx)
})
.collect();
c.bench_function(name, |b| {
b.iter(|| hint::black_box(interleave(values, &indices).unwrap()))
});
}
fn add_benchmark(c: &mut Criterion) {
let i32 = create_primitive_array::<Int32Type>(1024, 0.);
let i32_opt = create_primitive_array::<Int32Type>(1024, 0.5);
let string = create_string_array_with_len::<i32>(1024, 0., 20);
let string_opt = create_string_array_with_len::<i32>(1024, 0.5, 20);
let values = create_string_array_with_len::<i32>(10, 0.0, 20);
let dict = create_dict_from_values::<Int32Type>(1024, 0.0, &values);
let struct_i32_no_nulls_i32_no_nulls = StructArray::new(
Fields::from(vec![
Field::new("a", Int32Type::DATA_TYPE, false),
Field::new("b", Int32Type::DATA_TYPE, false),
]),
vec![
Arc::new(create_primitive_array::<Int32Type>(1024, 0.)),
Arc::new(create_primitive_array::<Int32Type>(1024, 0.)),
],
None,
);
let struct_string_no_nulls_string_no_nulls = StructArray::new(
Fields::from(vec![
Field::new("a", DataType::Utf8, false),
Field::new("b", DataType::Utf8, false),
]),
vec![
Arc::new(create_string_array_with_len::<i32>(1024, 0., 20)),
Arc::new(create_string_array_with_len::<i32>(1024, 0., 20)),
],
None,
);
let struct_i32_no_nulls_string_no_nulls = StructArray::new(
Fields::from(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Utf8, false),
]),
vec![
Arc::new(create_primitive_array::<Int32Type>(1024, 0.)),
Arc::new(create_string_array_with_len::<i32>(1024, 0., 20)),
],
None,
);
let values = create_string_array_with_len::<i32>(1024, 0.0, 20);
let sparse_dict = create_sparse_dict_from_values::<Int32Type>(1024, 0.0, &values, 10..20);
let string_view = create_string_view_array(1024, 0.0);
// use 8192 as a standard list size for better coverage
let list_i64 = create_primitive_list_array_with_seed::<i32, Int64Type>(8192, 0.1, 0.1, 20, 42);
let list_i64_no_nulls =
create_primitive_list_array_with_seed::<i32, Int64Type>(8192, 0.0, 0.0, 20, 42);
let cases: &[(&str, &dyn Array)] = &[
("i32(0.0)", &i32),
("i32(0.5)", &i32_opt),
("str(20, 0.0)", &string),
("str(20, 0.5)", &string_opt),
("dict(20, 0.0)", &dict),
("dict_sparse(20, 0.0)", &sparse_dict),
("str_view(0.0)", &string_view),
(
"struct(i32(0.0), i32(0.0)",
&struct_i32_no_nulls_i32_no_nulls,
),
(
"struct(str(20, 0.0), str(20, 0.0))",
&struct_string_no_nulls_string_no_nulls,
),
(
"struct(i32(0.0), str(20, 0.0)",
&struct_i32_no_nulls_string_no_nulls,
),
("list<i64>(0.1,0.1,20)", &list_i64),
("list<i64>(0.0,0.0,20)", &list_i64_no_nulls),
];
for (prefix, base) in cases {
let slices: &[(usize, &[_])] = &[
(100, &[0..100, 100..230, 450..1000]),
(400, &[0..100, 100..230, 450..1000]),
(1024, &[0..100, 100..230, 450..1000]),
(1024, &[0..100, 100..230, 450..1000, 0..1000]),
];
for (len, slice) in slices {
do_bench(c, prefix, *len, *base, slice);
}
}
for len in [100, 1024, 2048] {
bench_values(
c,
&format!("interleave dict_distinct {len}"),
100,
&[&dict, &sparse_dict],
);
}
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+179
View File
@@ -0,0 +1,179 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use criterion::*;
use arrow::datatypes::*;
use arrow::util::bench_util::{
create_primitive_array, create_string_array, create_string_array_with_len,
};
use arrow_array::RecordBatch;
use arrow_json::{LineDelimitedWriter, ReaderBuilder};
use std::hint;
use std::io::Cursor;
use std::sync::Arc;
#[allow(deprecated)]
fn do_bench(c: &mut Criterion, name: &str, json: &str, schema: SchemaRef) {
c.bench_function(name, |b| {
b.iter(|| {
let cursor = Cursor::new(hint::black_box(json));
let builder = ReaderBuilder::new(schema.clone()).with_batch_size(64);
let reader = builder.build(cursor).unwrap();
for next in reader {
next.unwrap();
}
})
});
}
fn small_bench_primitive(c: &mut Criterion) {
let schema = Arc::new(Schema::new(vec![
Field::new("c1", DataType::Utf8, true),
Field::new("c2", DataType::Float64, true),
Field::new("c3", DataType::UInt32, true),
Field::new("c4", DataType::Boolean, true),
]));
let json_content = r#"
{"c1": "eleven", "c2": 6.2222222225, "c3": 5.0, "c4": false}
{"c1": "twelve", "c2": -55555555555555.2, "c3": 3}
{"c1": null, "c2": 3, "c3": 125, "c4": null}
{"c2": -35, "c3": 100.0, "c4": true}
{"c1": "fifteen", "c2": null, "c4": true}
{"c1": "eleven", "c2": 6.2222222225, "c3": 5.0, "c4": false}
{"c1": "twelve", "c2": -55555555555555.2, "c3": 3}
{"c1": null, "c2": 3, "c3": 125, "c4": null}
{"c2": -35, "c3": 100.0, "c4": true}
{"c1": "fifteen", "c2": null, "c4": true}
"#;
do_bench(c, "small_bench_primitive", json_content, schema)
}
fn small_bench_primitive_with_utf8view(c: &mut Criterion) {
let schema = Arc::new(Schema::new(vec![
Field::new("c1", DataType::Utf8View, true),
Field::new("c2", DataType::Float64, true),
Field::new("c3", DataType::UInt32, true),
Field::new("c4", DataType::Boolean, true),
]));
let json_content = r#"
{"c1": "eleven", "c2": 6.2222222225, "c3": 5.0, "c4": false}
{"c1": "twelve", "c2": -55555555555555.2, "c3": 3}
{"c1": null, "c2": 3, "c3": 125, "c4": null}
{"c2": -35, "c3": 100.0, "c4": true}
{"c1": "fifteen", "c2": null, "c4": true}
{"c1": "eleven", "c2": 6.2222222225, "c3": 5.0, "c4": false}
{"c1": "twelve", "c2": -55555555555555.2, "c3": 3}
{"c1": null, "c2": 3, "c3": 125, "c4": null}
{"c2": -35, "c3": 100.0, "c4": true}
{"c1": "fifteen", "c2": null, "c4": true}
"#;
do_bench(
c,
"small_bench_primitive_with_utf8view",
json_content,
schema,
)
}
fn large_bench_primitive(c: &mut Criterion) {
let schema = Arc::new(Schema::new(vec![
Field::new("c1", DataType::Utf8, true),
Field::new("c2", DataType::Int32, true),
Field::new("c3", DataType::UInt32, true),
Field::new("c4", DataType::Utf8, true),
Field::new("c5", DataType::Utf8, true),
Field::new("c6", DataType::Float32, true),
]));
let c1 = Arc::new(create_string_array::<i32>(4096, 0.));
let c2 = Arc::new(create_primitive_array::<Int32Type>(4096, 0.));
let c3 = Arc::new(create_primitive_array::<UInt32Type>(4096, 0.));
let c4 = Arc::new(create_string_array_with_len::<i32>(4096, 0.2, 10));
let c5 = Arc::new(create_string_array_with_len::<i32>(4096, 0.2, 20));
let c6 = Arc::new(create_primitive_array::<Float32Type>(4096, 0.2));
let batch = RecordBatch::try_from_iter([
("c1", c1 as _),
("c2", c2 as _),
("c3", c3 as _),
("c4", c4 as _),
("c5", c5 as _),
("c6", c6 as _),
])
.unwrap();
let mut out = Vec::with_capacity(1024);
LineDelimitedWriter::new(&mut out).write(&batch).unwrap();
let json = std::str::from_utf8(&out).unwrap();
do_bench(c, "large_bench_primitive", json, schema)
}
fn small_bench_list(c: &mut Criterion) {
let schema = Arc::new(Schema::new(vec![
Field::new(
"c1",
DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),
true,
),
Field::new(
"c2",
DataType::List(Arc::new(Field::new_list_field(DataType::Float64, true))),
true,
),
Field::new(
"c3",
DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, true))),
true,
),
Field::new(
"c4",
DataType::List(Arc::new(Field::new_list_field(DataType::Boolean, true))),
true,
),
]));
let json = r#"
{"c1": ["eleven"], "c2": [6.2222222225, -3.2, null], "c3": [5.0, 6], "c4": [false, true]}
{"c1": ["twelve"], "c2": [-55555555555555.2, 12500000.0], "c3": [3, 4, 5]}
{"c1": null, "c2": [3], "c3": [125, 127, 129], "c4": [null, false, true]}
{"c2": [-35], "c3": [100.0, 200.0], "c4": null}
{"c1": ["fifteen"], "c2": [null, 2.1, 1.5, -3], "c4": [true, false, null]}
{"c1": ["fifteen"], "c2": [], "c4": [true, false, null]}
{"c1": ["eleven"], "c2": [6.2222222225, -3.2, null], "c3": [5.0, 6], "c4": [false, true]}
{"c1": ["twelve"], "c2": [-55555555555555.2, 12500000.0], "c3": [3, 4, 5]}
{"c1": null, "c2": [3], "c3": [125, 127, 129], "c4": [null, false, true]}
{"c2": [-35], "c3": [100.0, 200.0], "c4": null}
{"c1": ["fifteen"], "c2": [null, 2.1, 1.5, -3], "c4": [true, false, null]}
{"c1": ["fifteen"], "c2": [], "c4": [true, false, null]}
"#;
do_bench(c, "small_bench_list", json, schema)
}
fn criterion_benchmark(c: &mut Criterion) {
small_bench_primitive(c);
large_bench_primitive(c);
small_bench_list(c);
small_bench_primitive_with_utf8view(c);
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
+336
View File
@@ -0,0 +1,336 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use criterion::*;
use arrow::datatypes::*;
use arrow::util::bench_util::{
create_primitive_array, create_string_array, create_string_array_with_len,
create_string_dict_array,
};
use arrow::util::test_util::seedable_rng;
use arrow_array::{Array, ListArray, RecordBatch, StructArray};
use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
use arrow_json::{LineDelimitedWriter, ReaderBuilder};
use rand::Rng;
use serde::Serialize;
use std::sync::Arc;
const NUM_ROWS: usize = 65536;
fn do_bench(c: &mut Criterion, name: &str, batch: &RecordBatch) {
c.bench_function(name, |b| {
b.iter(|| {
let mut out = Vec::with_capacity(1024);
LineDelimitedWriter::new(&mut out).write(batch).unwrap();
out
})
});
}
fn create_mixed(len: usize) -> RecordBatch {
let c1 = Arc::new(create_string_array::<i32>(len, 0.));
let c2 = Arc::new(create_primitive_array::<Int32Type>(len, 0.));
let c3 = Arc::new(create_primitive_array::<UInt32Type>(len, 0.));
let c4 = Arc::new(create_string_array_with_len::<i32>(len, 0.2, 10));
let c5 = Arc::new(create_string_array_with_len::<i32>(len, 0.2, 20));
let c6 = Arc::new(create_primitive_array::<Float32Type>(len, 0.2));
RecordBatch::try_from_iter([
("c1", c1 as _),
("c2", c2 as _),
("c3", c3 as _),
("c4", c4 as _),
("c5", c5 as _),
("c6", c6 as _),
])
.unwrap()
}
fn create_nulls(len: usize) -> NullBuffer {
let mut rng = seedable_rng();
BooleanBuffer::from_iter((0..len).map(|_| rng.random_bool(0.2))).into()
}
fn create_offsets(len: usize) -> (usize, OffsetBuffer<i32>) {
let mut rng = seedable_rng();
let mut last_offset = 0;
let mut offsets = Vec::with_capacity(len + 1);
offsets.push(0);
for _ in 0..len {
let len = rng.random_range(0..10);
offsets.push(last_offset + len);
last_offset += len;
}
(
*offsets.last().unwrap() as _,
OffsetBuffer::new(offsets.into()),
)
}
fn create_nullable_struct(len: usize) -> StructArray {
let c2 = StructArray::from(create_mixed(len));
StructArray::new(
c2.fields().clone(),
c2.columns().to_vec(),
Some(create_nulls(c2.len())),
)
}
fn bench_float(c: &mut Criterion) {
let c1 = Arc::new(create_primitive_array::<Float32Type>(NUM_ROWS, 0.));
let c2 = Arc::new(create_primitive_array::<Float64Type>(NUM_ROWS, 0.));
let batch = RecordBatch::try_from_iter([("c1", c1 as _), ("c2", c2 as _)]).unwrap();
do_bench(c, "bench_float", &batch)
}
fn bench_integer(c: &mut Criterion) {
let c1 = Arc::new(create_primitive_array::<UInt64Type>(NUM_ROWS, 0.));
let c2 = Arc::new(create_primitive_array::<Int32Type>(NUM_ROWS, 0.));
let c3 = Arc::new(create_primitive_array::<UInt32Type>(NUM_ROWS, 0.));
let batch =
RecordBatch::try_from_iter([("c1", c1 as _), ("c2", c2 as _), ("c3", c3 as _)]).unwrap();
do_bench(c, "bench_integer", &batch)
}
fn bench_mixed(c: &mut Criterion) {
let batch = create_mixed(NUM_ROWS);
do_bench(c, "bench_mixed", &batch)
}
fn bench_dict_array(c: &mut Criterion) {
let c1 = Arc::new(create_string_dict_array::<Int32Type>(NUM_ROWS, 0., 30));
let c2 = Arc::new(create_string_dict_array::<Int32Type>(NUM_ROWS, 0., 20));
let c3 = Arc::new(create_string_dict_array::<Int32Type>(NUM_ROWS, 0.1, 20));
let batch =
RecordBatch::try_from_iter([("c1", c1 as _), ("c2", c2 as _), ("c3", c3 as _)]).unwrap();
do_bench(c, "bench_dict_array", &batch)
}
fn bench_string(c: &mut Criterion) {
let c1 = Arc::new(create_string_array::<i32>(NUM_ROWS, 0.));
let c2 = Arc::new(create_string_array_with_len::<i32>(NUM_ROWS, 0., 10));
let c3 = Arc::new(create_string_array_with_len::<i32>(NUM_ROWS, 0.1, 20));
let batch =
RecordBatch::try_from_iter([("c1", c1 as _), ("c2", c2 as _), ("c3", c3 as _)]).unwrap();
do_bench(c, "bench_string", &batch)
}
fn bench_struct(c: &mut Criterion) {
let c1 = Arc::new(create_string_array::<i32>(NUM_ROWS, 0.));
let c2 = Arc::new(StructArray::from(create_mixed(NUM_ROWS)));
let batch = RecordBatch::try_from_iter([("c1", c1 as _), ("c2", c2 as _)]).unwrap();
do_bench(c, "bench_struct", &batch)
}
fn bench_nullable_struct(c: &mut Criterion) {
let c1 = Arc::new(create_string_array::<i32>(NUM_ROWS, 0.));
let c2 = Arc::new(create_nullable_struct(NUM_ROWS));
let batch = RecordBatch::try_from_iter([("c1", c1 as _), ("c2", c2 as _)]).unwrap();
do_bench(c, "bench_nullable_struct", &batch)
}
fn bench_list(c: &mut Criterion) {
let (values_len, offsets) = create_offsets(NUM_ROWS);
let c1_values = Arc::new(create_string_array::<i32>(values_len, 0.));
let c1_field = Arc::new(Field::new_list_field(c1_values.data_type().clone(), false));
let c1 = Arc::new(ListArray::new(c1_field, offsets, c1_values, None));
let batch = RecordBatch::try_from_iter([("c1", c1 as _)]).unwrap();
do_bench(c, "bench_list", &batch)
}
fn bench_nullable_list(c: &mut Criterion) {
let (values_len, offsets) = create_offsets(NUM_ROWS);
let c1_values = Arc::new(create_string_array::<i32>(values_len, 0.1));
let c1_field = Arc::new(Field::new_list_field(c1_values.data_type().clone(), true));
let c1_nulls = create_nulls(NUM_ROWS);
let c1 = Arc::new(ListArray::new(c1_field, offsets, c1_values, Some(c1_nulls)));
let batch = RecordBatch::try_from_iter([("c1", c1 as _)]).unwrap();
do_bench(c, "bench_nullable_list", &batch)
}
fn bench_struct_list(c: &mut Criterion) {
let (values_len, offsets) = create_offsets(NUM_ROWS);
let c1_values = Arc::new(create_nullable_struct(values_len));
let c1_field = Arc::new(Field::new_list_field(c1_values.data_type().clone(), true));
let c1_nulls = create_nulls(NUM_ROWS);
let c1 = Arc::new(ListArray::new(c1_field, offsets, c1_values, Some(c1_nulls)));
let batch = RecordBatch::try_from_iter([("c1", c1 as _)]).unwrap();
do_bench(c, "bench_struct_list", &batch)
}
fn do_number_to_string_bench<S: Serialize>(
name: &str,
c: &mut Criterion,
schema: Arc<Schema>,
rows: Vec<S>,
) {
c.bench_function(name, |b| {
b.iter(|| {
let mut decoder = ReaderBuilder::new(schema.clone())
.with_coerce_primitive(true) // important for coercion
.build_decoder()
.expect("Failed to build decoder");
decoder.serialize(&rows).expect("Failed to serialize rows");
decoder
.flush()
.expect("Failed to flush")
.expect("No RecordBatch produced");
})
});
}
fn bench_i64_to_string(c: &mut Criterion) {
#[derive(Serialize)]
struct TestRow {
val: i64,
}
let schema = Arc::new(Schema::new(vec![Field::new("val", DataType::Utf8, false)]));
let a_bunch_of_numbers = create_primitive_array::<Int64Type>(NUM_ROWS, 0.0);
let rows: Vec<TestRow> = (0..NUM_ROWS)
.map(|i| TestRow {
val: a_bunch_of_numbers.value(i),
})
.collect();
do_number_to_string_bench("i64_to_string", c, schema, rows)
}
fn bench_i32_to_string(c: &mut Criterion) {
#[derive(Serialize)]
struct TestRow {
val: i32,
}
let schema = Arc::new(Schema::new(vec![Field::new("val", DataType::Utf8, false)]));
let a_bunch_of_numbers = create_primitive_array::<Int32Type>(NUM_ROWS, 0.0);
let rows: Vec<TestRow> = (0..NUM_ROWS)
.map(|i| TestRow {
val: a_bunch_of_numbers.value(i),
})
.collect();
do_number_to_string_bench("i32_to_string", c, schema, rows)
}
fn bench_f32_to_string(c: &mut Criterion) {
#[derive(Serialize)]
struct TestRow {
val: f32,
}
let schema = Arc::new(Schema::new(vec![Field::new("val", DataType::Utf8, false)]));
let a_bunch_of_numbers = create_primitive_array::<Float32Type>(NUM_ROWS, 0.0);
let rows: Vec<TestRow> = (0..NUM_ROWS)
.map(|i| TestRow {
val: a_bunch_of_numbers.value(i),
})
.collect();
do_number_to_string_bench("f32_to_string", c, schema, rows)
}
fn bench_f64_to_string(c: &mut Criterion) {
#[derive(Serialize)]
struct TestRow {
val: f64,
}
let schema = Arc::new(Schema::new(vec![Field::new("val", DataType::Utf8, false)]));
let a_bunch_of_numbers = create_primitive_array::<Float64Type>(NUM_ROWS, 0.0);
let rows: Vec<TestRow> = (0..NUM_ROWS)
.map(|i| TestRow {
val: a_bunch_of_numbers.value(i),
})
.collect();
do_number_to_string_bench("f64_to_string", c, schema, rows)
}
fn bench_mixed_numbers_to_string(c: &mut Criterion) {
#[derive(Serialize)]
struct TestRow {
val1: f64,
val2: f32,
val3: i64,
val4: i32,
}
let schema = Arc::new(Schema::new(vec![
Field::new("val1", DataType::Utf8, false),
Field::new("val2", DataType::Utf8, false),
Field::new("val3", DataType::Utf8, false),
Field::new("val4", DataType::Utf8, false),
]));
let f64_array = create_primitive_array::<Float64Type>(NUM_ROWS, 0.0);
let f32_array = create_primitive_array::<Float32Type>(NUM_ROWS, 0.0);
let i64_array = create_primitive_array::<Int64Type>(NUM_ROWS, 0.0);
let i32_array = create_primitive_array::<Int32Type>(NUM_ROWS, 0.0);
let rows: Vec<TestRow> = (0..NUM_ROWS)
.map(|i| TestRow {
val1: f64_array.value(i),
val2: f32_array.value(i),
val3: i64_array.value(i),
val4: i32_array.value(i),
})
.collect();
do_number_to_string_bench("mixed_numbers_to_string", c, schema, rows)
}
fn criterion_benchmark(c: &mut Criterion) {
bench_integer(c);
bench_float(c);
bench_string(c);
bench_mixed(c);
bench_dict_array(c);
bench_struct(c);
bench_nullable_struct(c);
bench_list(c);
bench_nullable_list(c);
bench_struct_list(c);
bench_f64_to_string(c);
bench_f32_to_string(c);
bench_i64_to_string(c);
bench_i32_to_string(c);
bench_mixed_numbers_to_string(c);
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
+48
View File
@@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::array::*;
use arrow::compute::kernels::length::length;
use std::hint;
fn bench_length(array: &StringArray) {
hint::black_box(length(array).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
fn double_vec<T: Clone>(v: Vec<T>) -> Vec<T> {
[&v[..], &v[..]].concat()
}
// double ["hello", " ", "world", "!"] 10 times
let mut values = vec!["one", "on", "o", ""];
for _ in 0..10 {
values = double_vec(values);
}
let array = StringArray::from(values);
c.bench_function("length", |b| b.iter(|| bench_length(&array)));
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+221
View File
@@ -0,0 +1,221 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::compute::{SortColumn, lexsort_to_indices};
use arrow::row::{RowConverter, SortField};
use arrow::util::bench_util::{
create_dict_from_values, create_primitive_array, create_string_array_with_len,
};
use arrow::util::data_gen::create_random_array;
use arrow_array::types::Int32Type;
use arrow_array::{Array, ArrayRef, UInt32Array};
use arrow_schema::{DataType, Field};
use criterion::{Criterion, criterion_group, criterion_main};
use std::{hint, sync::Arc};
#[derive(Copy, Clone)]
enum Column {
RequiredI32,
OptionalI32,
Required16CharString,
Optional16CharString,
Optional50CharString,
Optional100Value50CharStringDict,
RequiredI32List,
OptionalI32List,
Required4CharStringList,
Optional4CharStringList,
}
impl std::fmt::Debug for Column {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let s = match self {
Column::RequiredI32 => "i32",
Column::OptionalI32 => "i32_opt",
Column::Required16CharString => "str(16)",
Column::Optional16CharString => "str_opt(16)",
Column::Optional50CharString => "str_opt(50)",
Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
Column::RequiredI32List => "i32_list",
Column::OptionalI32List => "i32_list_opt",
Column::Required4CharStringList => "str_list(4)",
Column::Optional4CharStringList => "str_list_opt(4)",
};
f.write_str(s)
}
}
impl Column {
fn generate(self, size: usize) -> ArrayRef {
match self {
Column::RequiredI32 => Arc::new(create_primitive_array::<Int32Type>(size, 0.)),
Column::OptionalI32 => Arc::new(create_primitive_array::<Int32Type>(size, 0.2)),
Column::Required16CharString => {
Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
}
Column::Optional16CharString => {
Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
}
Column::Optional50CharString => {
Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
}
Column::Optional100Value50CharStringDict => {
Arc::new(create_dict_from_values::<Int32Type>(
size,
0.1,
&create_string_array_with_len::<i32>(100, 0., 50),
))
}
Column::RequiredI32List => {
let field = Field::new(
"_1",
DataType::List(Arc::new(Field::new_list_field(DataType::Int32, false))),
true,
);
create_random_array(&field, size, 0., 1.).unwrap()
}
Column::OptionalI32List => {
let field = Field::new(
"_1",
DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
true,
);
create_random_array(&field, size, 0.2, 1.).unwrap()
}
Column::Required4CharStringList => {
let field = Field::new(
"_1",
DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, false))),
true,
);
create_random_array(&field, size, 0., 1.).unwrap()
}
Column::Optional4CharStringList => {
let field = Field::new(
"_1",
DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),
true,
);
create_random_array(&field, size, 0.2, 1.).unwrap()
}
}
}
}
fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
let sort_columns: Vec<_> = arrays
.iter()
.cloned()
.map(|values| SortColumn {
values,
options: None,
})
.collect();
c.bench_function(&format!("lexsort_to_indices({columns:?}): {len}"), |b| {
b.iter(|| hint::black_box(lexsort_to_indices(&sort_columns, None).unwrap()))
});
c.bench_function(&format!("lexsort_rows({columns:?}): {len}"), |b| {
b.iter(|| {
hint::black_box({
let fields = arrays
.iter()
.map(|a| SortField::new(a.data_type().clone()))
.collect();
let converter = RowConverter::new(fields).unwrap();
let rows = converter.convert_columns(&arrays).unwrap();
let mut sort: Vec<_> = rows.iter().enumerate().collect();
sort.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
UInt32Array::from_iter_values(sort.iter().map(|(i, _)| *i as u32))
})
})
});
}
fn add_benchmark(c: &mut Criterion) {
let cases: &[&[Column]] = &[
&[Column::RequiredI32, Column::OptionalI32],
&[Column::RequiredI32, Column::Optional16CharString],
&[Column::RequiredI32, Column::Required16CharString],
&[Column::Optional16CharString, Column::Required16CharString],
&[
Column::Optional16CharString,
Column::Optional50CharString,
Column::Required16CharString,
],
&[
Column::Optional16CharString,
Column::Required16CharString,
Column::Optional16CharString,
Column::Optional16CharString,
Column::Optional16CharString,
],
&[
Column::OptionalI32,
Column::Optional100Value50CharStringDict,
],
&[
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
],
&[
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
Column::Required16CharString,
],
&[
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
Column::Optional50CharString,
],
&[
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
Column::Optional100Value50CharStringDict,
Column::Optional50CharString,
],
&[Column::OptionalI32, Column::RequiredI32List],
&[Column::OptionalI32, Column::OptionalI32List],
&[Column::OptionalI32List, Column::OptionalI32],
&[Column::RequiredI32, Column::Required4CharStringList],
&[Column::Required4CharStringList, Column::RequiredI32],
&[Column::RequiredI32, Column::Optional4CharStringList],
&[Column::Optional4CharStringList, Column::RequiredI32],
&[
Column::RequiredI32,
Column::RequiredI32List,
Column::Required16CharString,
],
&[
Column::OptionalI32,
Column::OptionalI32List,
Column::Optional50CharString,
],
];
for case in cases {
do_bench(c, case, 4096);
do_bench(c, case, 4096 * 8);
}
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+280
View File
@@ -0,0 +1,280 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use criterion::measurement::WallTime;
use criterion::{BenchmarkGroup, BenchmarkId, Criterion, criterion_group, criterion_main};
use rand::distr::{Distribution, StandardUniform};
use rand::prelude::StdRng;
use rand::{Rng, SeedableRng};
use std::hint;
use std::sync::Arc;
use arrow::array::*;
use arrow::datatypes::*;
use arrow::util::bench_util::*;
use arrow_select::merge::merge;
trait InputGenerator {
fn name(&self) -> &str;
/// Return an ArrayRef containing a single null value
fn generate_scalar_with_null_value(&self) -> ArrayRef;
/// Generate a `number_of_scalars` unique scalars
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef>;
/// Generate an array with the specified length and null percentage
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef;
}
struct GeneratePrimitive<T: ArrowPrimitiveType> {
description: String,
_marker: std::marker::PhantomData<T>,
}
impl<T> InputGenerator for GeneratePrimitive<T>
where
T: ArrowPrimitiveType,
StandardUniform: Distribution<T::Native>,
{
fn name(&self) -> &str {
self.description.as_str()
}
fn generate_scalar_with_null_value(&self) -> ArrayRef {
new_null_array(&T::DATA_TYPE, 1)
}
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef> {
let rng = StdRng::seed_from_u64(seed);
rng.sample_iter::<T::Native, _>(StandardUniform)
.take(number_of_scalars)
.map(|v: T::Native| {
Arc::new(PrimitiveArray::<T>::new_scalar(v).into_inner()) as ArrayRef
})
.collect()
}
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef {
Arc::new(create_primitive_array_with_seed::<T>(
array_length,
null_percentage,
seed,
))
}
}
struct GenerateBytes<Byte: ByteArrayType> {
range_length: std::ops::Range<usize>,
description: String,
_marker: std::marker::PhantomData<Byte>,
}
impl<Byte> InputGenerator for GenerateBytes<Byte>
where
Byte: ByteArrayType,
{
fn name(&self) -> &str {
self.description.as_str()
}
fn generate_scalar_with_null_value(&self) -> ArrayRef {
new_null_array(&Byte::DATA_TYPE, 1)
}
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef> {
let array = self.generate_array(seed, number_of_scalars, 0.0);
(0..number_of_scalars).map(|i| array.slice(i, 1)).collect()
}
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef {
let is_binary =
Byte::DATA_TYPE == DataType::Binary || Byte::DATA_TYPE == DataType::LargeBinary;
if is_binary {
Arc::new(create_binary_array_with_len_range_and_prefix_and_seed::<
Byte::Offset,
>(
array_length,
null_percentage,
self.range_length.start,
self.range_length.end - 1,
&[],
seed,
))
} else {
Arc::new(create_string_array_with_len_range_and_prefix_and_seed::<
Byte::Offset,
>(
array_length,
null_percentage,
self.range_length.start,
self.range_length.end - 1,
"",
seed,
))
}
}
}
fn mask_cases(len: usize) -> Vec<(&'static str, BooleanArray)> {
vec![
("all_true", create_boolean_array(len, 0.0, 1.0)),
("99pct_true", create_boolean_array(len, 0.0, 0.99)),
("90pct_true", create_boolean_array(len, 0.0, 0.9)),
("50pct_true", create_boolean_array(len, 0.0, 0.5)),
("10pct_true", create_boolean_array(len, 0.0, 0.1)),
("1pct_true", create_boolean_array(len, 0.0, 0.01)),
("all_false", create_boolean_array(len, 0.0, 0.0)),
("50pct_nulls", create_boolean_array(len, 0.5, 0.5)),
]
}
fn bench_merge_on_input_generator(c: &mut Criterion, input_generator: &impl InputGenerator) {
const ARRAY_LEN: usize = 8192;
let mut group =
c.benchmark_group(format!("merge_{ARRAY_LEN}_from_{}", input_generator.name()).as_str());
let null_scalar = input_generator.generate_scalar_with_null_value();
let [non_null_scalar_1, non_null_scalar_2]: [_; 2] = input_generator
.generate_non_null_scalars(42, 2)
.try_into()
.unwrap();
// For simplicity, we generate arrays with length ARRAY_LEN. Not all input values will be used.
let array_1_10pct_nulls = input_generator.generate_array(42, ARRAY_LEN, 0.1);
let array_2_10pct_nulls = input_generator.generate_array(18, ARRAY_LEN, 0.1);
let masks = mask_cases(ARRAY_LEN);
// Benchmarks for different scalar combinations
for (description, truthy, falsy) in &[
("null_vs_non_null_scalar", &null_scalar, &non_null_scalar_1),
(
"non_null_scalar_vs_null_scalar",
&non_null_scalar_1,
&null_scalar,
),
("non_nulls_scalars", &non_null_scalar_1, &non_null_scalar_2),
] {
bench_merge_input_on_all_masks(
description,
&mut group,
&masks,
&Scalar::new(truthy),
&Scalar::new(falsy),
);
}
bench_merge_input_on_all_masks(
"array_vs_non_null_scalar",
&mut group,
&masks,
&array_1_10pct_nulls,
&non_null_scalar_1,
);
bench_merge_input_on_all_masks(
"non_null_scalar_vs_array",
&mut group,
&masks,
&non_null_scalar_1,
&array_1_10pct_nulls,
);
bench_merge_input_on_all_masks(
"array_vs_array",
&mut group,
&masks,
&array_1_10pct_nulls,
&array_2_10pct_nulls,
);
group.finish();
}
fn bench_merge_input_on_all_masks(
description: &str,
group: &mut BenchmarkGroup<WallTime>,
masks: &[(&str, BooleanArray)],
truthy: &impl Datum,
falsy: &impl Datum,
) {
for (mask_description, mask) in masks {
let id = BenchmarkId::new(description, mask_description);
group.bench_with_input(id, mask, |b, mask| {
b.iter(|| hint::black_box(merge(mask, truthy, falsy)))
});
}
}
fn add_benchmark(c: &mut Criterion) {
// Primitive
bench_merge_on_input_generator(
c,
&GeneratePrimitive::<Int32Type> {
description: "i32".to_string(),
_marker: std::marker::PhantomData,
},
);
// Short strings
bench_merge_on_input_generator(
c,
&GenerateBytes::<GenericStringType<i32>> {
description: "short strings (3..10)".to_string(),
range_length: 3..10,
_marker: std::marker::PhantomData,
},
);
// Long strings
bench_merge_on_input_generator(
c,
&GenerateBytes::<GenericStringType<i32>> {
description: "long strings (100..400)".to_string(),
range_length: 100..400,
_marker: std::marker::PhantomData,
},
);
// Short Bytes
bench_merge_on_input_generator(
c,
&GenerateBytes::<GenericBinaryType<i32>> {
description: "short bytes (3..10)".to_string(),
range_length: 3..10,
_marker: std::marker::PhantomData,
},
);
// Long Bytes
bench_merge_on_input_generator(
c,
&GenerateBytes::<GenericBinaryType<i32>> {
description: "long bytes (100..400)".to_string(),
range_length: 100..400,
_marker: std::marker::PhantomData,
},
);
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+61
View File
@@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use rand::Rng;
extern crate arrow;
use arrow::util::test_util::seedable_rng;
use arrow::{array::*, util::bench_util::create_string_array};
fn create_slices(size: usize) -> Vec<(usize, usize)> {
let rng = &mut seedable_rng();
(0..size)
.map(|_| {
let start = rng.random_range(0..size / 2);
let end = rng.random_range(start + 1..size);
(start, end)
})
.collect()
}
fn bench<T: Array>(v1: &T, slices: &[(usize, usize)]) {
let data = v1.to_data();
let mut mutable = MutableArrayData::new(vec![&data], false, 5);
for (start, end) in slices {
mutable.extend(0, *start, *end)
}
mutable.freeze();
}
fn add_benchmark(c: &mut Criterion) {
let v1 = create_string_array::<i32>(1024, 0.0);
let v2 = create_slices(1024);
c.bench_function("mutable str 1024", |b| b.iter(|| bench(&v1, &v2)));
let v1 = create_string_array::<i32>(1024, 0.5);
let v2 = create_slices(1024);
c.bench_function("mutable str nulls 1024", |b| b.iter(|| bench(&v1, &v2)));
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+130
View File
@@ -0,0 +1,130 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use std::sync::Arc;
extern crate arrow;
use arrow::compute::kernels::sort::{SortColumn, lexsort};
use arrow::util::bench_util::*;
use arrow::{
array::*,
datatypes::{Float64Type, UInt8Type},
};
use arrow_ord::partition::partition;
use rand::distr::{Distribution, StandardUniform};
use std::hint;
fn create_array<T: ArrowPrimitiveType>(size: usize, with_nulls: bool) -> ArrayRef
where
StandardUniform: Distribution<T::Native>,
{
let null_density = if with_nulls { 0.5 } else { 0.0 };
let array = create_primitive_array::<T>(size, null_density);
Arc::new(array)
}
fn bench_partition(sorted_columns: &[ArrayRef]) {
hint::black_box(partition(sorted_columns).unwrap().ranges());
}
fn create_sorted_low_cardinality_data(length: usize) -> Vec<ArrayRef> {
let arr = Int64Array::from_iter_values(
std::iter::repeat_n(1, length / 4)
.chain(std::iter::repeat_n(2, length / 4))
.chain(std::iter::repeat_n(3, length / 4))
.chain(std::iter::repeat_n(4, length / 4)),
);
lexsort(
&[SortColumn {
values: Arc::new(arr),
options: None,
}],
None,
)
.unwrap()
}
fn create_sorted_float_data(pow: u32, with_nulls: bool) -> Vec<ArrayRef> {
lexsort(
&[
SortColumn {
values: create_array::<Float64Type>(2u64.pow(pow) as usize, with_nulls),
options: None,
},
SortColumn {
values: create_array::<Float64Type>(2u64.pow(pow) as usize, with_nulls),
options: None,
},
],
None,
)
.unwrap()
}
fn create_sorted_data(pow: u32, with_nulls: bool) -> Vec<ArrayRef> {
lexsort(
&[
SortColumn {
values: create_array::<UInt8Type>(2u64.pow(pow) as usize, with_nulls),
options: None,
},
SortColumn {
values: create_array::<UInt8Type>(2u64.pow(pow) as usize, with_nulls),
options: None,
},
],
None,
)
.unwrap()
}
fn add_benchmark(c: &mut Criterion) {
let sorted_columns = create_sorted_data(10, false);
c.bench_function("partition(u8) 2^10", |b| {
b.iter(|| bench_partition(&sorted_columns))
});
let sorted_columns = create_sorted_data(12, false);
c.bench_function("partition(u8) 2^12", |b| {
b.iter(|| bench_partition(&sorted_columns))
});
let sorted_columns = create_sorted_data(10, true);
c.bench_function("partition(u8) 2^10 with nulls", |b| {
b.iter(|| bench_partition(&sorted_columns))
});
let sorted_columns = create_sorted_data(12, true);
c.bench_function("partition(u8) 2^12 with nulls", |b| {
b.iter(|| bench_partition(&sorted_columns))
});
let sorted_columns = create_sorted_float_data(10, false);
c.bench_function("partition(f64) 2^10", |b| {
b.iter(|| bench_partition(&sorted_columns))
});
let sorted_columns = create_sorted_low_cardinality_data(1024);
c.bench_function("partition(low cardinality) 1024", |b| {
b.iter(|| bench_partition(&sorted_columns))
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
@@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::datatypes::Int32Type;
use arrow::{array::PrimitiveArray, util::bench_util::create_primitive_run_array};
use arrow_array::ArrayAccessor;
use criterion::{Criterion, criterion_group, criterion_main};
fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("primitive_run_accessor");
let mut do_bench = |physical_array_len: usize, logical_array_len: usize| {
group.bench_function(
format!("(run_array_len:{logical_array_len}, physical_array_len:{physical_array_len})"),
|b| {
let run_array = create_primitive_run_array::<Int32Type, Int32Type>(
logical_array_len,
physical_array_len,
);
let typed = run_array.downcast::<PrimitiveArray<Int32Type>>().unwrap();
b.iter(|| {
for i in 0..logical_array_len {
let _ = unsafe { typed.value_unchecked(i) };
}
})
},
);
};
do_bench(128, 512);
do_bench(256, 1024);
do_bench(512, 2048);
do_bench(1024, 4096);
do_bench(2048, 8192);
group.finish();
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
@@ -0,0 +1,78 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::array::UInt32Builder;
use arrow::compute::take;
use arrow::datatypes::{Int32Type, Int64Type};
use arrow::util::bench_util::*;
use arrow::util::test_util::seedable_rng;
use arrow_array::UInt32Array;
use criterion::{Criterion, criterion_group, criterion_main};
use rand::Rng;
use std::hint;
fn create_random_index(size: usize, null_density: f32, max_value: usize) -> UInt32Array {
let mut rng = seedable_rng();
let mut builder = UInt32Builder::with_capacity(size);
for _ in 0..size {
if rng.random::<f32>() < null_density {
builder.append_null();
} else {
let value = rng.random_range::<u32, _>(0u32..max_value as u32);
builder.append_value(value);
}
}
builder.finish()
}
fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("primitive_run_take");
let mut do_bench = |physical_array_len: usize, logical_array_len: usize, take_len: usize| {
let run_array = create_primitive_run_array::<Int32Type, Int64Type>(
logical_array_len,
physical_array_len,
);
let indices = create_random_index(take_len, 0.0, logical_array_len);
group.bench_function(
format!(
"(run_array_len:{logical_array_len}, physical_array_len:{physical_array_len}, take_len:{take_len})"),
|b| {
b.iter(|| {
hint::black_box(take(&run_array, &indices, None).unwrap());
})
},
);
};
do_bench(64, 512, 512);
do_bench(128, 512, 512);
do_bench(256, 1024, 512);
do_bench(256, 1024, 1024);
do_bench(512, 2048, 512);
do_bench(512, 2048, 1024);
do_bench(1024, 4096, 512);
do_bench(1024, 4096, 1024);
group.finish();
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
+52
View File
@@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::array::*;
use arrow::compute::kernels::regexp::*;
use arrow::util::bench_util::*;
use std::hint;
fn bench_regexp(arr: &GenericStringArray<i32>, regex_array: &dyn Datum) {
regexp_match(hint::black_box(arr), regex_array, None).unwrap();
}
fn add_benchmark(c: &mut Criterion) {
let size = 65536;
let val_len = 1000;
let arr_string = create_string_array_with_len::<i32>(size, 0.0, val_len);
let pattern_values = vec![r".*-(\d*)-.*"; size];
let pattern = GenericStringArray::<i32>::from(pattern_values);
c.bench_function("regexp", |b| b.iter(|| bench_regexp(&arr_string, &pattern)));
let pattern_values = vec![r".*-(\d*)-.*"];
let pattern = Scalar::new(GenericStringArray::<i32>::from(pattern_values));
c.bench_function("regexp scalar", |b| {
b.iter(|| bench_regexp(&arr_string, &pattern))
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+286
View File
@@ -0,0 +1,286 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
extern crate core;
use arrow::array::ArrayRef;
use arrow::datatypes::{Int64Type, UInt64Type};
use arrow::row::{RowConverter, SortField};
use arrow::util::bench_util::{
create_boolean_array, create_dict_from_values, create_primitive_array,
create_string_array_with_len, create_string_dict_array, create_string_view_array_with_len,
create_string_view_array_with_max_len,
};
use arrow::util::data_gen::create_random_array;
use arrow_array::Array;
use arrow_array::types::Int32Type;
use arrow_schema::{DataType, Field};
use criterion::Criterion;
use std::{hint, sync::Arc};
fn do_bench(c: &mut Criterion, name: &str, cols: Vec<ArrayRef>) {
let fields: Vec<_> = cols
.iter()
.map(|x| SortField::new(x.data_type().clone()))
.collect();
c.bench_function(&format!("convert_columns {name}"), |b| {
b.iter(|| {
let converter = RowConverter::new(fields.clone()).unwrap();
hint::black_box(converter.convert_columns(&cols).unwrap())
});
});
let converter = RowConverter::new(fields).unwrap();
let rows = converter.convert_columns(&cols).unwrap();
// using a pre-prepared row converter should be faster than the first time
c.bench_function(&format!("convert_columns_prepared {name}"), |b| {
b.iter(|| hint::black_box(converter.convert_columns(&cols).unwrap()));
});
c.bench_function(&format!("convert_rows {name}"), |b| {
b.iter(|| hint::black_box(converter.convert_rows(&rows).unwrap()));
});
let mut rows = converter.empty_rows(0, 0);
c.bench_function(&format!("append_rows {name}"), |b| {
let cols = cols.clone();
b.iter(|| {
rows.clear();
converter.append(&mut rows, &cols).unwrap();
hint::black_box(&mut rows);
});
});
}
fn bench_iter(c: &mut Criterion) {
let col = create_string_view_array_with_len(4096, 0., 100, false);
let converter = RowConverter::new(vec![SortField::new(col.data_type().clone())]).unwrap();
let rows = converter
.convert_columns(&[Arc::new(col) as ArrayRef])
.unwrap();
c.bench_function("iterate rows", |b| {
b.iter(|| {
for r in rows.iter() {
hint::black_box(r.as_ref());
}
})
});
}
fn row_bench(c: &mut Criterion) {
let cols = vec![Arc::new(create_primitive_array::<UInt64Type>(4096, 0.)) as ArrayRef];
do_bench(c, "4096 u64(0)", cols);
let cols = vec![Arc::new(create_primitive_array::<UInt64Type>(4096, 0.3)) as ArrayRef];
do_bench(c, "4096 u64(0.3)", cols);
let cols = vec![Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef];
do_bench(c, "4096 i64(0)", cols);
let cols = vec![Arc::new(create_primitive_array::<Int64Type>(4096, 0.3)) as ArrayRef];
do_bench(c, "4096 i64(0.3)", cols);
let cols = vec![Arc::new(create_boolean_array(4096, 0., 0.5)) as ArrayRef];
do_bench(c, "4096 bool(0, 0.5)", cols);
let cols = vec![Arc::new(create_boolean_array(4096, 0.3, 0.5)) as ArrayRef];
do_bench(c, "4096 bool(0.3, 0.5)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0., 10)) as ArrayRef];
do_bench(c, "4096 string(10, 0)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0., 30)) as ArrayRef];
do_bench(c, "4096 string(30, 0)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0., 100)) as ArrayRef];
do_bench(c, "4096 string(100, 0)", cols);
let cols = vec![Arc::new(create_string_array_with_len::<i32>(4096, 0.5, 100)) as ArrayRef];
do_bench(c, "4096 string(100, 0.5)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0., 10, false)) as ArrayRef];
do_bench(c, "4096 string view(10, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0., 30, false)) as ArrayRef];
do_bench(c, "4096 string view(30, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0., 100, false)) as ArrayRef];
do_bench(c, "4096 string view(100, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_len(4096, 0.5, 100, false)) as ArrayRef];
do_bench(c, "4096 string view(100, 0.5)", cols);
let cols = vec![Arc::new(create_string_view_array_with_max_len(4096, 0., 100)) as ArrayRef];
do_bench(c, "4096 string view(1..100, 0)", cols);
let cols = vec![Arc::new(create_string_view_array_with_max_len(4096, 0.5, 100)) as ArrayRef];
do_bench(c, "4096 string view(1..100, 0.5)", cols);
let cols = vec![Arc::new(create_string_dict_array::<Int32Type>(4096, 0., 10)) as ArrayRef];
do_bench(c, "4096 string_dictionary(10, 0)", cols);
let cols = vec![Arc::new(create_string_dict_array::<Int32Type>(4096, 0., 30)) as ArrayRef];
do_bench(c, "4096 string_dictionary(30, 0)", cols);
let cols = vec![Arc::new(create_string_dict_array::<Int32Type>(4096, 0., 100)) as ArrayRef];
do_bench(c, "4096 string_dictionary(100, 0)", cols.clone());
let cols = vec![Arc::new(create_string_dict_array::<Int32Type>(4096, 0.5, 100)) as ArrayRef];
do_bench(c, "4096 string_dictionary(100, 0.5)", cols.clone());
let values = create_string_array_with_len::<i32>(10, 0., 10);
let dict = create_dict_from_values::<Int32Type>(4096, 0., &values);
let cols = vec![Arc::new(dict) as ArrayRef];
do_bench(c, "4096 string_dictionary_low_cardinality(10, 0)", cols);
let values = create_string_array_with_len::<i32>(10, 0., 30);
let dict = create_dict_from_values::<Int32Type>(4096, 0., &values);
let cols = vec![Arc::new(dict) as ArrayRef];
do_bench(c, "4096 string_dictionary_low_cardinality(30, 0)", cols);
let values = create_string_array_with_len::<i32>(10, 0., 100);
let dict = create_dict_from_values::<Int32Type>(4096, 0., &values);
let cols = vec![Arc::new(dict) as ArrayRef];
do_bench(c, "4096 string_dictionary_low_cardinality(100, 0)", cols);
let cols = vec![
Arc::new(create_string_array_with_len::<i32>(4096, 0.5, 20)) as ArrayRef,
Arc::new(create_string_array_with_len::<i32>(4096, 0., 30)) as ArrayRef,
Arc::new(create_string_array_with_len::<i32>(4096, 0., 100)) as ArrayRef,
Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef,
];
do_bench(
c,
"4096 string(20, 0.5), string(30, 0), string(100, 0), i64(0)",
cols,
);
let cols = vec![
Arc::new(create_string_dict_array::<Int32Type>(4096, 0.5, 20)) as ArrayRef,
Arc::new(create_string_dict_array::<Int32Type>(4096, 0., 30)) as ArrayRef,
Arc::new(create_string_dict_array::<Int32Type>(4096, 0., 100)) as ArrayRef,
Arc::new(create_primitive_array::<Int64Type>(4096, 0.)) as ArrayRef,
];
do_bench(
c,
"4096 4096 string_dictionary(20, 0.5), string_dictionary(30, 0), string_dictionary(100, 0), i64(0)",
cols,
);
// List
let cols = vec![
create_random_array(
&Field::new(
"list",
DataType::List(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
),
4096,
0.,
1.0,
)
.unwrap(),
];
do_bench(c, "4096 list(0) of u64(0)", cols);
let cols = vec![
create_random_array(
&Field::new(
"list",
DataType::LargeList(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
),
4096,
0.,
1.0,
)
.unwrap(),
];
do_bench(c, "4096 large_list(0) of u64(0)", cols);
let cols = vec![
create_random_array(
&Field::new(
"list",
DataType::List(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
),
10,
0.,
1.0,
)
.unwrap(),
];
do_bench(c, "10 list(0) of u64(0)", cols);
let cols = vec![
create_random_array(
&Field::new(
"list",
DataType::LargeList(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
),
10,
0.,
1.0,
)
.unwrap(),
];
do_bench(c, "10 large_list(0) of u64(0)", cols);
let cols = vec![
create_random_array(
&Field::new(
"list",
DataType::List(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
),
4096,
0.,
1.0,
)
.unwrap()
.slice(10, 20),
];
do_bench(c, "4096 list(0) sliced to 10 of u64(0)", cols);
let cols = vec![
create_random_array(
&Field::new(
"list",
DataType::LargeList(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
),
4096,
0.,
1.0,
)
.unwrap()
.slice(10, 20),
];
do_bench(c, "4096 large_list(0) sliced to 10 of u64(0)", cols);
bench_iter(c);
}
criterion_group!(benches, row_bench);
criterion_main!(benches);
+326
View File
@@ -0,0 +1,326 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use std::sync::Arc;
extern crate arrow;
use arrow::compute::{SortColumn, lexsort, sort, sort_to_indices};
use arrow::datatypes::{Int16Type, Int32Type};
use arrow::util::bench_util::*;
use arrow::{array::*, datatypes::Float32Type};
use arrow_ord::rank::rank;
use std::hint;
fn create_f32_array(size: usize, with_nulls: bool) -> ArrayRef {
let null_density = if with_nulls { 0.5 } else { 0.0 };
let array = create_primitive_array::<Float32Type>(size, null_density);
Arc::new(array)
}
fn create_bool_array(size: usize, with_nulls: bool) -> ArrayRef {
let null_density = if with_nulls { 0.5 } else { 0.0 };
let true_density = 0.5;
let array = create_boolean_array(size, null_density, true_density);
Arc::new(array)
}
fn bench_sort(array: &dyn Array) {
hint::black_box(sort(array, None).unwrap());
}
fn bench_lexsort(array_a: &ArrayRef, array_b: &ArrayRef, limit: Option<usize>) {
let columns = vec![
SortColumn {
values: array_a.clone(),
options: None,
},
SortColumn {
values: array_b.clone(),
options: None,
},
];
hint::black_box(lexsort(&columns, limit).unwrap());
}
fn bench_sort_to_indices(array: &dyn Array, limit: Option<usize>) {
hint::black_box(sort_to_indices(array, None, limit).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
let arr = create_primitive_array::<Int32Type>(2usize.pow(10), 0.0);
c.bench_function("sort i32 2^10", |b| b.iter(|| bench_sort(&arr)));
c.bench_function("sort i32 to indices 2^10", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_primitive_array::<Int32Type>(2usize.pow(12), 0.0);
c.bench_function("sort i32 2^12", |b| b.iter(|| bench_sort(&arr)));
c.bench_function("sort i32 to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_primitive_array::<Int32Type>(2usize.pow(10), 0.5);
c.bench_function("sort i32 nulls 2^10", |b| b.iter(|| bench_sort(&arr)));
c.bench_function("sort i32 nulls to indices 2^10", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_primitive_array::<Int32Type>(2usize.pow(12), 0.5);
c.bench_function("sort i32 nulls 2^12", |b| b.iter(|| bench_sort(&arr)));
c.bench_function("sort i32 nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_f32_array(2_usize.pow(12), false);
c.bench_function("sort f32 2^12", |b| b.iter(|| bench_sort(&arr)));
c.bench_function("sort f32 to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_f32_array(2usize.pow(12), true);
c.bench_function("sort f32 nulls 2^12", |b| b.iter(|| bench_sort(&arr)));
c.bench_function("sort f32 nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_max_len::<i32>(2usize.pow(12), 0.0, 10);
c.bench_function("sort string[0-10] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_max_len::<i32>(2usize.pow(12), 0.5, 10);
c.bench_function("sort string[0-10] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_max_len::<i32>(2usize.pow(12), 0.0, 100);
c.bench_function("sort string[0-100] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_max_len::<i32>(2usize.pow(12), 0.5, 100);
c.bench_function("sort string[0-100] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array::<i32>(2usize.pow(12), 0.0);
c.bench_function("sort string[0-400] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array::<i32>(2usize.pow(12), 0.5);
c.bench_function("sort string[0-400] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.0, 10);
c.bench_function("sort string[10] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.5, 10);
c.bench_function("sort string[10] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.0, 100);
c.bench_function("sort string[100] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.5, 100);
c.bench_function("sort string[100] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.0, 1000);
c.bench_function("sort string[1000] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.5, 1000);
c.bench_function("sort string[1000] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
// This will generate string view arrays with 2^12 elements, each with a length fixed 10, and without nulls.
let arr = create_string_view_array_with_fixed_len(2usize.pow(12), 0.0, 10);
c.bench_function("sort string_view[10] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
// This will generate string view arrays with 2^12 elements, each with a length fixed 10, and with 50% nulls.
let arr = create_string_view_array_with_fixed_len(2usize.pow(12), 0.5, 10);
c.bench_function("sort string_view[10] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
// This will generate string view arrays with 2^12 elements, each with a length randomly chosen from 0 to max 400, and without nulls.
let arr = create_string_view_array(2usize.pow(12), 0.0);
c.bench_function("sort string_view[0-400] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
// This will generate string view arrays with 2^12 elements, each with a length randomly chosen from 0 to max 400, and with 50% nulls.
let arr = create_string_view_array(2usize.pow(12), 0.5);
c.bench_function("sort string_view[0-400] nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
// This will generate string view arrays with 2^12 elements, each with a length < 12 bytes which is inlined data, and without nulls.
let arr = create_string_view_array_with_max_len(2usize.pow(12), 0.0, 12);
c.bench_function("sort string_view_inlined[0-12] to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
// This will generate string view arrays with 2^12 elements, each with a length < 12 bytes which is inlined data, and with 50% nulls.
let arr = create_string_view_array_with_max_len(2usize.pow(12), 0.5, 12);
c.bench_function(
"sort string_view_inlined[0-12] nulls to indices 2^12",
|b| b.iter(|| bench_sort_to_indices(&arr, None)),
);
let arr = create_string_dict_array::<Int32Type>(2usize.pow(12), 0.0, 10);
c.bench_function("sort string[10] dict to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let arr = create_string_dict_array::<Int32Type>(2usize.pow(12), 0.5, 10);
c.bench_function("sort string[10] dict nulls to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&arr, None))
});
let run_encoded_array =
create_primitive_run_array::<Int16Type, Int32Type>(2usize.pow(12), 2usize.pow(10));
c.bench_function("sort primitive run 2^12", |b| {
b.iter(|| bench_sort(&run_encoded_array))
});
c.bench_function("sort primitive run to indices 2^12", |b| {
b.iter(|| bench_sort_to_indices(&run_encoded_array, None))
});
let arr_a = create_f32_array(2usize.pow(10), false);
let arr_b = create_f32_array(2usize.pow(10), false);
c.bench_function("lexsort (f32, f32) 2^10", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, None))
});
let arr_a = create_f32_array(2usize.pow(12), false);
let arr_b = create_f32_array(2usize.pow(12), false);
c.bench_function("lexsort (f32, f32) 2^12", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, None))
});
let arr_a = create_f32_array(2usize.pow(10), true);
let arr_b = create_f32_array(2usize.pow(10), true);
c.bench_function("lexsort (f32, f32) nulls 2^10", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, None))
});
let arr_a = create_f32_array(2usize.pow(12), true);
let arr_b = create_f32_array(2usize.pow(12), true);
c.bench_function("lexsort (f32, f32) nulls 2^12", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, None))
});
let arr_a = create_bool_array(2usize.pow(12), false);
let arr_b = create_bool_array(2usize.pow(12), false);
c.bench_function("lexsort (bool, bool) 2^12", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, None))
});
let arr_a = create_bool_array(2usize.pow(12), true);
let arr_b = create_bool_array(2usize.pow(12), true);
c.bench_function("lexsort (bool, bool) nulls 2^12", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, None))
});
let arr_a = create_f32_array(2usize.pow(12), false);
let arr_b = create_f32_array(2usize.pow(12), false);
c.bench_function("lexsort (f32, f32) 2^12 limit 10", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(10)))
});
let arr_a = create_f32_array(2usize.pow(12), false);
let arr_b = create_f32_array(2usize.pow(12), false);
c.bench_function("lexsort (f32, f32) 2^12 limit 100", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(100)))
});
let arr_a = create_f32_array(2usize.pow(12), false);
let arr_b = create_f32_array(2usize.pow(12), false);
c.bench_function("lexsort (f32, f32) 2^12 limit 1000", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(1000)))
});
let arr_a = create_f32_array(2usize.pow(12), false);
let arr_b = create_f32_array(2usize.pow(12), false);
c.bench_function("lexsort (f32, f32) 2^12 limit 2^12", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(2usize.pow(12))))
});
let arr_a = create_f32_array(2usize.pow(12), true);
let arr_b = create_f32_array(2usize.pow(12), true);
c.bench_function("lexsort (f32, f32) nulls 2^12 limit 10", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(10)))
});
c.bench_function("lexsort (f32, f32) nulls 2^12 limit 100", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(100)))
});
c.bench_function("lexsort (f32, f32) nulls 2^12 limit 1000", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(1000)))
});
c.bench_function("lexsort (f32, f32) nulls 2^12 limit 2^12", |b| {
b.iter(|| bench_lexsort(&arr_a, &arr_b, Some(2usize.pow(12))))
});
let arr = create_f32_array(2usize.pow(12), false);
c.bench_function("rank f32 2^12", |b| {
b.iter(|| hint::black_box(rank(&arr, None).unwrap()))
});
let arr = create_f32_array(2usize.pow(12), true);
c.bench_function("rank f32 nulls 2^12", |b| {
b.iter(|| hint::black_box(rank(&arr, None).unwrap()))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.0, 10);
c.bench_function("rank string[10] 2^12", |b| {
b.iter(|| hint::black_box(rank(&arr, None).unwrap()))
});
let arr = create_string_array_with_len::<i32>(2usize.pow(12), 0.5, 10);
c.bench_function("rank string[10] nulls 2^12", |b| {
b.iter(|| hint::black_box(rank(&arr, None).unwrap()))
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
@@ -0,0 +1,70 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::array::StringDictionaryBuilder;
use arrow::datatypes::Int32Type;
use criterion::{Criterion, criterion_group, criterion_main};
use rand::{Rng, rng};
/// Note: this is best effort, not all keys are necessarily present or unique
fn build_strings(dict_size: usize, total_size: usize, key_len: usize) -> Vec<String> {
let mut rng = rng();
let values: Vec<String> = (0..dict_size)
.map(|_| (0..key_len).map(|_| rng.random::<char>()).collect())
.collect();
(0..total_size)
.map(|_| values[rng.random_range(0..dict_size)].clone())
.collect()
}
fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("string_dictionary_builder");
let mut do_bench = |dict_size: usize, total_size: usize, key_len: usize| {
group.bench_function(
format!("(dict_size:{dict_size}, len:{total_size}, key_len: {key_len})"),
|b| {
let strings = build_strings(dict_size, total_size, key_len);
b.iter(|| {
let mut builder = StringDictionaryBuilder::<Int32Type>::with_capacity(
strings.len(),
key_len + 1,
(key_len + 1) * dict_size,
);
for val in &strings {
builder.append(val).unwrap();
}
builder.finish();
})
},
);
};
do_bench(20, 1000, 5);
do_bench(100, 1000, 5);
do_bench(100, 1000, 10);
do_bench(100, 10000, 10);
do_bench(100, 10000, 100);
group.finish();
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
@@ -0,0 +1,60 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::array::StringRunBuilder;
use arrow::datatypes::Int32Type;
use arrow::util::bench_util::create_string_array_for_runs;
use criterion::{Criterion, criterion_group, criterion_main};
fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("string_run_builder");
let mut do_bench = |physical_array_len: usize, logical_array_len: usize, string_len: usize| {
group.bench_function(
format!(
"(run_array_len:{logical_array_len}, physical_array_len:{physical_array_len}, string_len: {string_len})",
),
|b| {
let strings =
create_string_array_for_runs(physical_array_len, logical_array_len, string_len);
b.iter(|| {
let mut builder = StringRunBuilder::<Int32Type>::with_capacity(
physical_array_len,
(string_len + 1) * physical_array_len,
);
for val in &strings {
builder.append_value(val);
}
builder.finish();
})
},
);
};
do_bench(20, 1000, 5);
do_bench(100, 1000, 5);
do_bench(100, 1000, 10);
do_bench(100, 10000, 10);
do_bench(100, 10000, 100);
group.finish();
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
@@ -0,0 +1,82 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow::array::{Int32RunArray, StringArray, StringRunBuilder};
use arrow::datatypes::Int32Type;
use criterion::{Criterion, criterion_group, criterion_main};
use rand::{Rng, rng};
fn build_strings_runs(
physical_array_len: usize,
logical_array_len: usize,
string_len: usize,
) -> Int32RunArray {
let mut rng = rng();
let run_len = logical_array_len / physical_array_len;
let mut values: Vec<String> = (0..physical_array_len)
.map(|_| (0..string_len).map(|_| rng.random::<char>()).collect())
.flat_map(|s| std::iter::repeat_n(s, run_len))
.collect();
while values.len() < logical_array_len {
let last_val = values[values.len() - 1].clone();
values.push(last_val);
}
let mut builder = StringRunBuilder::<Int32Type>::with_capacity(
physical_array_len,
(string_len + 1) * physical_array_len,
);
builder.extend(values.into_iter().map(Some));
builder.finish()
}
fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("string_run_iterator");
let mut do_bench = |physical_array_len: usize, logical_array_len: usize, string_len: usize| {
group.bench_function(
format!(
"(run_array_len:{logical_array_len}, physical_array_len:{physical_array_len}, string_len: {string_len})"),
|b| {
let run_array =
build_strings_runs(physical_array_len, logical_array_len, string_len);
let typed = run_array.downcast::<StringArray>().unwrap();
b.iter(|| {
let iter = typed.into_iter();
for _ in iter {}
})
},
);
};
do_bench(256, 1024, 5);
do_bench(256, 1024, 25);
do_bench(256, 1024, 100);
do_bench(512, 2048, 5);
do_bench(512, 2048, 25);
do_bench(512, 2048, 100);
do_bench(1024, 4096, 5);
do_bench(1024, 4096, 25);
do_bench(1024, 4096, 100);
group.finish();
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
+66
View File
@@ -0,0 +1,66 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
extern crate arrow;
use arrow::array::*;
use arrow::compute::kernels::substring::*;
use arrow::util::bench_util::*;
use std::hint;
fn bench_substring(arr: &dyn Array, start: i64, length: Option<u64>) {
substring(hint::black_box(arr), start, length).unwrap();
}
fn bench_substring_by_char<O: OffsetSizeTrait>(
arr: &GenericStringArray<O>,
start: i64,
length: Option<u64>,
) {
substring_by_char(hint::black_box(arr), start, length).unwrap();
}
fn add_benchmark(c: &mut Criterion) {
let size = 65536;
let val_len = 1000;
let arr_string = create_string_array_with_len::<i32>(size, 0.0, val_len);
let arr_fsb = create_fsb_array(size, 0.0, val_len);
c.bench_function("substring utf8 (start = 0, length = None)", |b| {
b.iter(|| bench_substring(&arr_string, 0, None))
});
c.bench_function("substring utf8 (start = 1, length = str_len - 1)", |b| {
b.iter(|| bench_substring(&arr_string, 1, Some((val_len - 1) as u64)))
});
c.bench_function("substring utf8 by char", |b| {
b.iter(|| bench_substring_by_char(&arr_string, 1, Some((val_len - 1) as u64)))
});
c.bench_function("substring fixed size binary array", |b| {
b.iter(|| bench_substring(&arr_fsb, 1, Some((val_len - 1) as u64)))
});
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+211
View File
@@ -0,0 +1,211 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#[macro_use]
extern crate criterion;
use criterion::Criterion;
use rand::Rng;
extern crate arrow;
use arrow::compute::{TakeOptions, take};
use arrow::datatypes::*;
use arrow::util::test_util::seedable_rng;
use arrow::{array::*, util::bench_util::*};
use std::hint;
fn create_random_index(size: usize, null_density: f32) -> UInt32Array {
let mut rng = seedable_rng();
let mut builder = UInt32Builder::with_capacity(size);
for _ in 0..size {
if rng.random::<f32>() < null_density {
builder.append_null();
} else {
let value = rng.random_range::<u32, _>(0u32..size as u32);
builder.append_value(value);
}
}
builder.finish()
}
fn bench_take(values: &dyn Array, indices: &UInt32Array) {
hint::black_box(take(values, indices, None).unwrap());
}
fn bench_take_bounds_check(values: &dyn Array, indices: &UInt32Array) {
hint::black_box(take(values, indices, Some(TakeOptions { check_bounds: true })).unwrap());
}
fn add_benchmark(c: &mut Criterion) {
let values = create_primitive_array::<Int32Type>(512, 0.0);
let indices = create_random_index(512, 0.0);
c.bench_function("take i32 512", |b| b.iter(|| bench_take(&values, &indices)));
let values = create_primitive_array::<Int32Type>(1024, 0.0);
let indices = create_random_index(1024, 0.0);
c.bench_function("take i32 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let indices = create_random_index(1024, 0.5);
c.bench_function("take i32 null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_primitive_array::<Int32Type>(1024, 0.5);
let indices = create_random_index(1024, 0.0);
c.bench_function("take i32 null values 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let indices = create_random_index(1024, 0.5);
c.bench_function("take i32 null values null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_primitive_array::<Int32Type>(512, 0.0);
let indices = create_random_index(512, 0.0);
c.bench_function("take check bounds i32 512", |b| {
b.iter(|| bench_take_bounds_check(&values, &indices))
});
let values = create_primitive_array::<Int32Type>(1024, 0.0);
let indices = create_random_index(1024, 0.0);
c.bench_function("take check bounds i32 1024", |b| {
b.iter(|| bench_take_bounds_check(&values, &indices))
});
let values = create_boolean_array(512, 0.0, 0.5);
let indices = create_random_index(512, 0.0);
c.bench_function("take bool 512", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_boolean_array(1024, 0.0, 0.5);
let indices = create_random_index(1024, 0.0);
c.bench_function("take bool 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let indices = create_random_index(1024, 0.5);
c.bench_function("take bool null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_boolean_array(1024, 0.5, 0.5);
let indices = create_random_index(1024, 0.0);
c.bench_function("take bool null values 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_boolean_array(1024, 0.5, 0.5);
let indices = create_random_index(1024, 0.5);
c.bench_function("take bool null values null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_array::<i32>(512, 0.0);
let indices = create_random_index(512, 0.0);
c.bench_function("take str 512", |b| b.iter(|| bench_take(&values, &indices)));
let values = create_string_array::<i32>(1024, 0.0);
let indices = create_random_index(1024, 0.0);
c.bench_function("take str 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_array::<i32>(512, 0.0);
let indices = create_random_index(512, 0.5);
c.bench_function("take str null indices 512", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_array::<i32>(1024, 0.0);
let indices = create_random_index(1024, 0.5);
c.bench_function("take str null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_array::<i32>(1024, 0.5);
let indices = create_random_index(1024, 0.0);
c.bench_function("take str null values 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_array::<i32>(1024, 0.5);
let indices = create_random_index(1024, 0.5);
c.bench_function("take str null values null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_view_array(512, 0.0);
let indices = create_random_index(512, 0.0);
c.bench_function("take stringview 512", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_view_array(1024, 0.0);
let indices = create_random_index(1024, 0.0);
c.bench_function("take stringview 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_view_array(512, 0.0);
let indices = create_random_index(512, 0.5);
c.bench_function("take stringview null indices 512", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_view_array(1024, 0.0);
let indices = create_random_index(1024, 0.5);
c.bench_function("take stringview null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_view_array(1024, 0.5);
let indices = create_random_index(1024, 0.0);
c.bench_function("take stringview null values 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_string_view_array(1024, 0.5);
let indices = create_random_index(1024, 0.5);
c.bench_function("take stringview null values null indices 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_primitive_run_array::<Int32Type, Int32Type>(1024, 512);
let indices = create_random_index(1024, 0.0);
c.bench_function(
"take primitive run logical len: 1024, physical len: 512, indices: 1024",
|b| b.iter(|| bench_take(&values, &indices)),
);
let values = create_fsb_array(1024, 0.0, 12);
let indices = create_random_index(1024, 0.0);
c.bench_function("take primitive fsb value len: 12, indices: 1024", |b| {
b.iter(|| bench_take(&values, &indices))
});
let values = create_fsb_array(1024, 0.5, 12);
let indices = create_random_index(1024, 0.0);
c.bench_function(
"take primitive fsb value len: 12, null values, indices: 1024",
|b| b.iter(|| bench_take(&values, &indices)),
);
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);
+327
View File
@@ -0,0 +1,327 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use criterion::measurement::WallTime;
use criterion::{BenchmarkGroup, BenchmarkId, Criterion, criterion_group, criterion_main};
use rand::distr::{Distribution, StandardUniform};
use rand::prelude::StdRng;
use rand::{Rng, SeedableRng};
use std::hint;
use std::ops::Range;
use std::sync::Arc;
use arrow::array::*;
use arrow::datatypes::*;
use arrow::util::bench_util::*;
use arrow_select::zip::zip;
trait InputGenerator {
fn name(&self) -> &str;
/// Return an ArrayRef containing a single null value
fn generate_scalar_with_null_value(&self) -> ArrayRef;
/// Generate a `number_of_scalars` unique scalars
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef>;
/// Generate array with specified length and null percentage
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef;
}
struct GeneratePrimitive<T: ArrowPrimitiveType> {
description: String,
_marker: std::marker::PhantomData<T>,
}
impl<T> InputGenerator for GeneratePrimitive<T>
where
T: ArrowPrimitiveType,
StandardUniform: Distribution<T::Native>,
{
fn name(&self) -> &str {
self.description.as_str()
}
fn generate_scalar_with_null_value(&self) -> ArrayRef {
new_null_array(&T::DATA_TYPE, 1)
}
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef> {
let rng = StdRng::seed_from_u64(seed);
rng.sample_iter::<T::Native, _>(StandardUniform)
.take(number_of_scalars)
.map(|v: T::Native| {
Arc::new(PrimitiveArray::<T>::new_scalar(v).into_inner()) as ArrayRef
})
.collect()
}
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef {
Arc::new(create_primitive_array_with_seed::<T>(
array_length,
null_percentage,
seed,
))
}
}
struct GenerateBytes<Byte: ByteArrayType> {
range_length: std::ops::Range<usize>,
description: String,
_marker: std::marker::PhantomData<Byte>,
}
impl<Byte> InputGenerator for GenerateBytes<Byte>
where
Byte: ByteArrayType,
{
fn name(&self) -> &str {
self.description.as_str()
}
fn generate_scalar_with_null_value(&self) -> ArrayRef {
new_null_array(&Byte::DATA_TYPE, 1)
}
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef> {
let array = self.generate_array(seed, number_of_scalars, 0.0);
(0..number_of_scalars).map(|i| array.slice(i, 1)).collect()
}
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef {
let is_binary =
Byte::DATA_TYPE == DataType::Binary || Byte::DATA_TYPE == DataType::LargeBinary;
if is_binary {
Arc::new(create_binary_array_with_len_range_and_prefix_and_seed::<
Byte::Offset,
>(
array_length,
null_percentage,
self.range_length.start,
self.range_length.end - 1,
&[],
seed,
))
} else {
Arc::new(create_string_array_with_len_range_and_prefix_and_seed::<
Byte::Offset,
>(
array_length,
null_percentage,
self.range_length.start,
self.range_length.end - 1,
"",
seed,
))
}
}
}
struct GenerateStringView {
range: Range<usize>,
description: String,
_marker: std::marker::PhantomData<StringViewType>,
}
impl InputGenerator for GenerateStringView {
fn name(&self) -> &str {
self.description.as_str()
}
fn generate_scalar_with_null_value(&self) -> ArrayRef {
new_null_array(&DataType::Utf8View, 1)
}
fn generate_non_null_scalars(&self, seed: u64, number_of_scalars: usize) -> Vec<ArrayRef> {
let array = self.generate_array(seed, number_of_scalars, 0.0);
(0..number_of_scalars).map(|i| array.slice(i, 1)).collect()
}
fn generate_array(&self, seed: u64, array_length: usize, null_percentage: f32) -> ArrayRef {
Arc::new(create_string_view_array_with_len_range_and_seed(
array_length,
null_percentage,
self.range.clone(),
seed,
))
}
}
fn mask_cases(len: usize) -> Vec<(&'static str, BooleanArray)> {
vec![
("all_true", create_boolean_array(len, 0.0, 1.0)),
("99pct_true", create_boolean_array(len, 0.0, 0.99)),
("90pct_true", create_boolean_array(len, 0.0, 0.9)),
("50pct_true", create_boolean_array(len, 0.0, 0.5)),
("10pct_true", create_boolean_array(len, 0.0, 0.1)),
("1pct_true", create_boolean_array(len, 0.0, 0.01)),
("all_false", create_boolean_array(len, 0.0, 0.0)),
("50pct_nulls", create_boolean_array(len, 0.5, 0.5)),
]
}
fn bench_zip_on_input_generator(c: &mut Criterion, input_generator: &impl InputGenerator) {
const ARRAY_LEN: usize = 8192;
let mut group =
c.benchmark_group(format!("zip_{ARRAY_LEN}_from_{}", input_generator.name()).as_str());
let null_scalar = input_generator.generate_scalar_with_null_value();
let [non_null_scalar_1, non_null_scalar_2]: [_; 2] = input_generator
.generate_non_null_scalars(42, 2)
.try_into()
.unwrap();
let array_1_10pct_nulls = input_generator.generate_array(42, ARRAY_LEN, 0.1);
let array_2_10pct_nulls = input_generator.generate_array(18, ARRAY_LEN, 0.1);
let masks = mask_cases(ARRAY_LEN);
// Benchmarks for different scalar combinations
for (description, truthy, falsy) in &[
("null_vs_non_null_scalar", &null_scalar, &non_null_scalar_1),
(
"non_null_scalar_vs_null_scalar",
&non_null_scalar_1,
&null_scalar,
),
("non_nulls_scalars", &non_null_scalar_1, &non_null_scalar_2),
] {
bench_zip_input_on_all_masks(
description,
&mut group,
&masks,
&Scalar::new(truthy),
&Scalar::new(falsy),
);
}
bench_zip_input_on_all_masks(
"array_vs_non_null_scalar",
&mut group,
&masks,
&array_1_10pct_nulls,
&non_null_scalar_1,
);
bench_zip_input_on_all_masks(
"non_null_scalar_vs_array",
&mut group,
&masks,
&non_null_scalar_1,
&array_1_10pct_nulls,
);
bench_zip_input_on_all_masks(
"array_vs_array",
&mut group,
&masks,
&array_1_10pct_nulls,
&array_2_10pct_nulls,
);
group.finish();
}
fn bench_zip_input_on_all_masks(
description: &str,
group: &mut BenchmarkGroup<WallTime>,
masks: &[(&str, BooleanArray)],
truthy: &impl Datum,
falsy: &impl Datum,
) {
for (mask_description, mask) in masks {
let id = BenchmarkId::new(description, mask_description);
group.bench_with_input(id, mask, |b, mask| {
b.iter(|| hint::black_box(zip(mask, truthy, falsy)))
});
}
}
fn add_benchmark(c: &mut Criterion) {
// Primitive
bench_zip_on_input_generator(
c,
&GeneratePrimitive::<Int32Type> {
description: "i32".to_string(),
_marker: std::marker::PhantomData,
},
);
// Short strings
bench_zip_on_input_generator(
c,
&GenerateBytes::<GenericStringType<i32>> {
description: "short strings (3..10)".to_string(),
range_length: 3..10,
_marker: std::marker::PhantomData,
},
);
// Long strings
bench_zip_on_input_generator(
c,
&GenerateBytes::<GenericStringType<i32>> {
description: "long strings (100..400)".to_string(),
range_length: 100..400,
_marker: std::marker::PhantomData,
},
);
// Short Bytes
bench_zip_on_input_generator(
c,
&GenerateBytes::<GenericBinaryType<i32>> {
description: "short bytes (3..10)".to_string(),
range_length: 3..10,
_marker: std::marker::PhantomData,
},
);
// Long Bytes
bench_zip_on_input_generator(
c,
&GenerateBytes::<GenericBinaryType<i32>> {
description: "long bytes (100..400)".to_string(),
range_length: 100..400,
_marker: std::marker::PhantomData,
},
);
bench_zip_on_input_generator(
c,
&GenerateStringView {
description: "string_views size (3..10)".to_string(),
range: 3..10,
_marker: std::marker::PhantomData,
},
);
bench_zip_on_input_generator(
c,
&GenerateStringView {
description: "string_views size (10..100)".to_string(),
range: 10..100,
_marker: std::marker::PhantomData,
},
);
}
criterion_group!(benches, add_benchmark);
criterion_main!(benches);