Vendor dependencies

notes-service/vendor/arrow-row/src/fixed.rs

@@ -0,0 +1,509 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::array::PrimitiveArray;
+use crate::null_sentinel;
+use arrow_array::builder::BufferBuilder;
+use arrow_array::{ArrowPrimitiveType, BooleanArray, FixedSizeBinaryArray};
+use arrow_buffer::{
+    ArrowNativeType, BooleanBuffer, Buffer, IntervalDayTime, IntervalMonthDayNano, MutableBuffer,
+    NullBuffer, bit_util, i256,
+};
+use arrow_data::{ArrayData, ArrayDataBuilder};
+use arrow_schema::{DataType, SortOptions};
+use half::f16;
+
+pub trait FromSlice {
+    fn from_slice(slice: &[u8], invert: bool) -> Self;
+}
+
+impl<const N: usize> FromSlice for [u8; N] {
+    #[inline]
+    fn from_slice(slice: &[u8], invert: bool) -> Self {
+        let mut t: Self = slice.try_into().unwrap();
+        if invert {
+            t.iter_mut().for_each(|o| *o = !*o);
+        }
+        t
+    }
+}
+
+/// Encodes a value of a particular fixed width type into bytes according to the rules
+/// described on [`super::RowConverter`]
+pub trait FixedLengthEncoding: Copy {
+    const ENCODED_LEN: usize = 1 + std::mem::size_of::<Self::Encoded>();
+
+    type Encoded: Sized + Copy + FromSlice + AsRef<[u8]> + AsMut<[u8]>;
+
+    fn encode(self) -> Self::Encoded;
+
+    fn decode(encoded: Self::Encoded) -> Self;
+}
+
+impl FixedLengthEncoding for bool {
+    type Encoded = [u8; 1];
+
+    fn encode(self) -> [u8; 1] {
+        [self as u8]
+    }
+
+    fn decode(encoded: Self::Encoded) -> Self {
+        encoded[0] != 0
+    }
+}
+
+macro_rules! encode_signed {
+    ($n:expr, $t:ty) => {
+        impl FixedLengthEncoding for $t {
+            type Encoded = [u8; $n];
+
+            fn encode(self) -> [u8; $n] {
+                let mut b = self.to_be_bytes();
+                // Toggle top "sign" bit to ensure consistent sort order
+                b[0] ^= 0x80;
+                b
+            }
+
+            fn decode(mut encoded: Self::Encoded) -> Self {
+                // Toggle top "sign" bit
+                encoded[0] ^= 0x80;
+                Self::from_be_bytes(encoded)
+            }
+        }
+    };
+}
+
+encode_signed!(1, i8);
+encode_signed!(2, i16);
+encode_signed!(4, i32);
+encode_signed!(8, i64);
+encode_signed!(16, i128);
+encode_signed!(32, i256);
+
+macro_rules! encode_unsigned {
+    ($n:expr, $t:ty) => {
+        impl FixedLengthEncoding for $t {
+            type Encoded = [u8; $n];
+
+            fn encode(self) -> [u8; $n] {
+                self.to_be_bytes()
+            }
+
+            fn decode(encoded: Self::Encoded) -> Self {
+                Self::from_be_bytes(encoded)
+            }
+        }
+    };
+}
+
+encode_unsigned!(1, u8);
+encode_unsigned!(2, u16);
+encode_unsigned!(4, u32);
+encode_unsigned!(8, u64);
+
+impl FixedLengthEncoding for f16 {
+    type Encoded = [u8; 2];
+
+    fn encode(self) -> [u8; 2] {
+        // https://github.com/rust-lang/rust/blob/9c20b2a8cc7588decb6de25ac6a7912dcef24d65/library/core/src/num/f32.rs#L1176-L1260
+        let s = self.to_bits() as i16;
+        let val = s ^ (((s >> 15) as u16) >> 1) as i16;
+        val.encode()
+    }
+
+    fn decode(encoded: Self::Encoded) -> Self {
+        let bits = i16::decode(encoded);
+        let val = bits ^ (((bits >> 15) as u16) >> 1) as i16;
+        Self::from_bits(val as u16)
+    }
+}
+
+impl FixedLengthEncoding for f32 {
+    type Encoded = [u8; 4];
+
+    fn encode(self) -> [u8; 4] {
+        // https://github.com/rust-lang/rust/blob/9c20b2a8cc7588decb6de25ac6a7912dcef24d65/library/core/src/num/f32.rs#L1176-L1260
+        let s = self.to_bits() as i32;
+        let val = s ^ (((s >> 31) as u32) >> 1) as i32;
+        val.encode()
+    }
+
+    fn decode(encoded: Self::Encoded) -> Self {
+        let bits = i32::decode(encoded);
+        let val = bits ^ (((bits >> 31) as u32) >> 1) as i32;
+        Self::from_bits(val as u32)
+    }
+}
+
+impl FixedLengthEncoding for f64 {
+    type Encoded = [u8; 8];
+
+    fn encode(self) -> [u8; 8] {
+        // https://github.com/rust-lang/rust/blob/9c20b2a8cc7588decb6de25ac6a7912dcef24d65/library/core/src/num/f32.rs#L1176-L1260
+        let s = self.to_bits() as i64;
+        let val = s ^ (((s >> 63) as u64) >> 1) as i64;
+        val.encode()
+    }
+
+    fn decode(encoded: Self::Encoded) -> Self {
+        let bits = i64::decode(encoded);
+        let val = bits ^ (((bits >> 63) as u64) >> 1) as i64;
+        Self::from_bits(val as u64)
+    }
+}
+
+impl FixedLengthEncoding for IntervalDayTime {
+    type Encoded = [u8; 8];
+
+    fn encode(self) -> Self::Encoded {
+        let mut out = [0_u8; 8];
+        out[..4].copy_from_slice(&self.days.encode());
+        out[4..].copy_from_slice(&self.milliseconds.encode());
+        out
+    }
+
+    fn decode(encoded: Self::Encoded) -> Self {
+        Self {
+            days: i32::decode(encoded[..4].try_into().unwrap()),
+            milliseconds: i32::decode(encoded[4..].try_into().unwrap()),
+        }
+    }
+}
+
+impl FixedLengthEncoding for IntervalMonthDayNano {
+    type Encoded = [u8; 16];
+
+    fn encode(self) -> Self::Encoded {
+        let mut out = [0_u8; 16];
+        out[..4].copy_from_slice(&self.months.encode());
+        out[4..8].copy_from_slice(&self.days.encode());
+        out[8..].copy_from_slice(&self.nanoseconds.encode());
+        out
+    }
+
+    fn decode(encoded: Self::Encoded) -> Self {
+        Self {
+            months: i32::decode(encoded[..4].try_into().unwrap()),
+            days: i32::decode(encoded[4..8].try_into().unwrap()),
+            nanoseconds: i64::decode(encoded[8..].try_into().unwrap()),
+        }
+    }
+}
+
+/// Returns the total encoded length (including null byte) for a value of type `T::Native`
+pub const fn encoded_len<T>(_col: &PrimitiveArray<T>) -> usize
+where
+    T: ArrowPrimitiveType,
+    T::Native: FixedLengthEncoding,
+{
+    T::Native::ENCODED_LEN
+}
+
+/// Fixed width types are encoded as
+///
+/// - 1 byte `0` if null or `1` if valid
+/// - bytes of [`FixedLengthEncoding`]
+pub fn encode<T: FixedLengthEncoding>(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    values: &[T],
+    nulls: &NullBuffer,
+    opts: SortOptions,
+) {
+    for (value_idx, is_valid) in nulls.iter().enumerate() {
+        let offset = &mut offsets[value_idx + 1];
+        let end_offset = *offset + T::ENCODED_LEN;
+        if is_valid {
+            let to_write = &mut data[*offset..end_offset];
+            to_write[0] = 1;
+            let mut encoded = values[value_idx].encode();
+            if opts.descending {
+                // Flip bits to reverse order
+                encoded.as_mut().iter_mut().for_each(|v| *v = !*v)
+            }
+            to_write[1..].copy_from_slice(encoded.as_ref())
+        } else {
+            data[*offset] = null_sentinel(opts);
+        }
+        *offset = end_offset;
+    }
+}
+
+/// Encoding for non-nullable primitive arrays.
+/// Iterates directly over the `values`, and skips NULLs-checking.
+pub fn encode_not_null<T: FixedLengthEncoding>(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    values: &[T],
+    opts: SortOptions,
+) {
+    for (value_idx, val) in values.iter().enumerate() {
+        let offset = &mut offsets[value_idx + 1];
+        let end_offset = *offset + T::ENCODED_LEN;
+
+        let to_write = &mut data[*offset..end_offset];
+        to_write[0] = 1;
+        let mut encoded = val.encode();
+        if opts.descending {
+            // Flip bits to reverse order
+            encoded.as_mut().iter_mut().for_each(|v| *v = !*v)
+        }
+        to_write[1..].copy_from_slice(encoded.as_ref());
+
+        *offset = end_offset;
+    }
+}
+
+/// Boolean values are encoded as
+///
+/// - 1 byte `0` if null or `1` if valid
+/// - bytes of [`FixedLengthEncoding`]
+pub fn encode_boolean(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    values: &BooleanBuffer,
+    nulls: &NullBuffer,
+    opts: SortOptions,
+) {
+    for (idx, is_valid) in nulls.iter().enumerate() {
+        let offset = &mut offsets[idx + 1];
+        let end_offset = *offset + bool::ENCODED_LEN;
+        if is_valid {
+            let to_write = &mut data[*offset..end_offset];
+            to_write[0] = 1;
+            let mut encoded = values.value(idx).encode();
+            if opts.descending {
+                // Flip bits to reverse order
+                encoded.as_mut().iter_mut().for_each(|v| *v = !*v)
+            }
+            to_write[1..].copy_from_slice(encoded.as_ref())
+        } else {
+            data[*offset] = null_sentinel(opts);
+        }
+        *offset = end_offset;
+    }
+}
+
+/// Encoding for non-nullable boolean arrays.
+/// Iterates directly over `values`, and skips NULLs-checking.
+pub fn encode_boolean_not_null(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    values: &BooleanBuffer,
+    opts: SortOptions,
+) {
+    for (value_idx, val) in values.iter().enumerate() {
+        let offset = &mut offsets[value_idx + 1];
+        let end_offset = *offset + bool::ENCODED_LEN;
+
+        let to_write = &mut data[*offset..end_offset];
+        to_write[0] = 1;
+        let mut encoded = val.encode();
+        if opts.descending {
+            // Flip bits to reverse order
+            encoded.as_mut().iter_mut().for_each(|v| *v = !*v)
+        }
+        to_write[1..].copy_from_slice(encoded.as_ref());
+
+        *offset = end_offset;
+    }
+}
+
+pub fn encode_fixed_size_binary(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    array: &FixedSizeBinaryArray,
+    opts: SortOptions,
+) {
+    let len = array.value_length() as usize;
+    for (offset, maybe_val) in offsets.iter_mut().skip(1).zip(array.iter()) {
+        let end_offset = *offset + len + 1;
+        if let Some(val) = maybe_val {
+            let to_write = &mut data[*offset..end_offset];
+            to_write[0] = 1;
+            to_write[1..].copy_from_slice(&val[..len]);
+            if opts.descending {
+                // Flip bits to reverse order
+                to_write[1..1 + len].iter_mut().for_each(|v| *v = !*v)
+            }
+        } else {
+            data[*offset] = null_sentinel(opts);
+        }
+        *offset = end_offset;
+    }
+}
+
+/// Splits `len` bytes from `src`
+#[inline]
+fn split_off<'a>(src: &mut &'a [u8], len: usize) -> &'a [u8] {
+    let v = &src[..len];
+    *src = &src[len..];
+    v
+}
+
+/// Decodes a `BooleanArray` from rows
+pub fn decode_bool(rows: &mut [&[u8]], options: SortOptions) -> BooleanArray {
+    let true_val = match options.descending {
+        true => !1,
+        false => 1,
+    };
+
+    let len = rows.len();
+
+    let mut null_count = 0;
+    let mut nulls = MutableBuffer::new(bit_util::ceil(len, 64) * 8);
+    let mut values = MutableBuffer::new(bit_util::ceil(len, 64) * 8);
+
+    let chunks = len / 64;
+    let remainder = len % 64;
+    for chunk in 0..chunks {
+        let mut null_packed = 0;
+        let mut values_packed = 0;
+
+        for bit_idx in 0..64 {
+            let i = split_off(&mut rows[bit_idx + chunk * 64], 2);
+            let (null, value) = (i[0] == 1, i[1] == true_val);
+            null_count += !null as usize;
+            null_packed |= (null as u64) << bit_idx;
+            values_packed |= (value as u64) << bit_idx;
+        }
+
+        nulls.push(null_packed);
+        values.push(values_packed);
+    }
+
+    if remainder != 0 {
+        let mut null_packed = 0;
+        let mut values_packed = 0;
+
+        for bit_idx in 0..remainder {
+            let i = split_off(&mut rows[bit_idx + chunks * 64], 2);
+            let (null, value) = (i[0] == 1, i[1] == true_val);
+            null_count += !null as usize;
+            null_packed |= (null as u64) << bit_idx;
+            values_packed |= (value as u64) << bit_idx;
+        }
+
+        nulls.push(null_packed);
+        values.push(values_packed);
+    }
+
+    let builder = ArrayDataBuilder::new(DataType::Boolean)
+        .len(rows.len())
+        .null_count(null_count)
+        .add_buffer(values.into())
+        .null_bit_buffer(Some(nulls.into()));
+
+    // SAFETY:
+    // Buffers are the correct length
+    unsafe { BooleanArray::from(builder.build_unchecked()) }
+}
+
+/// Decodes a single byte from each row, interpreting `0x01` as a valid value
+/// and all other values as a null
+///
+/// Returns the null count and null buffer
+pub fn decode_nulls(rows: &[&[u8]]) -> (usize, Buffer) {
+    let mut null_count = 0;
+    let buffer = MutableBuffer::collect_bool(rows.len(), |idx| {
+        let valid = rows[idx][0] == 1;
+        null_count += !valid as usize;
+        valid
+    })
+    .into();
+    (null_count, buffer)
+}
+
+/// Decodes a `ArrayData` from rows based on the provided `FixedLengthEncoding` `T`
+///
+/// # Safety
+///
+/// `data_type` must be appropriate native type for `T`
+unsafe fn decode_fixed<T: FixedLengthEncoding + ArrowNativeType>(
+    rows: &mut [&[u8]],
+    data_type: DataType,
+    options: SortOptions,
+) -> ArrayData {
+    let len = rows.len();
+
+    let mut values = BufferBuilder::<T>::new(len);
+    let (null_count, nulls) = decode_nulls(rows);
+
+    for row in rows {
+        let i = split_off(row, T::ENCODED_LEN);
+        let value = T::Encoded::from_slice(&i[1..], options.descending);
+        values.append(T::decode(value));
+    }
+
+    let builder = ArrayDataBuilder::new(data_type)
+        .len(len)
+        .null_count(null_count)
+        .add_buffer(values.finish())
+        .null_bit_buffer(Some(nulls));
+
+    // SAFETY: Buffers correct length
+    unsafe { builder.build_unchecked() }
+}
+
+/// Decodes a `PrimitiveArray` from rows
+pub fn decode_primitive<T: ArrowPrimitiveType>(
+    rows: &mut [&[u8]],
+    data_type: DataType,
+    options: SortOptions,
+) -> PrimitiveArray<T>
+where
+    T::Native: FixedLengthEncoding,
+{
+    assert!(PrimitiveArray::<T>::is_compatible(&data_type));
+    // SAFETY:
+    // Validated data type above
+    unsafe { decode_fixed::<T::Native>(rows, data_type, options).into() }
+}
+
+/// Decodes a `FixedLengthBinary` from rows
+pub fn decode_fixed_size_binary(
+    rows: &mut [&[u8]],
+    size: i32,
+    options: SortOptions,
+) -> FixedSizeBinaryArray {
+    let len = rows.len();
+
+    let mut values = MutableBuffer::new(size as usize * rows.len());
+    let (null_count, nulls) = decode_nulls(rows);
+
+    let encoded_len = size as usize + 1;
+
+    for row in rows {
+        let i = split_off(row, encoded_len);
+        values.extend_from_slice(&i[1..]);
+    }
+
+    if options.descending {
+        for v in values.as_slice_mut() {
+            *v = !*v;
+        }
+    }
+
+    let builder = ArrayDataBuilder::new(DataType::FixedSizeBinary(size))
+        .len(len)
+        .null_count(null_count)
+        .add_buffer(values.into())
+        .null_bit_buffer(Some(nulls));
+
+    // SAFETY: Buffers correct length
+    unsafe { builder.build_unchecked().into() }
+}

notes-service/vendor/arrow-row/src/list.rs

@@ -0,0 +1,325 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{LengthTracker, RowConverter, Rows, SortField, fixed, null_sentinel};
+use arrow_array::{Array, FixedSizeListArray, GenericListArray, OffsetSizeTrait, new_null_array};
+use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
+use arrow_data::ArrayDataBuilder;
+use arrow_schema::{ArrowError, DataType, SortOptions};
+use std::{ops::Range, sync::Arc};
+
+pub fn compute_lengths<O: OffsetSizeTrait>(
+    lengths: &mut [usize],
+    rows: &Rows,
+    array: &GenericListArray<O>,
+) {
+    let shift = array.value_offsets()[0].as_usize();
+
+    let offsets = array.value_offsets().windows(2);
+    lengths
+        .iter_mut()
+        .zip(offsets)
+        .enumerate()
+        .for_each(|(idx, (length, offsets))| {
+            let start = offsets[0].as_usize() - shift;
+            let end = offsets[1].as_usize() - shift;
+            let range = array.is_valid(idx).then_some(start..end);
+            *length += encoded_len(rows, range);
+        });
+}
+
+fn encoded_len(rows: &Rows, range: Option<Range<usize>>) -> usize {
+    match range {
+        None => 1,
+        Some(range) => {
+            1 + range
+                .map(|i| super::variable::padded_length(Some(rows.row(i).as_ref().len())))
+                .sum::<usize>()
+        }
+    }
+}
+
+/// Encodes the provided `GenericListArray` to `out` with the provided `SortOptions`
+///
+/// `rows` should contain the encoded child elements
+pub fn encode<O: OffsetSizeTrait>(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    rows: &Rows,
+    opts: SortOptions,
+    array: &GenericListArray<O>,
+) {
+    let shift = array.value_offsets()[0].as_usize();
+
+    offsets
+        .iter_mut()
+        .skip(1)
+        .zip(array.value_offsets().windows(2))
+        .enumerate()
+        .for_each(|(idx, (offset, offsets))| {
+            let start = offsets[0].as_usize() - shift;
+            let end = offsets[1].as_usize() - shift;
+            let range = array.is_valid(idx).then_some(start..end);
+            let out = &mut data[*offset..];
+            *offset += encode_one(out, rows, range, opts)
+        });
+}
+
+#[inline]
+fn encode_one(
+    out: &mut [u8],
+    rows: &Rows,
+    range: Option<Range<usize>>,
+    opts: SortOptions,
+) -> usize {
+    match range {
+        None => super::variable::encode_null(out, opts),
+        Some(range) if range.start == range.end => super::variable::encode_empty(out, opts),
+        Some(range) => {
+            let mut offset = 0;
+            for i in range {
+                let row = rows.row(i);
+                offset += super::variable::encode_one(&mut out[offset..], Some(row.data), opts);
+            }
+            offset += super::variable::encode_empty(&mut out[offset..], opts);
+            offset
+        }
+    }
+}
+
+/// Decodes an array from `rows` with the provided `options`
+///
+/// # Safety
+///
+/// `rows` must contain valid data for the provided `converter`
+pub unsafe fn decode<O: OffsetSizeTrait>(
+    converter: &RowConverter,
+    rows: &mut [&[u8]],
+    field: &SortField,
+    validate_utf8: bool,
+) -> Result<GenericListArray<O>, ArrowError> {
+    let opts = field.options;
+
+    let mut values_bytes = 0;
+
+    let mut offset = 0;
+    let mut offsets = Vec::with_capacity(rows.len() + 1);
+    offsets.push(O::usize_as(0));
+
+    for row in rows.iter_mut() {
+        let mut row_offset = 0;
+        loop {
+            let decoded = super::variable::decode_blocks(&row[row_offset..], opts, |x| {
+                values_bytes += x.len();
+            });
+            if decoded <= 1 {
+                offsets.push(O::usize_as(offset));
+                break;
+            }
+            row_offset += decoded;
+            offset += 1;
+        }
+    }
+    O::from_usize(offset).expect("overflow");
+
+    let mut null_count = 0;
+    let nulls = MutableBuffer::collect_bool(rows.len(), |x| {
+        let valid = rows[x][0] != null_sentinel(opts);
+        null_count += !valid as usize;
+        valid
+    });
+
+    let mut values_offsets = Vec::with_capacity(offset);
+    let mut values_bytes = Vec::with_capacity(values_bytes);
+    for row in rows.iter_mut() {
+        let mut row_offset = 0;
+        loop {
+            let decoded = super::variable::decode_blocks(&row[row_offset..], opts, |x| {
+                values_bytes.extend_from_slice(x)
+            });
+            row_offset += decoded;
+            if decoded <= 1 {
+                break;
+            }
+            values_offsets.push(values_bytes.len());
+        }
+        *row = &row[row_offset..];
+    }
+
+    if opts.descending {
+        values_bytes.iter_mut().for_each(|o| *o = !*o);
+    }
+
+    let mut last_value_offset = 0;
+    let mut child_rows: Vec<_> = values_offsets
+        .into_iter()
+        .map(|offset| {
+            let v = &values_bytes[last_value_offset..offset];
+            last_value_offset = offset;
+            v
+        })
+        .collect();
+
+    let child = unsafe { converter.convert_raw(&mut child_rows, validate_utf8) }?;
+    assert_eq!(child.len(), 1);
+
+    let child_data = child[0].to_data();
+
+    // Since RowConverter flattens certain data types (i.e. Dictionary),
+    // we need to use updated data type instead of original field
+    let corrected_type = match &field.data_type {
+        DataType::List(inner_field) => DataType::List(Arc::new(
+            inner_field
+                .as_ref()
+                .clone()
+                .with_data_type(child_data.data_type().clone()),
+        )),
+        DataType::LargeList(inner_field) => DataType::LargeList(Arc::new(
+            inner_field
+                .as_ref()
+                .clone()
+                .with_data_type(child_data.data_type().clone()),
+        )),
+        _ => unreachable!(),
+    };
+
+    let builder = ArrayDataBuilder::new(corrected_type)
+        .len(rows.len())
+        .null_count(null_count)
+        .null_bit_buffer(Some(nulls.into()))
+        .add_buffer(Buffer::from_vec(offsets))
+        .add_child_data(child_data);
+
+    Ok(GenericListArray::from(unsafe { builder.build_unchecked() }))
+}
+
+pub fn compute_lengths_fixed_size_list(
+    tracker: &mut LengthTracker,
+    rows: &Rows,
+    array: &FixedSizeListArray,
+) {
+    let value_length = array.value_length().as_usize();
+    tracker.push_variable((0..array.len()).map(|idx| {
+        match array.is_valid(idx) {
+            true => {
+                1 + ((idx * value_length)..(idx + 1) * value_length)
+                    .map(|child_idx| rows.row(child_idx).as_ref().len())
+                    .sum::<usize>()
+            }
+            false => 1,
+        }
+    }))
+}
+
+/// Encodes the provided `FixedSizeListArray` to `out` with the provided `SortOptions`
+///
+/// `rows` should contain the encoded child elements
+pub fn encode_fixed_size_list(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    rows: &Rows,
+    opts: SortOptions,
+    array: &FixedSizeListArray,
+) {
+    let null_sentinel = null_sentinel(opts);
+    offsets
+        .iter_mut()
+        .skip(1)
+        .enumerate()
+        .for_each(|(idx, offset)| {
+            let value_length = array.value_length().as_usize();
+            match array.is_valid(idx) {
+                true => {
+                    data[*offset] = 0x01;
+                    *offset += 1;
+                    for child_idx in (idx * value_length)..(idx + 1) * value_length {
+                        let row = rows.row(child_idx);
+                        let end_offset = *offset + row.as_ref().len();
+                        data[*offset..end_offset].copy_from_slice(row.as_ref());
+                        *offset = end_offset;
+                    }
+                }
+                false => {
+                    data[*offset] = null_sentinel;
+                    *offset += 1;
+                }
+            };
+        })
+}
+
+/// Decodes a fixed size list array from `rows` with the provided `options`
+///
+/// # Safety
+///
+/// `rows` must contain valid data for the provided `converter`
+pub unsafe fn decode_fixed_size_list(
+    converter: &RowConverter,
+    rows: &mut [&[u8]],
+    field: &SortField,
+    validate_utf8: bool,
+    value_length: usize,
+) -> Result<FixedSizeListArray, ArrowError> {
+    let list_type = &field.data_type;
+    let element_type = match list_type {
+        DataType::FixedSizeList(element_field, _) => element_field.data_type(),
+        _ => {
+            return Err(ArrowError::InvalidArgumentError(format!(
+                "Expected FixedSizeListArray, found: {list_type}",
+            )));
+        }
+    };
+
+    let len = rows.len();
+    let (null_count, nulls) = fixed::decode_nulls(rows);
+
+    let null_element_encoded = converter.convert_columns(&[new_null_array(element_type, 1)])?;
+    let null_element_encoded = null_element_encoded.row(0);
+    let null_element_slice = null_element_encoded.as_ref();
+
+    let mut child_rows = Vec::new();
+    for row in rows {
+        let valid = row[0] == 1;
+        let mut row_offset = 1;
+        if !valid {
+            for _ in 0..value_length {
+                child_rows.push(null_element_slice);
+            }
+        } else {
+            for _ in 0..value_length {
+                let mut temp_child_rows = vec![&row[row_offset..]];
+                unsafe { converter.convert_raw(&mut temp_child_rows, validate_utf8) }?;
+                let decoded_bytes = row.len() - row_offset - temp_child_rows[0].len();
+                let next_offset = row_offset + decoded_bytes;
+                child_rows.push(&row[row_offset..next_offset]);
+                row_offset = next_offset;
+            }
+        }
+        *row = &row[row_offset..]; // Update row for the next decoder
+    }
+
+    let children = unsafe { converter.convert_raw(&mut child_rows, validate_utf8) }?;
+    let child_data = children.iter().map(|c| c.to_data()).collect();
+    let builder = ArrayDataBuilder::new(list_type.clone())
+        .len(len)
+        .null_count(null_count)
+        .null_bit_buffer(Some(nulls))
+        .child_data(child_data);
+
+    Ok(FixedSizeListArray::from(unsafe {
+        builder.build_unchecked()
+    }))
+}

notes-service/vendor/arrow-row/src/run.rs

@@ -0,0 +1,642 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{RowConverter, Rows, SortField, variable};
+use arrow_array::types::RunEndIndexType;
+use arrow_array::{PrimitiveArray, RunArray};
+use arrow_buffer::{ArrowNativeType, ScalarBuffer};
+use arrow_schema::{ArrowError, SortOptions};
+
+/// Computes the lengths of each row for a RunEndEncodedArray
+pub fn compute_lengths<R: RunEndIndexType>(
+    lengths: &mut [usize],
+    rows: &Rows,
+    array: &RunArray<R>,
+) {
+    let run_ends = array.run_ends().values();
+    let mut logical_start = 0;
+
+    // Iterate over each run and apply the same length to all logical positions in the run
+    for (physical_idx, &run_end) in run_ends.iter().enumerate() {
+        let logical_end = run_end.as_usize();
+        let row = rows.row(physical_idx);
+        let encoded_len = variable::encoded_len(Some(row.data));
+
+        // Add the same length for all logical positions in this run
+        for length in &mut lengths[logical_start..logical_end] {
+            *length += encoded_len;
+        }
+
+        logical_start = logical_end;
+    }
+}
+
+/// Encodes the provided `RunEndEncodedArray` to `out` with the provided `SortOptions`
+///
+/// `rows` should contain the encoded values
+pub fn encode<R: RunEndIndexType>(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    rows: &Rows,
+    opts: SortOptions,
+    array: &RunArray<R>,
+) {
+    let run_ends = array.run_ends();
+
+    let mut logical_idx = 0;
+    let mut offset_idx = 1; // Skip first offset
+
+    // Iterate over each run
+    for physical_idx in 0..run_ends.values().len() {
+        let run_end = run_ends.values()[physical_idx].as_usize();
+
+        // Process all elements in this run
+        while logical_idx < run_end && offset_idx < offsets.len() {
+            let offset = &mut offsets[offset_idx];
+            let out = &mut data[*offset..];
+
+            // Use variable-length encoding to make the data self-describing
+            let row = rows.row(physical_idx);
+            let bytes_written = variable::encode_one(out, Some(row.data), opts);
+            *offset += bytes_written;
+
+            logical_idx += 1;
+            offset_idx += 1;
+        }
+
+        // Break if we've processed all offsets
+        if offset_idx >= offsets.len() {
+            break;
+        }
+    }
+}
+
+/// Decodes a RunEndEncodedArray from `rows` with the provided `options`
+///
+/// # Safety
+///
+/// `rows` must contain valid data for the provided `converter`
+pub unsafe fn decode<R: RunEndIndexType>(
+    converter: &RowConverter,
+    rows: &mut [&[u8]],
+    field: &SortField,
+    validate_utf8: bool,
+) -> Result<RunArray<R>, ArrowError> {
+    if rows.is_empty() {
+        let values = unsafe { converter.convert_raw(&mut [], validate_utf8) }?;
+        let run_ends_array = PrimitiveArray::<R>::try_new(ScalarBuffer::from(vec![]), None)?;
+        return RunArray::<R>::try_new(&run_ends_array, &values[0]);
+    }
+
+    // Decode each row's REE data and collect the decoded values
+    let mut decoded_values = Vec::new();
+    let mut run_ends = Vec::new();
+    let mut unique_row_indices = Vec::new();
+
+    // Process each row to extract its REE data (following decode_binary pattern)
+    let mut decoded_data = Vec::new();
+    for (idx, row) in rows.iter_mut().enumerate() {
+        decoded_data.clear();
+        // Extract the decoded value data from this row
+        let consumed = variable::decode_blocks(row, field.options, |block| {
+            decoded_data.extend_from_slice(block);
+        });
+
+        // Handle bit inversion for descending sort (following decode_binary pattern)
+        if field.options.descending {
+            decoded_data.iter_mut().for_each(|b| *b = !*b);
+        }
+
+        // Update the row to point past the consumed REE data
+        *row = &row[consumed..];
+
+        // Check if this decoded value is the same as the previous one to identify runs
+        let is_new_run =
+            idx == 0 || decoded_data != decoded_values[*unique_row_indices.last().unwrap()];
+
+        if is_new_run {
+            // This is a new unique value - end the previous run if any
+            if idx > 0 {
+                run_ends.push(R::Native::usize_as(idx));
+            }
+            unique_row_indices.push(decoded_values.len());
+            let capacity = decoded_data.capacity();
+            decoded_values.push(std::mem::replace(
+                &mut decoded_data,
+                Vec::with_capacity(capacity),
+            ));
+        }
+    }
+    // Add the final run end
+    run_ends.push(R::Native::usize_as(rows.len()));
+
+    // Convert the unique decoded values using the row converter
+    let mut unique_rows: Vec<&[u8]> = decoded_values.iter().map(|v| v.as_slice()).collect();
+    let values = if unique_rows.is_empty() {
+        unsafe { converter.convert_raw(&mut [], validate_utf8) }?
+    } else {
+        unsafe { converter.convert_raw(&mut unique_rows, validate_utf8) }?
+    };
+
+    // Create run ends array
+    let run_ends_array = PrimitiveArray::<R>::try_new(ScalarBuffer::from(run_ends), None)?;
+
+    // Create the RunEndEncodedArray
+    RunArray::<R>::try_new(&run_ends_array, &values[0])
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{RowConverter, SortField};
+    use arrow_array::cast::AsArray;
+    use arrow_array::types::{Int16Type, Int32Type, Int64Type, RunEndIndexType};
+    use arrow_array::{Array, Int64Array, PrimitiveArray, RunArray, StringArray};
+    use arrow_schema::{DataType, SortOptions};
+    use std::sync::Arc;
+
+    fn assert_roundtrip<R: RunEndIndexType>(
+        array: &RunArray<R>,
+        run_end_type: DataType,
+        values_type: DataType,
+        sort_options: Option<SortOptions>,
+    ) {
+        let sort_field = if let Some(options) = sort_options {
+            SortField::new_with_options(
+                DataType::RunEndEncoded(
+                    Arc::new(arrow_schema::Field::new("run_ends", run_end_type, false)),
+                    Arc::new(arrow_schema::Field::new("values", values_type, true)),
+                ),
+                options,
+            )
+        } else {
+            SortField::new(DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", run_end_type, false)),
+                Arc::new(arrow_schema::Field::new("values", values_type, true)),
+            ))
+        };
+
+        let converter = RowConverter::new(vec![sort_field]).unwrap();
+
+        let rows = converter
+            .convert_columns(&[Arc::new(array.clone())])
+            .unwrap();
+
+        let arrays = converter.convert_rows(&rows).unwrap();
+        let result = arrays[0].as_any().downcast_ref::<RunArray<R>>().unwrap();
+
+        assert_eq!(array, result);
+    }
+
+    #[test]
+    fn test_run_end_encoded_supports_datatype() {
+        // Test that the RowConverter correctly supports run-end encoded arrays
+        assert!(RowConverter::supports_datatype(&DataType::RunEndEncoded(
+            Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+            Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+        )));
+    }
+
+    #[test]
+    fn test_run_end_encoded_round_trip_int16_int64s() {
+        // Test round-trip correctness for RunEndEncodedArray with Int64 values making sure it
+        // doesn't just work with eg. strings (which are all the other tests).
+
+        let values = Int64Array::from(vec![100, 200, 100, 300]);
+        let run_ends = vec![2, 3, 5, 6];
+        let array: RunArray<Int16Type> =
+            RunArray::try_new(&PrimitiveArray::from(run_ends), &values).unwrap();
+
+        assert_roundtrip(&array, DataType::Int16, DataType::Int64, None);
+    }
+
+    #[test]
+    fn test_run_end_encoded_round_trip_int32_int64s() {
+        // Test round-trip correctness for RunEndEncodedArray with Int64 values making sure it
+        // doesn't just work with eg. strings (which are all the other tests).
+
+        let values = Int64Array::from(vec![100, 200, 100, 300]);
+        let run_ends = vec![2, 3, 5, 6];
+        let array: RunArray<Int32Type> =
+            RunArray::try_new(&PrimitiveArray::from(run_ends), &values).unwrap();
+
+        assert_roundtrip(&array, DataType::Int32, DataType::Int64, None);
+    }
+
+    #[test]
+    fn test_run_end_encoded_round_trip_int64_int64s() {
+        // Test round-trip correctness for RunEndEncodedArray with Int64 values making sure it
+        // doesn't just work with eg. strings (which are all the other tests).
+
+        let values = Int64Array::from(vec![100, 200, 100, 300]);
+        let run_ends = vec![2, 3, 5, 6];
+        let array: RunArray<Int64Type> =
+            RunArray::try_new(&PrimitiveArray::from(run_ends), &values).unwrap();
+
+        assert_roundtrip(&array, DataType::Int64, DataType::Int64, None);
+    }
+
+    #[test]
+    fn test_run_end_encoded_round_trip_strings() {
+        // Test round-trip correctness for RunEndEncodedArray with strings
+
+        let array: RunArray<Int32Type> = vec!["b", "b", "a"].into_iter().collect();
+
+        assert_roundtrip(&array, DataType::Int32, DataType::Utf8, None);
+    }
+
+    #[test]
+    fn test_run_end_encoded_round_trip_strings_with_nulls() {
+        // Test round-trip correctness for RunEndEncodedArray with nulls
+
+        let array: RunArray<Int32Type> = vec![Some("b"), Some("b"), None, Some("a")]
+            .into_iter()
+            .collect();
+
+        assert_roundtrip(&array, DataType::Int32, DataType::Utf8, None);
+    }
+
+    #[test]
+    fn test_run_end_encoded_ascending_descending_round_trip() {
+        // Test round-trip correctness for ascending vs descending sort options
+
+        let values_asc =
+            arrow_array::StringArray::from(vec![Some("apple"), Some("banana"), Some("cherry")]);
+        let run_ends_asc = vec![2, 4, 6];
+        let run_array_asc: RunArray<Int32Type> = RunArray::try_new(
+            &arrow_array::PrimitiveArray::from(run_ends_asc),
+            &values_asc,
+        )
+        .unwrap();
+
+        // Test ascending order
+        assert_roundtrip(
+            &run_array_asc,
+            DataType::Int32,
+            DataType::Utf8,
+            Some(SortOptions {
+                descending: false,
+                nulls_first: true,
+            }),
+        );
+
+        // Test descending order
+        assert_roundtrip(
+            &run_array_asc,
+            DataType::Int32,
+            DataType::Utf8,
+            Some(SortOptions {
+                descending: true,
+                nulls_first: true,
+            }),
+        );
+    }
+
+    #[test]
+    fn test_run_end_encoded_sort_configurations_basic() {
+        // Test that different sort configurations work and can round-trip successfully
+
+        let test_array: RunArray<Int32Type> = vec!["test"].into_iter().collect();
+
+        // Test ascending order
+        assert_roundtrip(
+            &test_array,
+            DataType::Int32,
+            DataType::Utf8,
+            Some(SortOptions {
+                descending: false,
+                nulls_first: true,
+            }),
+        );
+
+        // Test descending order
+        assert_roundtrip(
+            &test_array,
+            DataType::Int32,
+            DataType::Utf8,
+            Some(SortOptions {
+                descending: true,
+                nulls_first: true,
+            }),
+        );
+    }
+
+    #[test]
+    fn test_run_end_encoded_nulls_first_last_configurations() {
+        // Test that nulls_first vs nulls_last configurations work
+
+        let simple_array: RunArray<Int32Type> = vec!["simple"].into_iter().collect();
+
+        let converter_nulls_first = RowConverter::new(vec![SortField::new_with_options(
+            DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            ),
+            SortOptions {
+                descending: false,
+                nulls_first: true,
+            },
+        )])
+        .unwrap();
+
+        let converter_nulls_last = RowConverter::new(vec![SortField::new_with_options(
+            DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            ),
+            SortOptions {
+                descending: false,
+                nulls_first: false,
+            },
+        )])
+        .unwrap();
+
+        // Test that both configurations can handle simple arrays
+        let rows_nulls_first = converter_nulls_first
+            .convert_columns(&[Arc::new(simple_array.clone())])
+            .unwrap();
+        let arrays_nulls_first = converter_nulls_first
+            .convert_rows(&rows_nulls_first)
+            .unwrap();
+        let result_nulls_first = arrays_nulls_first[0]
+            .as_any()
+            .downcast_ref::<RunArray<Int32Type>>()
+            .unwrap();
+
+        let rows_nulls_last = converter_nulls_last
+            .convert_columns(&[Arc::new(simple_array.clone())])
+            .unwrap();
+        let arrays_nulls_last = converter_nulls_last.convert_rows(&rows_nulls_last).unwrap();
+        let result_nulls_last = arrays_nulls_last[0]
+            .as_any()
+            .downcast_ref::<RunArray<Int32Type>>()
+            .unwrap();
+
+        // Both should successfully convert the simple array
+        assert_eq!(simple_array.len(), result_nulls_first.len());
+        assert_eq!(simple_array.len(), result_nulls_last.len());
+    }
+
+    #[test]
+    fn test_run_end_encoded_row_consumption() {
+        // This test verifies that ALL rows are properly consumed during decoding,
+        // not just the unique values. We test this by ensuring multi-column conversion
+        // works correctly - if rows aren't consumed properly, the second column would fail.
+
+        // Create a REE array with multiple runs
+        let array: RunArray<Int32Type> = vec!["a", "a", "b", "b", "b", "c"].into_iter().collect();
+        let string_array = StringArray::from(vec!["x", "y", "z", "w", "u", "v"]);
+
+        let multi_converter = RowConverter::new(vec![
+            SortField::new(DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            )),
+            SortField::new(DataType::Utf8),
+        ])
+        .unwrap();
+
+        let multi_rows = multi_converter
+            .convert_columns(&[Arc::new(array.clone()), Arc::new(string_array.clone())])
+            .unwrap();
+
+        // Convert back - this will test that all rows are consumed properly
+        let arrays = multi_converter.convert_rows(&multi_rows).unwrap();
+
+        // Verify both columns round-trip correctly
+        let result_ree = arrays[0]
+            .as_any()
+            .downcast_ref::<RunArray<Int32Type>>()
+            .unwrap();
+
+        let result_string = arrays[1].as_any().downcast_ref::<StringArray>().unwrap();
+
+        // This should pass - both arrays should be identical to originals
+        assert_eq!(result_ree.values().as_ref(), array.values().as_ref());
+        assert_eq!(result_ree.run_ends().values(), array.run_ends().values());
+        assert_eq!(*result_string, string_array);
+    }
+
+    #[test]
+    fn test_run_end_encoded_sorting_behavior() {
+        // Test that the binary row encoding actually produces the correct sort order
+
+        // Create REE arrays with different values to test sorting
+        let array1: RunArray<Int32Type> = vec!["apple", "apple"].into_iter().collect();
+        let array2: RunArray<Int32Type> = vec!["banana", "banana"].into_iter().collect();
+        let array3: RunArray<Int32Type> = vec!["cherry", "cherry"].into_iter().collect();
+
+        // Test ascending sort
+        let converter_asc = RowConverter::new(vec![SortField::new(DataType::RunEndEncoded(
+            Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+            Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+        ))])
+        .unwrap();
+
+        let rows1_asc = converter_asc
+            .convert_columns(&[Arc::new(array1.clone())])
+            .unwrap();
+        let rows2_asc = converter_asc
+            .convert_columns(&[Arc::new(array2.clone())])
+            .unwrap();
+        let rows3_asc = converter_asc
+            .convert_columns(&[Arc::new(array3.clone())])
+            .unwrap();
+
+        // For ascending: apple < banana < cherry
+        // So row bytes should sort: rows1 < rows2 < rows3
+        assert!(
+            rows1_asc.row(0) < rows2_asc.row(0),
+            "apple should come before banana in ascending order"
+        );
+        assert!(
+            rows2_asc.row(0) < rows3_asc.row(0),
+            "banana should come before cherry in ascending order"
+        );
+        assert!(
+            rows1_asc.row(0) < rows3_asc.row(0),
+            "apple should come before cherry in ascending order"
+        );
+
+        // Test descending sort
+        let converter_desc = RowConverter::new(vec![SortField::new_with_options(
+            DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            ),
+            arrow_schema::SortOptions {
+                descending: true,
+                nulls_first: true,
+            },
+        )])
+        .unwrap();
+
+        let rows1_desc = converter_desc
+            .convert_columns(&[Arc::new(array1.clone())])
+            .unwrap();
+        let rows2_desc = converter_desc
+            .convert_columns(&[Arc::new(array2.clone())])
+            .unwrap();
+        let rows3_desc = converter_desc
+            .convert_columns(&[Arc::new(array3.clone())])
+            .unwrap();
+
+        // For descending: cherry > banana > apple
+        // So row bytes should sort: rows3 < rows2 < rows1 (because byte comparison is ascending)
+        assert!(
+            rows3_desc.row(0) < rows2_desc.row(0),
+            "cherry should come before banana in descending order (byte-wise)"
+        );
+        assert!(
+            rows2_desc.row(0) < rows1_desc.row(0),
+            "banana should come before apple in descending order (byte-wise)"
+        );
+        assert!(
+            rows3_desc.row(0) < rows1_desc.row(0),
+            "cherry should come before apple in descending order (byte-wise)"
+        );
+    }
+
+    #[test]
+    fn test_run_end_encoded_null_sorting() {
+        // Test null handling in sort order
+
+        let array_with_nulls: RunArray<Int32Type> = vec![None, None].into_iter().collect();
+        let array_with_values: RunArray<Int32Type> = vec!["apple", "apple"].into_iter().collect();
+
+        // Test nulls_first = true
+        let converter_nulls_first = RowConverter::new(vec![SortField::new_with_options(
+            DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            ),
+            arrow_schema::SortOptions {
+                descending: false,
+                nulls_first: true,
+            },
+        )])
+        .unwrap();
+
+        let rows_nulls = converter_nulls_first
+            .convert_columns(&[Arc::new(array_with_nulls.clone())])
+            .unwrap();
+        let rows_values = converter_nulls_first
+            .convert_columns(&[Arc::new(array_with_values.clone())])
+            .unwrap();
+
+        // nulls should come before values when nulls_first = true
+        assert!(
+            rows_nulls.row(0) < rows_values.row(0),
+            "nulls should come before values when nulls_first=true"
+        );
+
+        // Test nulls_first = false
+        let converter_nulls_last = RowConverter::new(vec![SortField::new_with_options(
+            DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            ),
+            arrow_schema::SortOptions {
+                descending: false,
+                nulls_first: false,
+            },
+        )])
+        .unwrap();
+
+        let rows_nulls_last = converter_nulls_last
+            .convert_columns(&[Arc::new(array_with_nulls.clone())])
+            .unwrap();
+        let rows_values_last = converter_nulls_last
+            .convert_columns(&[Arc::new(array_with_values.clone())])
+            .unwrap();
+
+        // values should come before nulls when nulls_first = false
+        assert!(
+            rows_values_last.row(0) < rows_nulls_last.row(0),
+            "values should come before nulls when nulls_first=false"
+        );
+    }
+
+    #[test]
+    fn test_run_end_encoded_mixed_sorting() {
+        // Test sorting with mixed values and nulls to ensure complex scenarios work
+
+        let array1: RunArray<Int32Type> = vec![Some("apple"), None].into_iter().collect();
+        let array2: RunArray<Int32Type> = vec![None, Some("banana")].into_iter().collect();
+        let array3: RunArray<Int32Type> =
+            vec![Some("cherry"), Some("cherry")].into_iter().collect();
+
+        let converter = RowConverter::new(vec![SortField::new_with_options(
+            DataType::RunEndEncoded(
+                Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+                Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+            ),
+            arrow_schema::SortOptions {
+                descending: false,
+                nulls_first: true,
+            },
+        )])
+        .unwrap();
+
+        let rows1 = converter.convert_columns(&[Arc::new(array1)]).unwrap();
+        let rows2 = converter.convert_columns(&[Arc::new(array2)]).unwrap();
+        let rows3 = converter.convert_columns(&[Arc::new(array3)]).unwrap();
+
+        // With nulls_first=true, ascending:
+        // Row 0: array1[0]="apple", array2[0]=null, array3[0]="cherry" -> null < apple < cherry
+        // Row 1: array1[1]=null, array2[1]="banana", array3[1]="cherry" -> null < banana < cherry
+
+        // Compare first rows: null < apple < cherry
+        assert!(rows2.row(0) < rows1.row(0), "null should come before apple");
+        assert!(
+            rows1.row(0) < rows3.row(0),
+            "apple should come before cherry"
+        );
+
+        // Compare second rows: null < banana < cherry
+        assert!(
+            rows1.row(1) < rows2.row(1),
+            "null should come before banana"
+        );
+        assert!(
+            rows2.row(1) < rows3.row(1),
+            "banana should come before cherry"
+        );
+    }
+
+    #[test]
+    fn test_run_end_encoded_empty() {
+        // Test converting / decoding an empty RunEndEncodedArray
+        let values: Vec<&str> = vec![];
+        let array: RunArray<Int32Type> = values.into_iter().collect();
+
+        let converter = RowConverter::new(vec![SortField::new(DataType::RunEndEncoded(
+            Arc::new(arrow_schema::Field::new("run_ends", DataType::Int32, false)),
+            Arc::new(arrow_schema::Field::new("values", DataType::Utf8, true)),
+        ))])
+        .unwrap();
+
+        let rows = converter.convert_columns(&[Arc::new(array)]).unwrap();
+        assert_eq!(rows.num_rows(), 0);
+
+        // Likewise converting empty rows should yield an empty RunEndEncodedArray
+        let arrays = converter.convert_rows(&rows).unwrap();
+        assert_eq!(arrays.len(), 1);
+        // Verify both columns round-trip correctly
+        let result_ree = arrays[0].as_run::<Int32Type>();
+        assert_eq!(result_ree.len(), 0);
+    }
+}

notes-service/vendor/arrow-row/src/variable.rs

@@ -0,0 +1,420 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::null_sentinel;
+use arrow_array::builder::BufferBuilder;
+use arrow_array::types::ByteArrayType;
+use arrow_array::*;
+use arrow_buffer::bit_util::ceil;
+use arrow_buffer::{ArrowNativeType, MutableBuffer};
+use arrow_data::{ArrayDataBuilder, MAX_INLINE_VIEW_LEN};
+use arrow_schema::{DataType, SortOptions};
+use builder::make_view;
+
+/// The block size of the variable length encoding
+pub const BLOCK_SIZE: usize = 32;
+
+/// The first block is split into `MINI_BLOCK_COUNT` mini-blocks
+///
+/// This helps to reduce the space amplification for small strings
+pub const MINI_BLOCK_COUNT: usize = 4;
+
+/// The mini block size
+pub const MINI_BLOCK_SIZE: usize = BLOCK_SIZE / MINI_BLOCK_COUNT;
+
+/// The continuation token
+pub const BLOCK_CONTINUATION: u8 = 0xFF;
+
+/// Indicates an empty string
+pub const EMPTY_SENTINEL: u8 = 1;
+
+/// Indicates a non-empty string
+pub const NON_EMPTY_SENTINEL: u8 = 2;
+
+/// Returns the length of the encoded representation of a byte array, including the null byte
+#[inline]
+pub fn encoded_len(a: Option<&[u8]>) -> usize {
+    padded_length(a.map(|x| x.len()))
+}
+
+/// Returns the padded length of the encoded length of the given length
+#[inline]
+pub fn padded_length(a: Option<usize>) -> usize {
+    match a {
+        Some(a) if a <= BLOCK_SIZE => 1 + ceil(a, MINI_BLOCK_SIZE) * (MINI_BLOCK_SIZE + 1),
+        // Each miniblock ends with a 1 byte continuation, therefore add
+        // `(MINI_BLOCK_COUNT - 1)` additional bytes over non-miniblock size
+        Some(a) => MINI_BLOCK_COUNT + ceil(a, BLOCK_SIZE) * (BLOCK_SIZE + 1),
+        None => 1,
+    }
+}
+
+/// Variable length values are encoded as
+///
+/// - single `0_u8` if null
+/// - single `1_u8` if empty array
+/// - `2_u8` if not empty, followed by one or more blocks
+///
+/// where a block is encoded as
+///
+/// - [`BLOCK_SIZE`] bytes of string data, padded with 0s
+/// - `0xFF_u8` if this is not the last block for this string
+/// - otherwise the length of the block as a `u8`
+pub fn encode<'a, I: Iterator<Item = Option<&'a [u8]>>>(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    i: I,
+    opts: SortOptions,
+) {
+    for (offset, maybe_val) in offsets.iter_mut().skip(1).zip(i) {
+        *offset += encode_one(&mut data[*offset..], maybe_val, opts);
+    }
+}
+
+/// Calls [`encode`] with optimized iterator for generic byte arrays
+pub(crate) fn encode_generic_byte_array<T: ByteArrayType>(
+    data: &mut [u8],
+    offsets: &mut [usize],
+    input_array: &GenericByteArray<T>,
+    opts: SortOptions,
+) {
+    let input_offsets = input_array.value_offsets();
+    let bytes = input_array.values().as_slice();
+
+    if let Some(null_buffer) = input_array.nulls().filter(|x| x.null_count() > 0) {
+        let input_iter =
+            input_offsets
+                .windows(2)
+                .zip(null_buffer.iter())
+                .map(|(start_end, is_valid)| {
+                    if is_valid {
+                        let item_range = start_end[0].as_usize()..start_end[1].as_usize();
+                        // SAFETY: the offsets of the input are valid by construction
+                        // so it is ok to use unsafe here
+                        let item = unsafe { bytes.get_unchecked(item_range) };
+                        Some(item)
+                    } else {
+                        None
+                    }
+                });
+
+        encode(data, offsets, input_iter, opts);
+    } else {
+        // Skip null checks
+        let input_iter = input_offsets.windows(2).map(|start_end| {
+            let item_range = start_end[0].as_usize()..start_end[1].as_usize();
+            // SAFETY: the offsets of the input are valid by construction
+            // so it is ok to use unsafe here
+            let item = unsafe { bytes.get_unchecked(item_range) };
+            Some(item)
+        });
+
+        encode(data, offsets, input_iter, opts);
+    }
+}
+
+pub fn encode_null(out: &mut [u8], opts: SortOptions) -> usize {
+    out[0] = null_sentinel(opts);
+    1
+}
+
+pub fn encode_empty(out: &mut [u8], opts: SortOptions) -> usize {
+    out[0] = match opts.descending {
+        true => !EMPTY_SENTINEL,
+        false => EMPTY_SENTINEL,
+    };
+    1
+}
+
+#[inline]
+pub fn encode_one(out: &mut [u8], val: Option<&[u8]>, opts: SortOptions) -> usize {
+    match val {
+        None => encode_null(out, opts),
+        Some([]) => encode_empty(out, opts),
+        Some(val) => {
+            // Write `2_u8` to demarcate as non-empty, non-null string
+            out[0] = NON_EMPTY_SENTINEL;
+
+            let len = if val.len() <= BLOCK_SIZE {
+                1 + encode_blocks::<MINI_BLOCK_SIZE>(&mut out[1..], val)
+            } else {
+                let (initial, rem) = val.split_at(BLOCK_SIZE);
+                let offset = encode_blocks::<MINI_BLOCK_SIZE>(&mut out[1..], initial);
+                out[offset] = BLOCK_CONTINUATION;
+                1 + offset + encode_blocks::<BLOCK_SIZE>(&mut out[1 + offset..], rem)
+            };
+
+            if opts.descending {
+                // Invert bits
+                out[..len].iter_mut().for_each(|v| *v = !*v)
+            }
+            len
+        }
+    }
+}
+
+/// Writes `val` in `SIZE` blocks with the appropriate continuation tokens
+#[inline]
+fn encode_blocks<const SIZE: usize>(out: &mut [u8], val: &[u8]) -> usize {
+    let block_count = ceil(val.len(), SIZE);
+    let end_offset = block_count * (SIZE + 1);
+    let to_write = &mut out[..end_offset];
+
+    let chunks = val.chunks_exact(SIZE);
+    let remainder = chunks.remainder();
+    for (input, output) in chunks.clone().zip(to_write.chunks_exact_mut(SIZE + 1)) {
+        let input: &[u8; SIZE] = input.try_into().unwrap();
+        let out_block: &mut [u8; SIZE] = (&mut output[..SIZE]).try_into().unwrap();
+
+        *out_block = *input;
+
+        // Indicate that there are further blocks to follow
+        output[SIZE] = BLOCK_CONTINUATION;
+    }
+
+    if !remainder.is_empty() {
+        let start_offset = (block_count - 1) * (SIZE + 1);
+        to_write[start_offset..start_offset + remainder.len()].copy_from_slice(remainder);
+        *to_write.last_mut().unwrap() = remainder.len() as u8;
+    } else {
+        // We must overwrite the continuation marker written by the loop above
+        *to_write.last_mut().unwrap() = SIZE as u8;
+    }
+    end_offset
+}
+
+/// Decodes a single block of data
+/// The `f` function accepts a slice of the decoded data, it may be called multiple times
+pub fn decode_blocks(row: &[u8], options: SortOptions, mut f: impl FnMut(&[u8])) -> usize {
+    let (non_empty_sentinel, continuation) = match options.descending {
+        true => (!NON_EMPTY_SENTINEL, !BLOCK_CONTINUATION),
+        false => (NON_EMPTY_SENTINEL, BLOCK_CONTINUATION),
+    };
+
+    if row[0] != non_empty_sentinel {
+        // Empty or null string
+        return 1;
+    }
+
+    // Extracts the block length from the sentinel
+    let block_len = |sentinel: u8| match options.descending {
+        true => !sentinel as usize,
+        false => sentinel as usize,
+    };
+
+    let mut idx = 1;
+    for _ in 0..MINI_BLOCK_COUNT {
+        let sentinel = row[idx + MINI_BLOCK_SIZE];
+        if sentinel != continuation {
+            f(&row[idx..idx + block_len(sentinel)]);
+            return idx + MINI_BLOCK_SIZE + 1;
+        }
+        f(&row[idx..idx + MINI_BLOCK_SIZE]);
+        idx += MINI_BLOCK_SIZE + 1;
+    }
+
+    loop {
+        let sentinel = row[idx + BLOCK_SIZE];
+        if sentinel != continuation {
+            f(&row[idx..idx + block_len(sentinel)]);
+            return idx + BLOCK_SIZE + 1;
+        }
+        f(&row[idx..idx + BLOCK_SIZE]);
+        idx += BLOCK_SIZE + 1;
+    }
+}
+
+/// Returns the number of bytes of encoded data
+fn decoded_len(row: &[u8], options: SortOptions) -> usize {
+    let mut len = 0;
+    decode_blocks(row, options, |block| len += block.len());
+    len
+}
+
+/// Decodes a binary array from `rows` with the provided `options`
+pub fn decode_binary<I: OffsetSizeTrait>(
+    rows: &mut [&[u8]],
+    options: SortOptions,
+) -> GenericBinaryArray<I> {
+    let len = rows.len();
+    let mut null_count = 0;
+    let nulls = MutableBuffer::collect_bool(len, |x| {
+        let valid = rows[x][0] != null_sentinel(options);
+        null_count += !valid as usize;
+        valid
+    });
+
+    let values_capacity = rows.iter().map(|row| decoded_len(row, options)).sum();
+    let mut offsets = BufferBuilder::<I>::new(len + 1);
+    offsets.append(I::zero());
+    let mut values = MutableBuffer::new(values_capacity);
+
+    for row in rows {
+        let offset = decode_blocks(row, options, |b| values.extend_from_slice(b));
+        *row = &row[offset..];
+        offsets.append(I::from_usize(values.len()).expect("offset overflow"))
+    }
+
+    if options.descending {
+        values.as_slice_mut().iter_mut().for_each(|o| *o = !*o)
+    }
+
+    let d = match I::IS_LARGE {
+        true => DataType::LargeBinary,
+        false => DataType::Binary,
+    };
+
+    let builder = ArrayDataBuilder::new(d)
+        .len(len)
+        .null_count(null_count)
+        .null_bit_buffer(Some(nulls.into()))
+        .add_buffer(offsets.finish())
+        .add_buffer(values.into());
+
+    // SAFETY:
+    // Valid by construction above
+    unsafe { GenericBinaryArray::from(builder.build_unchecked()) }
+}
+
+fn decode_binary_view_inner(
+    rows: &mut [&[u8]],
+    options: SortOptions,
+    validate_utf8: bool,
+) -> BinaryViewArray {
+    let len = rows.len();
+    let inline_str_max_len = MAX_INLINE_VIEW_LEN as usize;
+
+    let mut null_count = 0;
+
+    let nulls = MutableBuffer::collect_bool(len, |x| {
+        let valid = rows[x][0] != null_sentinel(options);
+        null_count += !valid as usize;
+        valid
+    });
+
+    // If we are validating UTF-8, decode all string values (including short strings)
+    // into the values buffer and validate UTF-8 once. If not validating,
+    // we save memory by only copying long strings to the values buffer, as short strings
+    // will be inlined into the view and do not need to be stored redundantly.
+    let values_capacity = if validate_utf8 {
+        // Capacity for all long and short strings
+        rows.iter().map(|row| decoded_len(row, options)).sum()
+    } else {
+        // Capacity for all long strings plus room for one short string
+        rows.iter().fold(0, |acc, row| {
+            let len = decoded_len(row, options);
+            if len > inline_str_max_len {
+                acc + len
+            } else {
+                acc
+            }
+        }) + inline_str_max_len
+    };
+    let mut values = MutableBuffer::new(values_capacity);
+
+    let mut views = BufferBuilder::<u128>::new(len);
+    for row in rows {
+        let start_offset = values.len();
+        let offset = decode_blocks(row, options, |b| values.extend_from_slice(b));
+        // Measure string length via change in values buffer.
+        // Used to check if decoded value should be truncated (short string) when validate_utf8 is false
+        let decoded_len = values.len() - start_offset;
+        if row[0] == null_sentinel(options) {
+            debug_assert_eq!(offset, 1);
+            debug_assert_eq!(start_offset, values.len());
+            views.append(0);
+        } else {
+            // Safety: we just appended the data to the end of the buffer
+            let val = unsafe { values.get_unchecked_mut(start_offset..) };
+
+            if options.descending {
+                val.iter_mut().for_each(|o| *o = !*o);
+            }
+
+            let view = make_view(val, 0, start_offset as u32);
+            views.append(view);
+
+            // truncate inline string in values buffer if validate_utf8 is false
+            if !validate_utf8 && decoded_len <= inline_str_max_len {
+                values.truncate(start_offset);
+            }
+        }
+        *row = &row[offset..];
+    }
+
+    if validate_utf8 {
+        // the values contains all data, no matter if it is short or long
+        // we can validate utf8 in one go.
+        std::str::from_utf8(values.as_slice()).unwrap();
+    }
+
+    let builder = ArrayDataBuilder::new(DataType::BinaryView)
+        .len(len)
+        .null_count(null_count)
+        .null_bit_buffer(Some(nulls.into()))
+        .add_buffer(views.finish())
+        .add_buffer(values.into());
+
+    // SAFETY:
+    // Valid by construction above
+    unsafe { BinaryViewArray::from(builder.build_unchecked()) }
+}
+
+/// Decodes a binary view array from `rows` with the provided `options`
+pub fn decode_binary_view(rows: &mut [&[u8]], options: SortOptions) -> BinaryViewArray {
+    decode_binary_view_inner(rows, options, false)
+}
+
+/// Decodes a string array from `rows` with the provided `options`
+///
+/// # Safety
+///
+/// The row must contain valid UTF-8 data
+pub unsafe fn decode_string<I: OffsetSizeTrait>(
+    rows: &mut [&[u8]],
+    options: SortOptions,
+    validate_utf8: bool,
+) -> GenericStringArray<I> {
+    let decoded = decode_binary::<I>(rows, options);
+
+    if validate_utf8 {
+        return GenericStringArray::from(decoded);
+    }
+
+    let builder = decoded
+        .into_data()
+        .into_builder()
+        .data_type(GenericStringArray::<I>::DATA_TYPE);
+
+    // SAFETY:
+    // Row data must have come from a valid UTF-8 array
+    GenericStringArray::from(unsafe { builder.build_unchecked() })
+}
+
+/// Decodes a string view array from `rows` with the provided `options`
+///
+/// # Safety
+///
+/// The row must contain valid UTF-8 data
+pub unsafe fn decode_string_view(
+    rows: &mut [&[u8]],
+    options: SortOptions,
+    validate_utf8: bool,
+) -> StringViewArray {
+    let view = decode_binary_view_inner(rows, options, validate_utf8);
+    unsafe { view.to_string_view_unchecked() }
+}