Vendor dependencies

This commit is contained in:
2026-08-01 16:11:49 +03:00
parent 7f139a0241
commit 6b5e7f0f8b
29706 changed files with 9575646 additions and 0 deletions
+183
View File
@@ -0,0 +1,183 @@
//! ARM64 CPU feature detection support.
//!
//! Unfortunately ARM instructions to detect CPU features cannot be called from
//! unprivileged userspace code, so this implementation relies on OS-specific
//! APIs for feature detection.
// Evaluate the given `$body` expression any of the supplied target features
// are not enabled. Otherwise returns true.
#[macro_export]
#[doc(hidden)]
macro_rules! __unless_target_features {
($($tf:tt),+ => $body:expr ) => {
{
#[cfg(not(all($(target_feature=$tf,)*)))]
$body
#[cfg(all($(target_feature=$tf,)*))]
true
}
};
}
// Linux runtime detection of target CPU features using `getauxval`.
#[cfg(any(target_os = "linux", target_os = "android"))]
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {{
let hwcaps = $crate::aarch64::getauxval_hwcap();
$($crate::check!(hwcaps, $tf) & )+ true
}};
}
/// Linux helper function for calling `getauxval` to get `AT_HWCAP`.
#[cfg(any(target_os = "linux", target_os = "android"))]
pub fn getauxval_hwcap() -> u64 {
unsafe { libc::getauxval(libc::AT_HWCAP) }
}
// Apple platform's runtime detection of target CPU features using `sysctlbyname`.
#[cfg(target_vendor = "apple")]
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {{
$($crate::check!($tf) & )+ true
}};
}
// Linux `expand_check_macro`
#[cfg(any(target_os = "linux", target_os = "android"))]
macro_rules! __expand_check_macro {
($(($name:tt, $hwcap:ident)),* $(,)?) => {
#[macro_export]
#[doc(hidden)]
macro_rules! check {
$(
($hwcaps:expr, $name) => {
(($hwcaps & $crate::aarch64::hwcaps::$hwcap) != 0)
};
)*
}
};
}
// Linux `expand_check_macro`
#[cfg(any(target_os = "linux", target_os = "android"))]
__expand_check_macro! {
("aes", AES), // Enable AES support.
("dit", DIT), // Enable DIT support.
("sha2", SHA2), // Enable SHA1 and SHA256 support.
("sha3", SHA3), // Enable SHA512 and SHA3 support.
("sm4", SM4), // Enable SM3 and SM4 support.
}
/// Linux hardware capabilities mapped to target features.
///
/// Note that LLVM target features are coarser grained than what Linux supports
/// and imply more capabilities under each feature. This module attempts to
/// provide that mapping accordingly.
///
/// See this issue for more info: <https://github.com/RustCrypto/utils/issues/395>
#[cfg(any(target_os = "linux", target_os = "android"))]
pub mod hwcaps {
use libc::c_ulong;
pub const AES: c_ulong = libc::HWCAP_AES | libc::HWCAP_PMULL;
pub const DIT: c_ulong = libc::HWCAP_DIT;
pub const SHA2: c_ulong = libc::HWCAP_SHA2;
pub const SHA3: c_ulong = libc::HWCAP_SHA3 | libc::HWCAP_SHA512;
pub const SM4: c_ulong = libc::HWCAP_SM3 | libc::HWCAP_SM4;
}
// Apple OS (macOS, iOS, watchOS, and tvOS) `check!` macro.
//
// NOTE: several of these instructions (e.g. `aes`, `sha2`) can be assumed to
// be present on all Apple ARM64 hardware.
//
// Newer CPU instructions now have nodes within sysctl's `hw.optional`
// namespace, however the ones that do not can safely be assumed to be
// present on all Apple ARM64 devices, now and for the foreseeable future.
//
// See discussion on this issue for more information:
// <https://github.com/RustCrypto/utils/issues/378>
#[cfg(target_vendor = "apple")]
#[macro_export]
#[doc(hidden)]
macro_rules! check {
("aes") => {
true
};
("dit") => {
// https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Enable-DIT-for-constant-time-cryptographic-operations
unsafe {
$crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_DIT\0")
}
};
("sha2") => {
true
};
("sha3") => {
unsafe {
// `sha3` target feature implies SHA-512 as well
$crate::aarch64::sysctlbyname(b"hw.optional.armv8_2_sha512\0")
&& $crate::aarch64::sysctlbyname(b"hw.optional.armv8_2_sha3\0")
}
};
("sm4") => {
false
};
}
/// Apple helper function for calling `sysctlbyname`.
///
/// <https://developer.apple.com/documentation/kernel/1387446-sysctlbyname>
///
/// # Panics
/// If `name` is not NUL terminated
#[cfg(target_vendor = "apple")]
#[must_use]
pub unsafe fn sysctlbyname(name: &[u8]) -> bool {
assert_eq!(
name.last().cloned(),
Some(0),
"name is not NUL terminated: {:?}",
name
);
let mut value: u32 = 0;
let mut size = size_of::<u32>();
// SAFETY:
// - `name` is being cast from a valid byte slice we asserted was NUL terminated above.
// - `value` is a properly-aligned, writable integer.
// - `size` is initialized to the size of `value` (4-bytes).
// - The last two arguments,`newp` and `newlen`, are for setting system parameters, which we
// aren't doing here (and requires root privileges). The docs say the following:
// - `newp`: "Specify NULL if you dont want to set the attributes value"
// - `newlen`: "Specify 0 if you dont want to set the attributes value"
let rc = unsafe {
libc::sysctlbyname(
name.as_ptr().cast::<i8>(),
(&raw mut value).cast::<libc::c_void>(),
&raw mut size,
core::ptr::null_mut(),
0,
)
};
assert_eq!(size, 4, "unexpected sysctlbyname(3) result size");
assert_eq!(rc, 0, "sysctlbyname returned error code: {}", rc);
value != 0
}
// On other targets, runtime CPU feature detection is unavailable
#[cfg(not(any(target_vendor = "apple", target_os = "linux", target_os = "android",)))]
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {
false
};
}
+109
View File
@@ -0,0 +1,109 @@
#![no_std]
#![doc = include_str!("../README.md")]
#![doc(
html_logo_url = "https://raw.githubusercontent.com/RustCrypto/media/6ee8e381/logo.svg",
html_favicon_url = "https://raw.githubusercontent.com/RustCrypto/media/6ee8e381/logo.svg"
)]
#[cfg(not(miri))]
#[cfg(target_arch = "aarch64")]
#[doc(hidden)]
pub mod aarch64;
#[cfg(not(miri))]
#[cfg(target_arch = "loongarch64")]
#[doc(hidden)]
pub mod loongarch64;
#[cfg(not(miri))]
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
mod x86;
#[cfg(miri)]
mod miri;
#[cfg(not(any(
target_arch = "aarch64",
target_arch = "loongarch64",
target_arch = "x86",
target_arch = "x86_64"
)))]
compile_error!("This crate works only on `aarch64`, `loongarch64`, `x86`, and `x86-64` targets.");
/// Create module with CPU feature detection code.
#[macro_export]
macro_rules! new {
($mod_name:ident, $($tf:tt),+ $(,)?) => {
mod $mod_name {
use core::sync::atomic::{AtomicU8, Ordering::Relaxed};
const UNINIT: u8 = u8::max_value();
static STORAGE: AtomicU8 = AtomicU8::new(UNINIT);
/// Initialization token
#[derive(Copy, Clone, Debug)]
pub struct InitToken(());
impl InitToken {
/// Initialize token, performing CPU feature detection.
pub fn init() -> Self {
init()
}
/// Initialize token and return a `bool` indicating if the feature is supported.
pub fn init_get() -> (Self, bool) {
init_get()
}
/// Get initialized value.
#[inline(always)]
pub fn get(&self) -> bool {
$crate::__unless_target_features! {
$($tf),+ => {
STORAGE.load(Relaxed) == 1
}
}
}
}
/// Get stored value and initialization token,
/// initializing underlying storage if needed.
#[inline]
pub fn init_get() -> (InitToken, bool) {
let res = $crate::__unless_target_features! {
$($tf),+ => {
#[cold]
fn init_inner() -> bool {
let res = $crate::__detect_target_features!($($tf),+);
STORAGE.store(res as u8, Relaxed);
res
}
// Relaxed ordering is fine, as we only have a single atomic variable.
let val = STORAGE.load(Relaxed);
if val == UNINIT {
init_inner()
} else {
val == 1
}
}
};
(InitToken(()), res)
}
/// Initialize underlying storage if needed and get initialization token.
#[inline]
pub fn init() -> InitToken {
init_get().0
}
/// Initialize underlying storage if needed and get stored value.
#[inline]
pub fn get() -> bool {
init_get().1
}
}
};
}
+128
View File
@@ -0,0 +1,128 @@
//! LoongArch64 CPU feature detection support.
//!
//! This implementation relies on OS-specific APIs for feature detection.
// Evaluate the given `$body` expression any of the supplied target features
// are not enabled. Otherwise returns true.
#[macro_export]
#[doc(hidden)]
macro_rules! __unless_target_features {
($($tf:tt),+ => $body:expr ) => {
{
#[cfg(not(all($(target_feature=$tf,)*)))]
$body
#[cfg(all($(target_feature=$tf,)*))]
true
}
};
}
// Linux runtime detection of target CPU features using `getauxval`.
#[cfg(target_os = "linux")]
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {{
let cpucfg1: usize;
let cpucfg2: usize;
let cpucfg3: usize;
unsafe {
std::arch::asm!(
"cpucfg {}, {}",
"cpucfg {}, {}",
"cpucfg {}, {}",
out(reg) cpucfg1, in(reg) 1,
out(reg) cpucfg2, in(reg) 2,
out(reg) cpucfg3, in(reg) 3,
options(pure, nomem, preserves_flags, nostack)
);
}
let hwcaps = $crate::loongarch64::getauxval_hwcap();
$($crate::check!(cpucfg1, cpucfg2, cpucfg3, hwcaps, $tf) & )+ true
}};
}
/// Linux helper function for calling `getauxval` to get `AT_HWCAP`.
#[cfg(target_os = "linux")]
pub fn getauxval_hwcap() -> u64 {
unsafe { libc::getauxval(libc::AT_HWCAP) }
}
#[macro_export]
#[doc(hidden)]
macro_rules! check {
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "32s") => {
(($cpucfg1 & 1) != 0 || ($cpucfg1 & 2) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "f") => {
(($cpucfg2 & 2) != 0 && ($hwcaps & $crate::loongarch64::hwcaps::FPU) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "d") => {
(($cpucfg2 & 4) != 0 && ($hwcaps & $crate::loongarch64::hwcaps::FPU) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "frecipe") => {
(($cpucfg2 & (1 << 25)) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "div32") => {
(($cpucfg2 & (1 << 26)) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "lsx") => {
(($hwcaps & $crate::loongarch64::hwcaps::LSX) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "lasx") => {
(($hwcaps & $crate::loongarch64::hwcaps::LASX) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "lam-bh") => {
(($cpucfg2 & (1 << 27)) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "lamcas") => {
(($cpucfg2 & (1 << 28)) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "ld-seq-sa") => {
(($cpucfg3 & (1 << 23)) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "scq") => {
(($cpucfg2 & (1 << 30)) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "lbt") => {
(($hwcaps & $crate::loongarch64::hwcaps::LBT_X86) != 0
&& ($hwcaps & $crate::loongarch64::hwcaps::LBT_ARM) != 0
&& ($hwcaps & $crate::loongarch64::hwcaps::LBT_MIPS) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "lvz") => {
(($hwcaps & $crate::loongarch64::hwcaps::LVZ) != 0)
};
($cpucfg1:expr, $cpucfg2:expr, $cpucfg3:expr, $hwcaps:expr, "ual") => {
(($hwcaps & $crate::loongarch64::hwcaps::UAL) != 0)
};
}
/// Linux hardware capabilities mapped to target features.
///
/// Note that LLVM target features are coarser grained than what Linux supports
/// and imply more capabilities under each feature. This module attempts to
/// provide that mapping accordingly.
#[cfg(target_os = "linux")]
pub mod hwcaps {
use libc::c_ulong;
pub const UAL: c_ulong = libc::HWCAP_LOONGARCH_UAL;
pub const FPU: c_ulong = libc::HWCAP_LOONGARCH_FPU;
pub const LSX: c_ulong = libc::HWCAP_LOONGARCH_LSX;
pub const LASX: c_ulong = libc::HWCAP_LOONGARCH_LASX;
pub const LVZ: c_ulong = libc::HWCAP_LOONGARCH_LVZ;
pub const LBT_X86: c_ulong = libc::HWCAP_LOONGARCH_LBT_X86;
pub const LBT_ARM: c_ulong = libc::HWCAP_LOONGARCH_LBT_ARM;
pub const LBT_MIPS: c_ulong = libc::HWCAP_LOONGARCH_LBT_MIPS;
}
// On other targets, runtime CPU feature detection is unavailable
#[cfg(not(target_os = "linux"))]
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {
false
};
}
+20
View File
@@ -0,0 +1,20 @@
//! Minimal miri support.
//!
//! Miri is an interpreter, and though it tries to emulate the target CPU
//! it does not support any target features.
#[macro_export]
#[doc(hidden)]
macro_rules! __unless_target_features {
($($tf:tt),+ => $body:expr ) => {
false
};
}
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {
false
};
}
+149
View File
@@ -0,0 +1,149 @@
//! x86/x86-64 CPU feature detection support.
//!
//! Portable, `no_std`-friendly implementation that relies on the x86 `CPUID`
//! instruction for feature detection.
/// Evaluate the given `$body` expression any of the supplied target features
/// are not enabled. Otherwise returns true.
///
/// The `$body` expression is not evaluated on SGX targets, and returns false
/// on these targets unless *all* supplied target features are enabled.
#[macro_export]
#[doc(hidden)]
macro_rules! __unless_target_features {
($($tf:tt),+ => $body:expr ) => {{
#[cfg(not(all($(target_feature=$tf,)*)))]
{
#[cfg(not(any(target_env = "sgx", target_os = "none", target_os = "uefi")))]
$body
// CPUID is not available on SGX. Freestanding and UEFI targets
// do not support SIMD features with default compilation flags.
#[cfg(any(target_env = "sgx", target_os = "none", target_os = "uefi"))]
false
}
#[cfg(all($(target_feature=$tf,)*))]
true
}};
}
/// Use CPUID to detect the presence of all supplied target features.
#[macro_export]
#[doc(hidden)]
macro_rules! __detect_target_features {
($($tf:tt),+) => {{
#[cfg(target_arch = "x86")]
use core::arch::x86::{__cpuid, __cpuid_count, CpuidResult};
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult};
unsafe fn cpuid(leaf: u32) -> CpuidResult {
__cpuid(leaf)
}
unsafe fn cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult {
__cpuid_count(leaf, sub_leaf)
}
let cr = unsafe {
[cpuid(1), cpuid_count(7, 0), cpuid_count(7, 1)]
};
$($crate::check!(cr, $tf) & )+ true
}};
}
/// Check that OS supports required SIMD registers
#[macro_export]
#[doc(hidden)]
macro_rules! __xgetbv {
($cr:expr, $mask:expr) => {{
#[cfg(target_arch = "x86")]
use core::arch::x86 as arch;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64 as arch;
// Check bits 26 and 27
let xmask = 0b11 << 26;
let xsave = $cr[0].ecx & xmask == xmask;
if xsave {
let xcr0 = unsafe { arch::_xgetbv(arch::_XCR_XFEATURE_ENABLED_MASK) };
(xcr0 & $mask) == $mask
} else {
false
}
}};
}
macro_rules! __expand_check_macro {
($(($name:tt, $reg_cap:tt $(, $i:expr, $reg:ident, $offset:expr)*)),* $(,)?) => {
#[macro_export]
#[doc(hidden)]
macro_rules! check {
$(
($cr:expr, $name) => {{
// Register bits are listed here:
// https://wiki.osdev.org/CPU_Registers_x86#Extended_Control_Registers
let reg_cap = match $reg_cap {
// Bit 1
"xmm" => $crate::__xgetbv!($cr, 0b10),
// Bits 1 and 2
"ymm" => $crate::__xgetbv!($cr, 0b110),
// Bits 1, 2, 5, 6, and 7
"zmm" => $crate::__xgetbv!($cr, 0b1110_0110),
_ => true,
};
reg_cap
$(
& ($cr[$i].$reg & (1 << $offset) != 0)
)*
}};
)*
}
};
}
__expand_check_macro! {
("sse3", "", 0, ecx, 0),
("pclmulqdq", "", 0, ecx, 1),
("ssse3", "", 0, ecx, 9),
("fma", "ymm", 0, ecx, 12, 0, ecx, 28),
("sse4.1", "", 0, ecx, 19),
("sse4.2", "", 0, ecx, 20),
("popcnt", "", 0, ecx, 23),
("aes", "", 0, ecx, 25),
("avx", "xmm", 0, ecx, 28),
("rdrand", "", 0, ecx, 30),
("mmx", "", 0, edx, 23),
("sse", "", 0, edx, 25),
("sse2", "", 0, edx, 26),
("sgx", "", 1, ebx, 2),
("bmi1", "", 1, ebx, 3),
("bmi2", "", 1, ebx, 8),
("avx2", "ymm", 1, ebx, 5, 0, ecx, 28),
("avx512f", "zmm", 1, ebx, 16),
("avx512dq", "zmm", 1, ebx, 17),
("rdseed", "", 1, ebx, 18),
("adx", "", 1, ebx, 19),
("avx512ifma", "zmm", 1, ebx, 21),
("avx512pf", "zmm", 1, ebx, 26),
("avx512er", "zmm", 1, ebx, 27),
("avx512cd", "zmm", 1, ebx, 28),
("sha", "", 1, ebx, 29),
("avx512bw", "zmm", 1, ebx, 30),
("avx512vl", "zmm", 1, ebx, 31),
("avx512vbmi", "zmm", 1, ecx, 1),
("avx512vbmi2", "zmm", 1, ecx, 6),
("gfni", "zmm", 1, ecx, 8),
("vaes", "zmm", 1, ecx, 9),
("vpclmulqdq", "zmm", 1, ecx, 10),
("avx512bitalg", "zmm", 1, ecx, 12),
("avx512vpopcntdq", "zmm", 1, ecx, 14),
("sha512", "ymm", 2, eax, 0),
("sm3", "xmm", 2, eax, 1),
("sm4", "ymm", 2, eax, 2),
}