Vendor dependencies

This commit is contained in:
2026-08-01 16:11:49 +03:00
parent 7f139a0241
commit 6b5e7f0f8b
29706 changed files with 9575646 additions and 0 deletions
@@ -0,0 +1,111 @@
use crate::DecodeV2;
use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::{fmt, io};
type BrotliState = brotli::BrotliState<StandardAlloc, StandardAlloc, StandardAlloc>;
pub struct BrotliDecoder {
// `BrotliState` is very large (over 2kb) which is why we're boxing it.
state: Box<BrotliState>,
}
impl Default for BrotliDecoder {
fn default() -> Self {
Self {
state: Box::new(Self::new_brotli_state()),
}
}
}
impl BrotliDecoder {
fn new_brotli_state() -> BrotliState {
BrotliState::new(
StandardAlloc::default(),
StandardAlloc::default(),
StandardAlloc::default(),
)
}
pub fn new() -> Self {
Self::default()
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<BrotliResult> {
let in_buf = input.unwritten();
let out_buf = output.initialize_unwritten();
let mut input_len = 0;
let mut output_len = 0;
let status = match BrotliDecompressStream(
&mut in_buf.len(),
&mut input_len,
in_buf,
&mut out_buf.len(),
&mut output_len,
out_buf,
&mut 0,
&mut self.state,
) {
BrotliResult::ResultFailure => return Err(io::Error::other("brotli error")),
status => status,
};
input.advance(input_len);
output.advance(output_len);
Ok(status)
}
}
impl DecodeV2 for BrotliDecoder {
fn reinit(&mut self) -> io::Result<()> {
*self.state = Self::new_brotli_state();
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<bool> {
match self.decode(input, output)? {
BrotliResult::ResultSuccess => Ok(true),
BrotliResult::NeedsMoreOutput | BrotliResult::NeedsMoreInput => Ok(false),
BrotliResult::ResultFailure => unreachable!(),
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
match self.decode(&mut PartialBuffer::new(&[][..]), output)? {
BrotliResult::ResultSuccess | BrotliResult::NeedsMoreInput => Ok(true),
BrotliResult::NeedsMoreOutput => Ok(false),
BrotliResult::ResultFailure => unreachable!(),
}
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
match self.decode(&mut PartialBuffer::new(&[][..]), output)? {
BrotliResult::ResultSuccess => Ok(true),
BrotliResult::NeedsMoreOutput => Ok(false),
BrotliResult::NeedsMoreInput => Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"reached unexpected EOF",
)),
BrotliResult::ResultFailure => unreachable!(),
}
}
}
impl fmt::Debug for BrotliDecoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BrotliDecoder")
.field("decompress", &"<no debug>")
.finish()
}
}
@@ -0,0 +1,95 @@
use crate::{brotli::params::EncoderParams, EncodeV2};
use brotli::enc::{
backward_references::BrotliEncoderParams,
encode::{BrotliEncoderOperation, BrotliEncoderStateStruct},
StandardAlloc,
};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::{fmt, io};
pub struct BrotliEncoder {
state: BrotliEncoderStateStruct<StandardAlloc>,
}
impl BrotliEncoder {
pub fn new(params: EncoderParams) -> Self {
let params = BrotliEncoderParams::from(params);
let mut state = BrotliEncoderStateStruct::new(StandardAlloc::default());
state.params = params;
Self { state }
}
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
op: BrotliEncoderOperation,
) -> io::Result<()> {
let in_buf = input.unwritten();
let out_buf = output.initialize_unwritten();
let mut input_len = 0;
let mut output_len = 0;
if !self.state.compress_stream(
op,
&mut in_buf.len(),
in_buf,
&mut input_len,
&mut out_buf.len(),
out_buf,
&mut output_len,
&mut None,
&mut |_, _, _, _| (),
) {
return Err(io::Error::other("brotli error"));
}
input.advance(input_len);
output.advance(output_len);
Ok(())
}
}
impl EncodeV2 for BrotliEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<()> {
self.encode(
input,
output,
BrotliEncoderOperation::BROTLI_OPERATION_PROCESS,
)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
self.encode(
&mut PartialBuffer::new(&[][..]),
output,
BrotliEncoderOperation::BROTLI_OPERATION_FLUSH,
)?;
Ok(!self.state.has_more_output())
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
self.encode(
&mut PartialBuffer::new(&[][..]),
output,
BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
)?;
Ok(self.state.is_finished())
}
}
impl fmt::Debug for BrotliEncoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BrotliEncoder")
.field("compress", &"<no debug>")
.finish()
}
}
@@ -0,0 +1,5 @@
mod decoder;
mod encoder;
pub mod params;
pub use self::{decoder::BrotliDecoder, encoder::BrotliEncoder};
@@ -0,0 +1,92 @@
//! This module contains Brotli-specific types for async-compression.
use brotli::enc::backward_references::{BrotliEncoderMode, BrotliEncoderParams};
use compression_core::Level;
/// Brotli compression parameters builder. This is a stable wrapper around Brotli's own encoder
/// params type, to abstract over different versions of the Brotli library.
///
/// See the [Brotli documentation](https://www.brotli.org/encode.html#a9a8) for more information on
/// these parameters.
///
/// # Examples
///
/// ```
/// use compression_codecs::brotli;
///
/// let params = brotli::params::EncoderParams::default()
/// .window_size(12)
/// .text_mode();
/// ```
#[derive(Debug, Clone, Default)]
pub struct EncoderParams {
inner: BrotliEncoderParams,
}
impl From<EncoderParams> for BrotliEncoderParams {
fn from(value: EncoderParams) -> Self {
value.inner
}
}
impl EncoderParams {
pub fn quality(mut self, level: Level) -> Self {
let quality = match level {
Level::Fastest => Some(0),
Level::Best => Some(11),
Level::Precise(quality) => Some(quality.clamp(0, 11)),
_ => None,
};
match quality {
Some(quality) => self.inner.quality = quality,
None => {
let default_params = BrotliEncoderParams::default();
self.inner.quality = default_params.quality;
}
}
self
}
/// Sets window size in bytes (as a power of two).
///
/// Used as Brotli's `lgwin` parameter.
///
/// `window_size` is clamped to `0 <= window_size <= 24`.
pub fn window_size(mut self, window_size: i32) -> Self {
self.inner.lgwin = window_size.clamp(0, 24);
self
}
/// Sets input block size in bytes (as a power of two).
///
/// Used as Brotli's `lgblock` parameter.
///
/// `block_size` is clamped to `16 <= block_size <= 24`.
pub fn block_size(mut self, block_size: i32) -> Self {
self.inner.lgblock = block_size.clamp(16, 24);
self
}
/// Sets hint for size of data to be compressed.
pub fn size_hint(mut self, size_hint: usize) -> Self {
self.inner.size_hint = size_hint;
self
}
/// Sets encoder to text mode.
///
/// If input data is known to be UTF-8 text, this allows the compressor to make assumptions and
/// optimizations.
///
/// Used as Brotli's `mode` parameter.
pub fn text_mode(mut self) -> Self {
self.inner.mode = BrotliEncoderMode::BROTLI_MODE_TEXT;
self
}
pub fn mode(mut self, mode: BrotliEncoderMode) -> Self {
self.inner.mode = mode;
self
}
}
@@ -0,0 +1,123 @@
use crate::DecodeV2;
use bzip2::{Decompress, Status};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::{fmt, io};
pub struct BzDecoder {
decompress: Decompress,
stream_ended: bool,
}
impl fmt::Debug for BzDecoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"BzDecoder {{total_in: {}, total_out: {}}}",
self.decompress.total_in(),
self.decompress.total_out()
)
}
}
impl Default for BzDecoder {
fn default() -> Self {
Self {
decompress: Decompress::new(false),
stream_ended: false,
}
}
}
impl BzDecoder {
pub fn new() -> Self {
Self::default()
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<Status> {
let prior_in = self.decompress.total_in();
let prior_out = self.decompress.total_out();
let status = self
.decompress
// Safety: We **trust** bzip2 to only write initialized data to it
.decompress_uninit(input.unwritten(), unsafe { output.unwritten_mut() })
.map_err(io::Error::other)?;
input.advance((self.decompress.total_in() - prior_in) as usize);
// Safety: We **trust** bzip2 to write bytes properly
unsafe {
output.assume_init_and_advance((self.decompress.total_out() - prior_out) as usize)
};
// Track when stream has properly ended
if status == Status::StreamEnd {
self.stream_ended = true;
}
Ok(status)
}
}
impl DecodeV2 for BzDecoder {
fn reinit(&mut self) -> io::Result<()> {
self.decompress = Decompress::new(false);
self.stream_ended = false;
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<bool> {
match self.decode(input, output)? {
// Decompression went fine, nothing much to report.
Status::Ok => Ok(false),
// The Flush action on a compression went ok.
Status::FlushOk => unreachable!(),
// THe Run action on compression went ok.
Status::RunOk => unreachable!(),
// The Finish action on compression went ok.
Status::FinishOk => unreachable!(),
// The stream's end has been met, meaning that no more data can be input.
Status::StreamEnd => Ok(true),
// There was insufficient memory in the input or output buffer to complete
// the request, but otherwise everything went normally.
Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()),
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
self.decode(&mut PartialBuffer::new(&[][..]), output)?;
loop {
let old_len = output.written_len();
self.decode(&mut PartialBuffer::new(&[][..]), output)?;
if output.written_len() == old_len {
break;
}
}
Ok(!output.has_no_spare_space())
}
fn finish(&mut self, _output: &mut WriteBuffer<'_>) -> io::Result<bool> {
if self.stream_ended {
Ok(true)
} else {
Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"bzip2 stream did not finish",
))
}
}
}
@@ -0,0 +1,145 @@
use crate::{bzip2::params::Bzip2EncoderParams, EncodeV2};
use bzip2::{Action, Compress, Compression, Status};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::{fmt, io};
pub struct BzEncoder {
compress: Compress,
}
impl fmt::Debug for BzEncoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"BzEncoder {{total_in: {}, total_out: {}}}",
self.compress.total_in(),
self.compress.total_out()
)
}
}
impl BzEncoder {
/// Creates a new stream prepared for compression.
///
/// The `work_factor` parameter controls how the compression phase behaves
/// when presented with worst case, highly repetitive, input data. If
/// compression runs into difficulties caused by repetitive data, the
/// library switches from the standard sorting algorithm to a fallback
/// algorithm. The fallback is slower than the standard algorithm by perhaps
/// a factor of three, but always behaves reasonably, no matter how bad the
/// input.
///
/// Lower values of `work_factor` reduce the amount of effort the standard
/// algorithm will expend before resorting to the fallback. You should set
/// this parameter carefully; too low, and many inputs will be handled by
/// the fallback algorithm and so compress rather slowly, too high, and your
/// average-to-worst case compression times can become very large. The
/// default value of 30 gives reasonable behaviour over a wide range of
/// circumstances.
///
/// Allowable values range from 0 to 250 inclusive. 0 is a special case,
/// equivalent to using the default value of 30.
pub fn new(params: Bzip2EncoderParams, work_factor: u32) -> Self {
let params = Compression::from(params);
Self {
compress: Compress::new(params, work_factor),
}
}
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
action: Action,
) -> io::Result<Status> {
let prior_in = self.compress.total_in();
let prior_out = self.compress.total_out();
let status = self
.compress
// Safety: We **trust** bzip2 to only write initialized bytes into it
.compress_uninit(input.unwritten(), unsafe { output.unwritten_mut() }, action)
.map_err(io::Error::other)?;
input.advance((self.compress.total_in() - prior_in) as usize);
// Safety: We **trust** bzip2 to properly write bytes into it
unsafe { output.assume_init_and_advance((self.compress.total_out() - prior_out) as usize) };
Ok(status)
}
}
impl EncodeV2 for BzEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<()> {
match self.encode(input, output, Action::Run)? {
// Decompression went fine, nothing much to report.
Status::Ok => Ok(()),
// The Flush action on a compression went ok.
Status::FlushOk => unreachable!(),
// The Run action on compression went ok.
Status::RunOk => Ok(()),
// The Finish action on compression went ok.
Status::FinishOk => unreachable!(),
// The stream's end has been met, meaning that no more data can be input.
Status::StreamEnd => unreachable!(),
// There was insufficient memory in the input or output buffer to complete
// the request, but otherwise everything went normally.
Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()),
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
match self.encode(&mut PartialBuffer::new(&[][..]), output, Action::Flush)? {
// Decompression went fine, nothing much to report.
Status::Ok => unreachable!(),
// The Flush action on a compression went ok.
Status::FlushOk => Ok(false),
// The Run action on compression went ok.
Status::RunOk => Ok(true),
// The Finish action on compression went ok.
Status::FinishOk => unreachable!(),
// The stream's end has been met, meaning that no more data can be input.
Status::StreamEnd => unreachable!(),
// There was insufficient memory in the input or output buffer to complete
// the request, but otherwise everything went normally.
Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()),
}
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
match self.encode(&mut PartialBuffer::new(&[][..]), output, Action::Finish)? {
// Decompression went fine, nothing much to report.
Status::Ok => Ok(false),
// The Flush action on a compression went ok.
Status::FlushOk => unreachable!(),
// The Run action on compression went ok.
Status::RunOk => unreachable!(),
// The Finish action on compression went ok.
Status::FinishOk => Ok(false),
// The stream's end has been met, meaning that no more data can be input.
Status::StreamEnd => Ok(true),
// There was insufficient memory in the input or output buffer to complete
// the request, but otherwise everything went normally.
Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()),
}
}
}
@@ -0,0 +1,5 @@
mod decoder;
mod encoder;
pub mod params;
pub use self::{decoder::BzDecoder, encoder::BzEncoder};
@@ -0,0 +1,34 @@
use std::convert::TryInto;
use bzip2::Compression;
use compression_core::Level;
#[derive(Debug)]
pub struct Bzip2EncoderParams {
inner: Compression,
}
impl From<Bzip2EncoderParams> for Compression {
fn from(value: Bzip2EncoderParams) -> Self {
value.inner
}
}
impl From<Level> for Bzip2EncoderParams {
fn from(value: Level) -> Self {
let fastest = bzip2::Compression::fast();
let best = bzip2::Compression::best();
let inner = match value {
Level::Fastest => fastest,
Level::Best => best,
Level::Precise(quality) => bzip2::Compression::new(
quality
.try_into()
.unwrap_or(0)
.clamp(fastest.level(), best.level()),
),
_ => bzip2::Compression::default(),
};
Self { inner }
}
}
@@ -0,0 +1,45 @@
use crate::{DecodeV2, FlateDecoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::io::Result;
#[derive(Debug)]
pub struct DeflateDecoder {
inner: FlateDecoder,
}
impl Default for DeflateDecoder {
fn default() -> Self {
Self {
inner: FlateDecoder::new(false),
}
}
}
impl DeflateDecoder {
pub fn new() -> Self {
Self::default()
}
}
impl DecodeV2 for DeflateDecoder {
fn reinit(&mut self) -> Result<()> {
self.inner.reinit()?;
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
self.inner.decode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
@@ -0,0 +1,38 @@
use crate::{flate::params::FlateEncoderParams, EncodeV2, FlateEncoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::io::Result;
#[derive(Debug)]
pub struct DeflateEncoder {
inner: FlateEncoder,
}
impl DeflateEncoder {
pub fn new(level: FlateEncoderParams) -> Self {
Self {
inner: FlateEncoder::new(level, false),
}
}
pub fn get_ref(&self) -> &FlateEncoder {
&self.inner
}
}
impl EncodeV2 for DeflateEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()> {
self.inner.encode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
@@ -0,0 +1,4 @@
mod decoder;
mod encoder;
pub use self::{decoder::DeflateDecoder, encoder::DeflateEncoder};
@@ -0,0 +1,77 @@
use crate::DecodeV2;
use compression_core::util::{PartialBuffer, WriteBuffer};
use deflate64::InflaterManaged;
use std::io::{Error, ErrorKind, Result};
#[derive(Debug)]
pub struct Deflate64Decoder {
inflater: Box<InflaterManaged>,
}
impl Default for Deflate64Decoder {
fn default() -> Self {
Self {
inflater: Box::new(InflaterManaged::new()),
}
}
}
impl Deflate64Decoder {
pub fn new() -> Self {
Self::default()
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
let result = self
.inflater
// Safety: We **trust** deflate64 to not write uninitialized bytes
.inflate_uninit(input.unwritten(), unsafe { output.unwritten_mut() });
if result.data_error {
Err(Error::new(ErrorKind::InvalidData, "invalid data"))
} else {
input.advance(result.bytes_consumed);
// Safety: We **trust** deflate64 to properly write bytes into buffer
unsafe { output.assume_init_and_advance(result.bytes_written) };
Ok(self.inflater.finished() && self.inflater.available_output() == 0)
}
}
}
impl DecodeV2 for Deflate64Decoder {
fn reinit(&mut self) -> Result<()> {
*self.inflater = InflaterManaged::new();
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
self.decode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.decode(&mut PartialBuffer::new(&[]), output)?;
loop {
let old_len = output.written_len();
self.decode(&mut PartialBuffer::new(&[]), output)?;
if output.written_len() == old_len {
break;
}
}
Ok(!output.has_no_spare_space())
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.decode(&mut PartialBuffer::new(&[]), output)
}
}
@@ -0,0 +1,3 @@
mod decoder;
pub use self::decoder::Deflate64Decoder;
@@ -0,0 +1,95 @@
use crate::DecodeV2;
use compression_core::util::{PartialBuffer, WriteBuffer};
use flate2::{Decompress, FlushDecompress, Status};
use std::io;
#[derive(Debug)]
pub struct FlateDecoder {
zlib_header: bool,
decompress: Decompress,
}
impl FlateDecoder {
pub(crate) fn new(zlib_header: bool) -> Self {
Self {
zlib_header,
decompress: Decompress::new(zlib_header),
}
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
flush: FlushDecompress,
) -> io::Result<Status> {
let prior_in = self.decompress.total_in();
let prior_out = self.decompress.total_out();
let status = self
.decompress
// Safety: We **trust** flate2 to not write uninitialized bytes into buffer
.decompress_uninit(input.unwritten(), unsafe { output.unwritten_mut() }, flush)?;
input.advance((self.decompress.total_in() - prior_in) as usize);
// Safety: We **trust** flate2 to write bytes into buffer properly
unsafe {
output.assume_init_and_advance((self.decompress.total_out() - prior_out) as usize)
};
Ok(status)
}
}
impl DecodeV2 for FlateDecoder {
fn reinit(&mut self) -> io::Result<()> {
self.decompress.reset(self.zlib_header);
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<bool> {
match self.decode(input, output, FlushDecompress::None)? {
Status::Ok => Ok(false),
Status::StreamEnd => Ok(true),
Status::BufError => Err(io::Error::other("unexpected BufError")),
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
self.decode(
&mut PartialBuffer::new(&[][..]),
output,
FlushDecompress::Sync,
)?;
loop {
let old_len = output.written_len();
self.decode(
&mut PartialBuffer::new(&[][..]),
output,
FlushDecompress::None,
)?;
if output.written_len() == old_len {
break;
}
}
Ok(!output.has_no_spare_space())
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
match self.decode(
&mut PartialBuffer::new(&[][..]),
output,
FlushDecompress::Finish,
)? {
Status::Ok => Ok(false),
Status::StreamEnd => Ok(true),
Status::BufError => Err(io::Error::other("unexpected BufError")),
}
}
}
@@ -0,0 +1,103 @@
use crate::{flate::params::FlateEncoderParams, EncodeV2};
use compression_core::util::{PartialBuffer, WriteBuffer};
use flate2::{Compress, FlushCompress, Status};
use std::io;
#[derive(Debug)]
pub struct FlateEncoder {
compress: Compress,
flushed: bool,
}
impl FlateEncoder {
pub fn new(level: FlateEncoderParams, zlib_header: bool) -> Self {
let level = flate2::Compression::from(level);
Self {
compress: Compress::new(level, zlib_header),
flushed: true,
}
}
pub fn get_ref(&self) -> &Compress {
&self.compress
}
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
flush: FlushCompress,
) -> io::Result<Status> {
let prior_in = self.compress.total_in();
let prior_out = self.compress.total_out();
let status = self
.compress
// Safety: We **trust** flate2 to not write uninitialized bytes into buffer
.compress_uninit(input.unwritten(), unsafe { output.unwritten_mut() }, flush)?;
input.advance((self.compress.total_in() - prior_in) as usize);
// Safety: We **trust** flate2 to write bytes properly into buffer
unsafe { output.assume_init_and_advance((self.compress.total_out() - prior_out) as usize) };
Ok(status)
}
}
impl EncodeV2 for FlateEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<()> {
self.flushed = false;
match self.encode(input, output, FlushCompress::None)? {
Status::Ok => Ok(()),
Status::StreamEnd => unreachable!(),
Status::BufError => Err(io::Error::other("unexpected BufError")),
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
// We need to keep track of whether we've already flushed otherwise we'll just keep writing
// out sync blocks continuously and probably never complete flushing.
if self.flushed {
return Ok(true);
}
self.encode(
&mut PartialBuffer::new(&[][..]),
output,
FlushCompress::Sync,
)?;
loop {
let old_len = output.written_len();
self.encode(
&mut PartialBuffer::new(&[][..]),
output,
FlushCompress::None,
)?;
if output.written_len() == old_len {
break;
}
}
let internal_flushed = !output.has_no_spare_space();
self.flushed = internal_flushed;
Ok(internal_flushed)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
self.flushed = false;
match self.encode(
&mut PartialBuffer::new(&[][..]),
output,
FlushCompress::Finish,
)? {
Status::Ok => Ok(false),
Status::StreamEnd => Ok(true),
Status::BufError => Err(io::Error::other("unexpected BufError")),
}
}
}
@@ -0,0 +1,5 @@
mod decoder;
mod encoder;
pub mod params;
pub use self::{decoder::FlateDecoder, encoder::FlateEncoder};
@@ -0,0 +1,40 @@
use std::convert::TryInto;
use compression_core::Level;
#[derive(Debug, Clone)]
pub struct FlateEncoderParams {
inner: flate2::Compression,
}
impl From<flate2::Compression> for FlateEncoderParams {
fn from(inner: flate2::Compression) -> Self {
Self { inner }
}
}
impl From<FlateEncoderParams> for flate2::Compression {
fn from(value: FlateEncoderParams) -> Self {
value.inner
}
}
impl From<Level> for FlateEncoderParams {
fn from(value: Level) -> Self {
let fastest = flate2::Compression::fast();
let best = flate2::Compression::best();
let none = flate2::Compression::none();
let inner = match value {
Level::Fastest => fastest,
Level::Best => best,
Level::Precise(quality) => flate2::Compression::new(
quality
.try_into()
.unwrap_or(0)
.clamp(none.level(), best.level()),
),
_ => flate2::Compression::default(),
};
Self { inner }
}
}
@@ -0,0 +1,160 @@
use super::header;
use crate::{DecodeV2, FlateDecoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use flate2::Crc;
use std::io::{Error, ErrorKind, Result};
#[derive(Debug)]
enum State {
Header(header::Parser),
Decoding,
Footer(PartialBuffer<[u8; 8]>),
Done,
}
#[derive(Debug)]
pub struct GzipDecoder {
inner: FlateDecoder,
crc: Crc,
state: State,
}
fn check_footer(crc: &Crc, input: &[u8; 8]) -> Result<()> {
let crc_sum = crc.sum().to_le_bytes();
let bytes_read = crc.amount().to_le_bytes();
if crc_sum != input[0..4] {
return Err(Error::new(
ErrorKind::InvalidData,
"CRC computed does not match",
));
}
if bytes_read != input[4..8] {
return Err(Error::new(
ErrorKind::InvalidData,
"amount of bytes read does not match",
));
}
Ok(())
}
impl Default for GzipDecoder {
fn default() -> Self {
Self {
inner: FlateDecoder::new(false),
crc: Crc::new(),
state: State::Header(header::Parser::default()),
}
}
}
impl GzipDecoder {
pub fn new() -> Self {
Self::default()
}
fn process(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
inner: impl Fn(&mut Self, &mut PartialBuffer<&[u8]>, &mut WriteBuffer<'_>) -> Result<bool>,
) -> Result<bool> {
loop {
match &mut self.state {
State::Header(parser) => {
if parser.input(&mut self.crc, input)?.is_some() {
self.crc.reset();
self.state = State::Decoding;
}
}
State::Decoding => {
let prior = output.written_len();
let res = inner(self, input, output);
if output.written_len() > prior {
// update CRC even if there was an error
self.crc.update(&output.written()[prior..]);
}
let done = res?;
if done {
self.state = State::Footer([0; 8].into());
}
}
State::Footer(footer) => {
footer.copy_unwritten_from(input);
if footer.unwritten().is_empty() {
check_footer(&self.crc, footer.get_mut())?;
self.state = State::Done;
}
}
State::Done => {}
};
if let State::Done = self.state {
return Ok(true);
}
if input.unwritten().is_empty() || output.has_no_spare_space() {
return Ok(false);
}
}
}
}
impl DecodeV2 for GzipDecoder {
fn reinit(&mut self) -> Result<()> {
self.inner.reinit()?;
self.crc.reset();
self.state = State::Header(header::Parser::default());
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
self.process(input, output, |this, input, output| {
this.inner.decode(input, output)
})
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
loop {
match self.state {
State::Header(_) | State::Footer(_) | State::Done => return Ok(true),
State::Decoding => {
let prior = output.written_len();
let done = self.inner.flush(output)?;
self.crc.update(&output.written()[prior..]);
if done {
return Ok(true);
}
}
};
if output.has_no_spare_space() {
return Ok(false);
}
}
}
fn finish(&mut self, _output: &mut WriteBuffer<'_>) -> Result<bool> {
// Because of the footer we have to have already flushed all the data out before we get here
if let State::Done = self.state {
Ok(true)
} else {
Err(Error::from(ErrorKind::UnexpectedEof))
}
}
}
@@ -0,0 +1,160 @@
use crate::{flate::params::FlateEncoderParams, EncodeV2, FlateEncoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use flate2::{Compression, Crc};
use std::io;
#[derive(Debug)]
enum State {
Header(PartialBuffer<[u8; 10]>),
Encoding,
Footer(PartialBuffer<[u8; 8]>),
Done,
}
#[derive(Debug)]
pub struct GzipEncoder {
inner: FlateEncoder,
crc: Crc,
state: State,
}
fn header(level: Compression) -> [u8; 10] {
let level_byte = if level.level() >= Compression::best().level() {
0x02
} else if level.level() <= Compression::fast().level() {
0x04
} else {
0x00
};
[0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, level_byte, 0xff]
}
impl GzipEncoder {
pub fn new(level: FlateEncoderParams) -> Self {
Self {
inner: FlateEncoder::new(level.clone(), false),
crc: Crc::new(),
state: State::Header(header(Compression::from(level)).into()),
}
}
fn footer(&mut self) -> [u8; 8] {
let mut output = [0; 8];
output[..4].copy_from_slice(&self.crc.sum().to_le_bytes());
output[4..].copy_from_slice(&self.crc.amount().to_le_bytes());
output
}
}
impl EncodeV2 for GzipEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<()> {
loop {
match &mut self.state {
State::Header(header) => {
output.copy_unwritten_from(&mut *header);
if header.unwritten().is_empty() {
self.state = State::Encoding;
}
}
State::Encoding => {
let prior_written = input.written().len();
self.inner.encode(input, output)?;
self.crc.update(&input.written()[prior_written..]);
}
State::Footer(_) | State::Done => {
return Err(io::Error::other("encode after complete"));
}
};
if input.unwritten().is_empty() || output.has_no_spare_space() {
return Ok(());
}
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
loop {
let done = match &mut self.state {
State::Header(header) => {
output.copy_unwritten_from(&mut *header);
if header.unwritten().is_empty() {
self.state = State::Encoding;
}
false
}
State::Encoding => self.inner.flush(output)?,
State::Footer(footer) => {
output.copy_unwritten_from(&mut *footer);
if footer.unwritten().is_empty() {
self.state = State::Done;
true
} else {
false
}
}
State::Done => true,
};
if done {
return Ok(true);
}
if output.has_no_spare_space() {
return Ok(false);
}
}
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
loop {
match &mut self.state {
State::Header(header) => {
output.copy_unwritten_from(&mut *header);
if header.unwritten().is_empty() {
self.state = State::Encoding;
}
}
State::Encoding => {
if self.inner.finish(output)? {
self.state = State::Footer(self.footer().into());
}
}
State::Footer(footer) => {
output.copy_unwritten_from(&mut *footer);
if footer.unwritten().is_empty() {
self.state = State::Done;
}
}
State::Done => {}
};
if let State::Done = self.state {
return Ok(true);
}
if output.has_no_spare_space() {
return Ok(false);
}
}
}
}
@@ -0,0 +1,189 @@
use compression_core::util::PartialBuffer;
use flate2::Crc;
use std::io;
#[derive(Debug, Default)]
struct Flags {
_ascii: bool,
crc: bool,
extra: bool,
filename: bool,
comment: bool,
}
#[derive(Debug, Default)]
pub(super) struct Header {
flags: Flags,
}
#[derive(Debug)]
enum State {
Fixed(PartialBuffer<[u8; 10]>),
ExtraLen(PartialBuffer<[u8; 2]>),
Extra(usize),
Filename,
Comment,
Crc(PartialBuffer<[u8; 2]>),
Done,
}
impl Default for State {
fn default() -> Self {
State::Fixed(<_>::default())
}
}
#[derive(Debug, Default)]
pub(super) struct Parser {
state: State,
header: Header,
}
impl Header {
fn parse(input: &[u8; 10]) -> io::Result<Self> {
if input[0..3] != [0x1f, 0x8b, 0x08] {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Invalid gzip header",
));
}
let flag = input[3];
let flags = Flags {
_ascii: (flag & 0b0000_0001) != 0,
crc: (flag & 0b0000_0010) != 0,
extra: (flag & 0b0000_0100) != 0,
filename: (flag & 0b0000_1000) != 0,
comment: (flag & 0b0001_0000) != 0,
};
Ok(Header { flags })
}
}
fn consume_input(crc: &mut Crc, n: usize, input: &mut PartialBuffer<&[u8]>) {
crc.update(&input.unwritten()[..n]);
input.advance(n);
}
fn consume_cstr(crc: &mut Crc, input: &mut PartialBuffer<&[u8]>) -> Option<()> {
if let Some(len) = memchr::memchr(0, input.unwritten()) {
consume_input(crc, len + 1, input);
Some(())
} else {
consume_input(crc, input.unwritten().len(), input);
None
}
}
impl Parser {
pub(super) fn input(
&mut self,
crc: &mut Crc,
input: &mut PartialBuffer<&[u8]>,
) -> io::Result<Option<Header>> {
loop {
match &mut self.state {
State::Fixed(data) => {
data.copy_unwritten_from(input);
if data.unwritten().is_empty() {
let data = data.get_mut();
crc.update(data);
self.header = Header::parse(data)?;
self.state = State::ExtraLen(<_>::default());
} else {
break Ok(None);
}
}
State::ExtraLen(data) => {
if !self.header.flags.extra {
self.state = State::Filename;
continue;
}
data.copy_unwritten_from(input);
if data.unwritten().is_empty() {
let data = data.get_mut();
crc.update(data);
let len = u16::from_le_bytes(*data);
self.state = State::Extra(len.into());
} else {
break Ok(None);
}
}
State::Extra(bytes_to_consume) => {
let n = input.unwritten().len().min(*bytes_to_consume);
*bytes_to_consume -= n;
consume_input(crc, n, input);
if *bytes_to_consume == 0 {
self.state = State::Filename;
} else {
break Ok(None);
}
}
State::Filename => {
if !self.header.flags.filename {
self.state = State::Comment;
continue;
}
if consume_cstr(crc, input).is_none() {
break Ok(None);
}
self.state = State::Comment;
}
State::Comment => {
if !self.header.flags.comment {
self.state = State::Crc(<_>::default());
continue;
}
if consume_cstr(crc, input).is_none() {
break Ok(None);
}
self.state = State::Crc(<_>::default());
}
State::Crc(data) => {
let header = std::mem::take(&mut self.header);
if !self.header.flags.crc {
self.state = State::Done;
break Ok(Some(header));
}
data.copy_unwritten_from(input);
break if data.unwritten().is_empty() {
let data = data.take().into_inner();
self.state = State::Done;
let checksum = crc.sum().to_le_bytes();
if data == checksum[..2] {
Ok(Some(header))
} else {
Err(io::Error::new(
io::ErrorKind::InvalidData,
"CRC computed for header does not match",
))
}
} else {
Ok(None)
};
}
State::Done => break Err(io::Error::other("parser used after done")),
}
}
}
}
@@ -0,0 +1,5 @@
mod decoder;
mod encoder;
mod header;
pub use self::{decoder::GzipDecoder, encoder::GzipEncoder};
+234
View File
@@ -0,0 +1,234 @@
//! Adaptors for various compression algorithms.
#![cfg_attr(docsrs, feature(doc_cfg))]
use std::io::Result;
pub use compression_core as core;
#[cfg(feature = "brotli")]
pub mod brotli;
#[cfg(feature = "bzip2")]
pub mod bzip2;
#[cfg(feature = "deflate")]
pub mod deflate;
#[cfg(feature = "deflate64")]
pub mod deflate64;
#[cfg(feature = "flate2")]
pub mod flate;
#[cfg(feature = "gzip")]
pub mod gzip;
#[cfg(feature = "lz4")]
pub mod lz4;
#[cfg(feature = "lzma")]
pub mod lzma;
#[cfg(feature = "xz")]
pub mod xz;
#[cfg(feature = "lzma")]
pub mod xz2;
#[cfg(feature = "zlib")]
pub mod zlib;
#[cfg(feature = "zstd")]
pub mod zstd;
use compression_core::util::{PartialBuffer, WriteBuffer};
#[cfg(feature = "brotli")]
pub use self::brotli::{BrotliDecoder, BrotliEncoder};
#[cfg(feature = "bzip2")]
pub use self::bzip2::{BzDecoder, BzEncoder};
#[cfg(feature = "deflate")]
pub use self::deflate::{DeflateDecoder, DeflateEncoder};
#[cfg(feature = "deflate64")]
pub use self::deflate64::Deflate64Decoder;
#[cfg(feature = "flate2")]
pub use self::flate::{FlateDecoder, FlateEncoder};
#[cfg(feature = "gzip")]
pub use self::gzip::{GzipDecoder, GzipEncoder};
#[cfg(feature = "lz4")]
pub use self::lz4::{Lz4Decoder, Lz4Encoder};
#[cfg(feature = "lzma")]
pub use self::lzma::{LzmaDecoder, LzmaEncoder};
#[cfg(feature = "xz")]
pub use self::xz::{XzDecoder, XzEncoder};
#[cfg(feature = "lzma")]
pub use self::xz2::{Xz2Decoder, Xz2Encoder, Xz2FileFormat};
#[cfg(feature = "zlib")]
pub use self::zlib::{ZlibDecoder, ZlibEncoder};
#[cfg(feature = "zstd")]
pub use self::zstd::{ZstdDecoder, ZstdEncoder};
fn forward_output<R>(
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
f: impl FnOnce(&mut WriteBuffer<'_>) -> R,
) -> R {
let written_len = output.written_len();
let output_buffer = output.get_mut();
let mut write_buffer = WriteBuffer::new_initialized(output_buffer.as_mut());
write_buffer.advance(written_len);
let result = f(&mut write_buffer);
let new_written_len = write_buffer.written_len();
output.advance(new_written_len - written_len);
result
}
fn forward_input_output<R>(
input: &mut PartialBuffer<impl AsRef<[u8]>>,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
f: impl FnOnce(&mut PartialBuffer<&[u8]>, &mut WriteBuffer<'_>) -> R,
) -> R {
let written_len = input.written_len();
let input_buffer = input.get_mut();
let mut partial_buffer = PartialBuffer::new(input_buffer.as_ref());
partial_buffer.advance(written_len);
let result = forward_output(output, |output| f(&mut partial_buffer, output));
let new_written_len = partial_buffer.written_len();
input.advance(new_written_len - written_len);
result
}
pub trait Encode {
fn encode(
&mut self,
input: &mut PartialBuffer<impl AsRef<[u8]>>,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<()>;
/// Returns whether the internal buffers are flushed
fn flush(&mut self, output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>)
-> Result<bool>;
/// Returns whether the internal buffers are flushed and the end of the stream is written
fn finish(
&mut self,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool>;
}
impl<T: EncodeV2 + ?Sized> Encode for T {
fn encode(
&mut self,
input: &mut PartialBuffer<impl AsRef<[u8]>>,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<()> {
forward_input_output(input, output, |input, output| {
EncodeV2::encode(self, input, output)
})
}
fn flush(
&mut self,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool> {
forward_output(output, |output| EncodeV2::flush(self, output))
}
fn finish(
&mut self,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool> {
forward_output(output, |output| EncodeV2::finish(self, output))
}
}
/// version 2 of [`Encode`] that is trait object safe.
///
/// The different from [`Encode`] is that:
/// - It doesn't have any generic in it, so it is trait object safe
/// - It uses [`WriteBuffer`] for output, which will support uninitialized buffer.
pub trait EncodeV2 {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()>;
/// Returns whether the internal buffers are flushed
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool>;
/// Returns whether the internal buffers are flushed and the end of the stream is written
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool>;
}
pub trait Decode {
/// Reinitializes this decoder ready to decode a new member/frame of data.
fn reinit(&mut self) -> Result<()>;
/// Returns whether the end of the stream has been read
fn decode(
&mut self,
input: &mut PartialBuffer<impl AsRef<[u8]>>,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool>;
/// Returns whether the internal buffers are flushed
fn flush(&mut self, output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>)
-> Result<bool>;
/// Returns whether the internal buffers are flushed
fn finish(
&mut self,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool>;
}
impl<T: DecodeV2 + ?Sized> Decode for T {
fn reinit(&mut self) -> Result<()> {
DecodeV2::reinit(self)
}
fn decode(
&mut self,
input: &mut PartialBuffer<impl AsRef<[u8]>>,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool> {
forward_input_output(input, output, |input, output| {
DecodeV2::decode(self, input, output)
})
}
fn flush(
&mut self,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool> {
forward_output(output, |output| DecodeV2::flush(self, output))
}
fn finish(
&mut self,
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
) -> Result<bool> {
forward_output(output, |output| DecodeV2::finish(self, output))
}
}
/// version 2 [`Decode`] that is trait object safe.
///
/// The different from [`Decode`] is that:
/// - It doesn't have any generic in it, so it is trait object safe
/// - It uses [`WriteBuffer`] for output, which will support uninitialized buffer.
pub trait DecodeV2 {
/// Reinitializes this decoder ready to decode a new member/frame of data.
fn reinit(&mut self) -> Result<()>;
/// Returns whether the end of the stream has been read
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool>;
/// Returns whether the internal buffers are flushed
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool>;
/// Returns whether the internal buffers are flushed
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool>;
}
pub trait DecodedSize {
/// Returns the size of the input when uncompressed.
fn decoded_size(input: &[u8]) -> Result<u64>;
}
@@ -0,0 +1,120 @@
use crate::DecodeV2;
use compression_core::{
unshared::Unshared,
util::{PartialBuffer, WriteBuffer},
};
use lz4::liblz4::{
check_error, LZ4FDecompressionContext, LZ4F_createDecompressionContext, LZ4F_decompress,
LZ4F_freeDecompressionContext, LZ4F_resetDecompressionContext, LZ4F_VERSION,
};
use std::io::Result;
#[derive(Debug)]
struct DecoderContext {
ctx: LZ4FDecompressionContext,
}
#[derive(Debug)]
pub struct Lz4Decoder {
ctx: Unshared<DecoderContext>,
stream_ended: bool,
}
impl DecoderContext {
fn new() -> Result<Self> {
let mut context = LZ4FDecompressionContext(core::ptr::null_mut());
check_error(unsafe { LZ4F_createDecompressionContext(&mut context, LZ4F_VERSION) })?;
Ok(Self { ctx: context })
}
}
impl Drop for DecoderContext {
fn drop(&mut self) {
unsafe { LZ4F_freeDecompressionContext(self.ctx) };
}
}
impl Default for Lz4Decoder {
fn default() -> Self {
Self {
ctx: Unshared::new(DecoderContext::new().unwrap()),
stream_ended: false,
}
}
}
impl Lz4Decoder {
pub fn new() -> Self {
Self::default()
}
}
impl DecodeV2 for Lz4Decoder {
fn reinit(&mut self) -> Result<()> {
unsafe { LZ4F_resetDecompressionContext(self.ctx.get_mut().ctx) };
self.stream_ended = false;
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
let mut input_size = input.unwritten().len();
// Safety: We **trust** lz4 bytes to properly function as expected,
// only write decompressed, initialized data into the buffer properly.
let result = unsafe {
let out_buf = output.unwritten_mut();
let mut output_size = out_buf.len();
let result = check_error(LZ4F_decompress(
self.ctx.get_mut().ctx,
out_buf.as_mut_ptr() as *mut _,
&mut output_size,
input.unwritten().as_ptr(),
&mut input_size,
core::ptr::null(),
))?;
output.assume_init_and_advance(output_size);
result
};
input.advance(input_size);
let finished = result == 0;
if finished {
self.stream_ended = true;
}
Ok(finished)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.decode(&mut PartialBuffer::new(&[][..]), output)?;
loop {
let old_len = output.written_len();
self.decode(&mut PartialBuffer::new(&[][..]), output)?;
if output.written_len() == old_len {
break;
}
}
Ok(!output.has_no_spare_space())
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.flush(output)?;
if self.stream_ended {
Ok(true)
} else {
Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"lz4 stream did not finish",
))
}
}
}
@@ -0,0 +1,300 @@
use crate::{lz4::params::EncoderParams, EncodeV2};
use compression_core::{
unshared::Unshared,
util::{PartialBuffer, WriteBuffer},
};
use lz4::liblz4::{
check_error, LZ4FCompressionContext, LZ4FPreferences, LZ4F_compressBegin, LZ4F_compressBound,
LZ4F_compressEnd, LZ4F_compressUpdate, LZ4F_createCompressionContext, LZ4F_flush,
LZ4F_freeCompressionContext, LZ4F_VERSION,
};
use std::io::{self, Result};
// https://github.com/lz4/lz4/blob/9d53d8bb6c4120345a0966e5d8b16d7def1f32c5/lib/lz4frame.h#L281
const LZ4F_HEADER_SIZE_MAX: usize = 19;
#[derive(Debug)]
struct EncoderContext {
ctx: LZ4FCompressionContext,
}
#[derive(Clone, Copy, Debug)]
enum State {
Header,
Encoding,
Footer,
Done,
}
enum Lz4Fn<'a, 'b> {
Begin,
Update {
input: &'a mut PartialBuffer<&'b [u8]>,
},
Flush,
End,
}
#[derive(Debug)]
pub struct Lz4Encoder {
ctx: Unshared<EncoderContext>,
state: State,
preferences: LZ4FPreferences,
limit: usize,
maybe_buffer: Option<PartialBuffer<Vec<u8>>>,
/// Minimum dst buffer size for a block
block_buffer_size: usize,
/// Minimum dst buffer size for flush/end
flush_buffer_size: usize,
}
// minimum size of destination buffer for compressing `src_size` bytes
fn min_dst_size(src_size: usize, preferences: &LZ4FPreferences) -> usize {
unsafe { LZ4F_compressBound(src_size, preferences) }
}
impl EncoderContext {
fn new() -> Result<Self> {
let mut context = LZ4FCompressionContext(core::ptr::null_mut());
check_error(unsafe { LZ4F_createCompressionContext(&mut context, LZ4F_VERSION) })?;
Ok(Self { ctx: context })
}
}
impl Drop for EncoderContext {
fn drop(&mut self) {
unsafe { LZ4F_freeCompressionContext(self.ctx) };
}
}
impl Lz4Encoder {
pub fn new(params: EncoderParams) -> Self {
let preferences = LZ4FPreferences::from(params);
let block_size = preferences.frame_info.block_size_id.get_size();
let block_buffer_size = min_dst_size(block_size, &preferences);
let flush_buffer_size = min_dst_size(0, &preferences);
Self {
ctx: Unshared::new(EncoderContext::new().unwrap()),
state: State::Header,
preferences,
limit: block_size,
maybe_buffer: None,
block_buffer_size,
flush_buffer_size,
}
}
pub fn buffer_size(&self) -> usize {
self.block_buffer_size
}
fn drain_buffer(&mut self, output: &mut WriteBuffer<'_>) -> (usize, usize) {
match self.maybe_buffer.as_mut() {
Some(buffer) => {
let drained_bytes = output.copy_unwritten_from(buffer);
(drained_bytes, buffer.unwritten().len())
}
None => (0, 0),
}
}
fn write(&mut self, lz4_fn: Lz4Fn<'_, '_>, output: &mut WriteBuffer<'_>) -> Result<usize> {
let (drained_before, undrained) = self.drain_buffer(output);
if undrained > 0 || output.has_no_spare_space() {
return Ok(drained_before);
}
let mut src_size = 0;
let min_dst_size = match &lz4_fn {
Lz4Fn::Begin => LZ4F_HEADER_SIZE_MAX,
Lz4Fn::Update { input } => {
src_size = input.unwritten().len().min(self.limit);
min_dst_size(src_size, &self.preferences)
}
Lz4Fn::Flush | Lz4Fn::End => self.flush_buffer_size,
};
// Safety: We **trust** lz4 to not write uninitialized bytes
let out_buf = unsafe { output.unwritten_mut() };
let output_len = out_buf.len();
let (dst_buffer, dst_size, maybe_internal_buffer) = if min_dst_size > output_len {
let buffer_size = self.block_buffer_size;
let buffer = self
.maybe_buffer
.get_or_insert_with(|| PartialBuffer::new(Vec::with_capacity(buffer_size)));
buffer.reset();
buffer.get_mut().clear();
(
buffer.get_mut().spare_capacity_mut().as_mut_ptr(),
buffer_size,
Some(buffer),
)
} else {
(out_buf.as_mut_ptr(), output_len, None)
};
let dst_buffer = dst_buffer as *mut u8;
let len = match lz4_fn {
Lz4Fn::Begin => {
let len = check_error(unsafe {
LZ4F_compressBegin(
self.ctx.get_mut().ctx,
dst_buffer,
dst_size,
&self.preferences,
)
})?;
self.state = State::Encoding;
len
}
Lz4Fn::Update { input } => {
let len = check_error(unsafe {
LZ4F_compressUpdate(
self.ctx.get_mut().ctx,
dst_buffer,
dst_size,
input.unwritten().as_ptr(),
src_size,
core::ptr::null(),
)
})?;
input.advance(src_size);
len
}
Lz4Fn::Flush => check_error(unsafe {
LZ4F_flush(
self.ctx.get_mut().ctx,
dst_buffer,
dst_size,
core::ptr::null(),
)
})?,
Lz4Fn::End => {
let len = check_error(unsafe {
LZ4F_compressEnd(
self.ctx.get_mut().ctx,
dst_buffer,
dst_size,
core::ptr::null(),
)
})?;
self.state = State::Footer;
len
}
};
let drained_after = if let Some(internal_buffer) = maybe_internal_buffer {
// Safety: We **trust** lz4 to properly write data into the buffer
unsafe {
internal_buffer.get_mut().set_len(len);
}
let (d, _) = self.drain_buffer(output);
d
} else {
// Safety: We **trust** lz4 to properly write data into the buffer
unsafe { output.assume_init_and_advance(len) };
len
};
Ok(drained_before + drained_after)
}
}
impl EncodeV2 for Lz4Encoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()> {
loop {
match self.state {
State::Header => {
self.write(Lz4Fn::Begin, output)?;
}
State::Encoding => {
self.write(Lz4Fn::Update { input }, output)?;
}
State::Footer | State::Done => {
return Err(io::Error::other("encode after complete"));
}
}
if input.unwritten().is_empty() || output.has_no_spare_space() {
return Ok(());
}
}
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
loop {
let done = match self.state {
State::Header => {
self.write(Lz4Fn::Begin, output)?;
false
}
State::Encoding => {
let len = self.write(Lz4Fn::Flush, output)?;
len == 0
}
State::Footer => {
let (_, undrained) = self.drain_buffer(output);
if undrained == 0 {
self.state = State::Done;
true
} else {
false
}
}
State::Done => true,
};
if done {
return Ok(true);
}
if output.has_no_spare_space() {
return Ok(false);
}
}
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
loop {
match self.state {
State::Header => {
self.write(Lz4Fn::Begin, output)?;
}
State::Encoding => {
self.write(Lz4Fn::End, output)?;
}
State::Footer => {
let (_, undrained) = self.drain_buffer(output);
if undrained == 0 {
self.state = State::Done;
}
}
State::Done => {}
}
if let State::Done = self.state {
return Ok(true);
}
if output.has_no_spare_space() {
return Ok(false);
}
}
}
}
@@ -0,0 +1,5 @@
mod decoder;
mod encoder;
pub mod params;
pub use self::{decoder::Lz4Decoder, encoder::Lz4Encoder};
@@ -0,0 +1,102 @@
//! This module contains lz4-specific types for async-compression.
use std::convert::TryInto;
use compression_core::Level;
pub use lz4::liblz4::BlockSize;
use lz4::{
liblz4::{BlockChecksum, FrameType, LZ4FFrameInfo, LZ4FPreferences},
BlockMode, ContentChecksum,
};
/// lz4 compression parameters builder. This is a stable wrapper around lz4's own encoder
/// params type, to abstract over different versions of the lz4 library.
///
/// See the [lz4 documentation](https://github.com/lz4/lz4/blob/dev/doc/lz4frame_manual.html)
/// for more information on these parameters.
///
/// # Examples
///
/// ```
/// use compression_codecs::lz4;
///
/// let params = lz4::params::EncoderParams::default()
/// .block_size(lz4::params::BlockSize::Max1MB)
/// .content_checksum(true);
/// ```
#[derive(Clone, Debug, Default)]
pub struct EncoderParams {
block_size: Option<BlockSize>,
block_checksum: Option<BlockChecksum>,
content_checksum: Option<ContentChecksum>,
level: Level,
}
impl EncoderParams {
pub fn level(mut self, level: Level) -> Self {
self.level = level;
self
}
/// Sets input block size.
pub fn block_size(mut self, block_size: BlockSize) -> Self {
self.block_size = Some(block_size);
self
}
/// Add a 32-bit checksum of frame's decompressed data.
pub fn content_checksum(mut self, enable: bool) -> Self {
self.content_checksum = Some(if enable {
ContentChecksum::ChecksumEnabled
} else {
ContentChecksum::NoChecksum
});
self
}
/// Each block followed by a checksum of block's compressed data.
pub fn block_checksum(mut self, enable: bool) -> Self {
self.block_checksum = Some(if enable {
BlockChecksum::BlockChecksumEnabled
} else {
BlockChecksum::NoBlockChecksum
});
self
}
}
impl From<EncoderParams> for LZ4FPreferences {
fn from(value: EncoderParams) -> Self {
let block_size_id = value.block_size.clone().unwrap_or(BlockSize::Default);
let content_checksum_flag = value
.content_checksum
.clone()
.unwrap_or(ContentChecksum::NoChecksum);
let block_checksum_flag = value
.block_checksum
.clone()
.unwrap_or(BlockChecksum::NoBlockChecksum);
let compression_level = match value.level {
Level::Fastest => 0,
Level::Best => 12,
Level::Precise(quality) => quality.try_into().unwrap_or(0).clamp(0, 12),
_ => 0,
};
LZ4FPreferences {
frame_info: LZ4FFrameInfo {
block_size_id,
block_mode: BlockMode::Linked,
content_checksum_flag,
frame_type: FrameType::Frame,
content_size: 0,
dict_id: 0,
block_checksum_flag,
},
compression_level,
auto_flush: 0,
favor_dec_speed: 0,
reserved: [0; 3],
}
}
}
@@ -0,0 +1,63 @@
use crate::{DecodeV2, DecodedSize, Xz2Decoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::{convert::TryInto, io::Result};
/// Lzma decoding stream
#[derive(Debug)]
pub struct LzmaDecoder {
inner: Xz2Decoder,
}
impl From<Xz2Decoder> for LzmaDecoder {
fn from(inner: Xz2Decoder) -> Self {
Self { inner }
}
}
impl Default for LzmaDecoder {
fn default() -> Self {
Self {
inner: Xz2Decoder::new(usize::MAX.try_into().unwrap()),
}
}
}
impl LzmaDecoder {
pub fn new() -> Self {
Self::default()
}
pub fn with_memlimit(memlimit: u64) -> Self {
Self {
inner: Xz2Decoder::new(memlimit),
}
}
}
impl DecodeV2 for LzmaDecoder {
fn reinit(&mut self) -> Result<()> {
self.inner.reinit()
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
self.inner.decode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
impl DecodedSize for LzmaDecoder {
fn decoded_size(input: &[u8]) -> Result<u64> {
Xz2Decoder::decoded_size(input)
}
}
@@ -0,0 +1,45 @@
use crate::{EncodeV2, Xz2Encoder, Xz2FileFormat};
use compression_core::{
util::{PartialBuffer, WriteBuffer},
Level,
};
use std::io::Result;
/// Lzma encoding stream
#[derive(Debug)]
pub struct LzmaEncoder {
inner: Xz2Encoder,
}
impl LzmaEncoder {
pub fn new(level: Level) -> Self {
Self {
inner: Xz2Encoder::new(Xz2FileFormat::Lzma, level),
}
}
}
impl From<Xz2Encoder> for LzmaEncoder {
fn from(inner: Xz2Encoder) -> Self {
Self { inner }
}
}
impl EncodeV2 for LzmaEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()> {
self.inner.encode(input, output)
}
fn flush(&mut self, _output: &mut WriteBuffer<'_>) -> Result<bool> {
// Flush on LZMA 1 is not supported
Ok(true)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
@@ -0,0 +1,6 @@
mod decoder;
mod encoder;
pub mod params;
pub use self::{decoder::LzmaDecoder, encoder::LzmaEncoder};
@@ -0,0 +1,416 @@
//! Clone-able structs to build the non-clone-ables in liblzma
use std::convert::TryFrom;
#[cfg(feature = "xz-parallel")]
use std::num::NonZeroU32;
/// Used to control how the LZMA stream is created.
#[derive(Debug, Clone)]
pub enum LzmaEncoderParams {
Easy {
preset: u32,
check: liblzma::stream::Check,
},
Lzma {
options: LzmaOptions,
},
Raw {
filters: LzmaFilters,
},
Stream {
filters: LzmaFilters,
check: liblzma::stream::Check,
},
#[cfg(feature = "xz-parallel")]
MultiThread {
builder: MtStreamBuilder,
},
}
impl TryFrom<&LzmaEncoderParams> for liblzma::stream::Stream {
type Error = liblzma::stream::Error;
fn try_from(value: &LzmaEncoderParams) -> Result<Self, Self::Error> {
let stream = match value {
LzmaEncoderParams::Easy { preset, check } => Self::new_easy_encoder(*preset, *check)?,
LzmaEncoderParams::Lzma { options } => {
let options = liblzma::stream::LzmaOptions::try_from(options)?;
Self::new_lzma_encoder(&options)?
}
LzmaEncoderParams::Raw { filters } => {
let filters = liblzma::stream::Filters::try_from(filters)?;
Self::new_raw_encoder(&filters)?
}
LzmaEncoderParams::Stream { filters, check } => {
let filters = liblzma::stream::Filters::try_from(filters)?;
Self::new_stream_encoder(&filters, *check)?
}
#[cfg(feature = "xz-parallel")]
LzmaEncoderParams::MultiThread { builder } => {
let builder = liblzma::stream::MtStreamBuilder::try_from(builder)?;
builder.encoder()?
}
};
Ok(stream)
}
}
/// Directly translate to how the stream is constructed
#[derive(Clone, Debug)]
pub enum LzmaDecoderParams {
Auto {
mem_limit: u64,
flags: u32,
},
Lzip {
mem_limit: u64,
flags: u32,
},
Lzma {
mem_limit: u64,
},
Raw {
filters: LzmaFilters,
},
Stream {
mem_limit: u64,
flags: u32,
},
#[cfg(feature = "xz-parallel")]
MultiThread {
builder: MtStreamBuilder,
},
}
impl TryFrom<&LzmaDecoderParams> for liblzma::stream::Stream {
type Error = liblzma::stream::Error;
fn try_from(value: &LzmaDecoderParams) -> Result<Self, Self::Error> {
let stream = match value {
LzmaDecoderParams::Auto { mem_limit, flags } => {
Self::new_auto_decoder(*mem_limit, *flags)?
}
LzmaDecoderParams::Lzip { mem_limit, flags } => {
Self::new_lzip_decoder(*mem_limit, *flags)?
}
LzmaDecoderParams::Lzma { mem_limit } => Self::new_lzma_decoder(*mem_limit)?,
LzmaDecoderParams::Stream { mem_limit, flags } => {
Self::new_stream_decoder(*mem_limit, *flags)?
}
LzmaDecoderParams::Raw { filters } => {
let filters = liblzma::stream::Filters::try_from(filters)?;
Self::new_raw_decoder(&filters)?
}
#[cfg(feature = "xz-parallel")]
LzmaDecoderParams::MultiThread { builder } => {
let builder = liblzma::stream::MtStreamBuilder::try_from(builder)?;
builder.decoder()?
}
};
Ok(stream)
}
}
/// Clone-able `liblzma::Filters`.
#[derive(Default, Clone, Debug)]
pub struct LzmaFilters {
filters: Vec<LzmaFilter>,
}
impl LzmaFilters {
/// Add `LzmaFilter` to the collection
pub fn add_filter(mut self, filter: LzmaFilter) -> Self {
self.filters.push(filter);
self
}
}
/// An individual filter directly corresponding to liblzma Filters method calls
#[derive(Debug, Clone)]
pub enum LzmaFilter {
Arm(Option<Vec<u8>>),
Arm64(Option<Vec<u8>>),
ArmThumb(Option<Vec<u8>>),
Delta(Option<Vec<u8>>),
Ia64(Option<Vec<u8>>),
Lzma1(LzmaOptions),
Lzma1Properties(Vec<u8>),
Lzma2(LzmaOptions),
Lzma2Properties(Vec<u8>),
PowerPc(Option<Vec<u8>>),
Sparc(Option<Vec<u8>>),
X86(Option<Vec<u8>>),
}
impl TryFrom<&LzmaFilters> for liblzma::stream::Filters {
type Error = liblzma::stream::Error;
fn try_from(value: &LzmaFilters) -> Result<Self, Self::Error> {
let mut filters = liblzma::stream::Filters::new();
for f in value.filters.iter() {
match f {
LzmaFilter::Arm(Some(p)) => filters.arm_properties(p)?,
LzmaFilter::Arm(None) => filters.arm(),
LzmaFilter::Arm64(Some(p)) => filters.arm64_properties(p)?,
LzmaFilter::Arm64(None) => filters.arm64(),
LzmaFilter::ArmThumb(Some(p)) => filters.arm_thumb_properties(p)?,
LzmaFilter::ArmThumb(None) => filters.arm_thumb(),
LzmaFilter::Delta(Some(p)) => filters.delta_properties(p)?,
LzmaFilter::Delta(None) => filters.delta(),
LzmaFilter::Ia64(Some(p)) => filters.ia64_properties(p)?,
LzmaFilter::Ia64(None) => filters.ia64(),
LzmaFilter::Lzma1(opts) => {
let opts = liblzma::stream::LzmaOptions::try_from(opts)?;
filters.lzma1(&opts)
}
LzmaFilter::Lzma1Properties(p) => filters.lzma1_properties(p)?,
LzmaFilter::Lzma2(opts) => {
let opts = liblzma::stream::LzmaOptions::try_from(opts)?;
filters.lzma2(&opts)
}
LzmaFilter::Lzma2Properties(p) => filters.lzma2_properties(p)?,
LzmaFilter::PowerPc(Some(p)) => filters.powerpc_properties(p)?,
LzmaFilter::PowerPc(None) => filters.powerpc(),
LzmaFilter::Sparc(Some(p)) => filters.sparc_properties(p)?,
LzmaFilter::Sparc(None) => filters.sparc(),
LzmaFilter::X86(Some(p)) => filters.x86_properties(p)?,
LzmaFilter::X86(None) => filters.x86(),
};
}
Ok(filters)
}
}
/// A builder for liblzma::LzmaOptions, so that it can be cloned
#[derive(Default, Clone)]
pub struct LzmaOptions {
preset: Option<u32>,
depth: Option<u32>,
dict_size: Option<u32>,
literal_context_bits: Option<u32>,
literal_position_bits: Option<u32>,
match_finder: Option<liblzma::stream::MatchFinder>,
mode: Option<liblzma::stream::Mode>,
nice_len: Option<u32>,
position_bits: Option<u32>,
}
impl std::fmt::Debug for LzmaOptions {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let match_finder = self.match_finder.map(|m| match m {
liblzma::stream::MatchFinder::HashChain3 => "HashChain3",
liblzma::stream::MatchFinder::HashChain4 => "HashChain4",
liblzma::stream::MatchFinder::BinaryTree2 => "BTree2",
liblzma::stream::MatchFinder::BinaryTree3 => "BTree3",
liblzma::stream::MatchFinder::BinaryTree4 => "BTree4",
});
let mode = self.mode.map(|m| match m {
liblzma::stream::Mode::Fast => "Fast",
liblzma::stream::Mode::Normal => "Normal",
});
f.debug_struct("LzmaOptions")
.field("preset", &self.preset)
.field("depth", &self.depth)
.field("dict_size", &self.dict_size)
.field("literal_context_bits", &self.literal_context_bits)
.field("literal_position_bits", &self.literal_position_bits)
.field("match_finder", &match_finder)
.field("mode", &mode)
.field("nice_len", &self.nice_len)
.field("position_bits", &self.position_bits)
.finish()
}
}
impl LzmaOptions {
pub fn preset(mut self, value: u32) -> Self {
self.preset = Some(value);
self
}
pub fn depth(mut self, value: u32) -> Self {
self.depth = Some(value);
self
}
pub fn dict_size(mut self, value: u32) -> Self {
self.dict_size = Some(value);
self
}
pub fn literal_context_bits(mut self, value: u32) -> Self {
self.literal_context_bits = Some(value);
self
}
pub fn literal_position_bits(mut self, value: u32) -> Self {
self.literal_position_bits = Some(value);
self
}
pub fn match_finder(mut self, value: liblzma::stream::MatchFinder) -> Self {
self.match_finder = Some(value);
self
}
pub fn mode(mut self, value: liblzma::stream::Mode) -> Self {
self.mode = Some(value);
self
}
pub fn nice_len(mut self, value: u32) -> Self {
self.nice_len = Some(value);
self
}
pub fn position_bits(mut self, value: u32) -> Self {
self.position_bits = Some(value);
self
}
}
impl TryFrom<&LzmaOptions> for liblzma::stream::LzmaOptions {
type Error = liblzma::stream::Error;
fn try_from(value: &LzmaOptions) -> Result<Self, Self::Error> {
let mut s = match value.preset {
Some(preset) => liblzma::stream::LzmaOptions::new_preset(preset)?,
None => liblzma::stream::LzmaOptions::new(),
};
if let Some(depth) = value.depth {
s.depth(depth);
}
if let Some(dict_size) = value.dict_size {
s.dict_size(dict_size);
}
if let Some(bits) = value.literal_context_bits {
s.literal_context_bits(bits);
}
if let Some(bits) = value.literal_position_bits {
s.literal_position_bits(bits);
}
if let Some(mf) = value.match_finder {
s.match_finder(mf);
}
if let Some(mode) = value.mode {
s.mode(mode);
}
if let Some(len) = value.nice_len {
s.nice_len(len);
}
if let Some(bits) = value.position_bits {
s.position_bits(bits);
}
Ok(s)
}
}
#[cfg(feature = "xz-parallel")]
#[derive(Default, Clone)]
/// Used to build a clonable mt stream builder
pub struct MtStreamBuilder {
block_size: Option<u64>,
preset: Option<u32>,
check: Option<liblzma::stream::Check>,
filters: Option<LzmaFilters>,
mem_limit_stop: Option<u64>,
mem_limit_threading: Option<u64>,
threads: Option<NonZeroU32>,
timeout_ms: Option<u32>,
}
#[cfg(feature = "xz-parallel")]
impl std::fmt::Debug for MtStreamBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let check = self.check.map(|s| match s {
liblzma::stream::Check::None => "None",
liblzma::stream::Check::Crc32 => "Crc32",
liblzma::stream::Check::Crc64 => "Crc64",
liblzma::stream::Check::Sha256 => "Sha256",
});
f.debug_struct("MtStreamBuilder")
.field("block_size", &self.block_size)
.field("preset", &self.preset)
.field("check", &check)
.field("filters", &self.filters)
.field("mem_limit_stop", &self.mem_limit_stop)
.field("mem_limit_threading", &self.mem_limit_threading)
.field("threads", &self.threads)
.field("timeout_ms", &self.timeout_ms)
.finish()
}
}
#[cfg(feature = "xz-parallel")]
impl MtStreamBuilder {
pub fn block_size(&mut self, block_size: u64) -> &mut Self {
self.block_size = Some(block_size);
self
}
pub fn preset(&mut self, preset: u32) -> &mut Self {
self.preset = Some(preset);
self
}
pub fn check(&mut self, check: liblzma::stream::Check) -> &mut Self {
self.check = Some(check);
self
}
pub fn filters(&mut self, filters: LzmaFilters) -> &mut Self {
self.filters = Some(filters);
self
}
pub fn mem_limit_stop(&mut self, mem_limit_stop: u64) -> &mut Self {
self.mem_limit_stop = Some(mem_limit_stop);
self
}
pub fn mem_limit_threading(&mut self, mem_limit_threading: u64) -> &mut Self {
self.mem_limit_threading = Some(mem_limit_threading);
self
}
pub fn threads(&mut self, threads: NonZeroU32) -> &mut Self {
self.threads = Some(threads);
self
}
pub fn timeout_ms(&mut self, timeout_ms: u32) -> &mut Self {
self.timeout_ms = Some(timeout_ms);
self
}
}
#[cfg(feature = "xz-parallel")]
impl TryFrom<&MtStreamBuilder> for liblzma::stream::MtStreamBuilder {
type Error = liblzma::stream::Error;
fn try_from(value: &MtStreamBuilder) -> Result<Self, Self::Error> {
let mut mt = liblzma::stream::MtStreamBuilder::new();
if let Some(block_size) = value.block_size {
mt.block_size(block_size);
}
if let Some(preset) = value.preset {
mt.preset(preset);
}
if let Some(check) = value.check {
mt.check(check);
}
if let Some(filters) = &value.filters {
let filters = liblzma::stream::Filters::try_from(filters)?;
mt.filters(filters);
}
if let Some(memlimit) = value.mem_limit_stop {
mt.memlimit_stop(memlimit);
}
if let Some(memlimit) = value.mem_limit_threading {
mt.memlimit_threading(memlimit);
}
if let Some(threads) = value.threads {
mt.threads(threads.get());
}
if let Some(timeout) = value.timeout_ms {
mt.timeout_ms(timeout);
}
Ok(mt)
}
}
@@ -0,0 +1,99 @@
use crate::{DecodeV2, DecodedSize, Xz2Decoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::{
convert::TryInto,
io::{Error, ErrorKind, Result},
};
/// Xz decoding stream
#[derive(Debug)]
pub struct XzDecoder {
inner: Xz2Decoder,
skip_padding: Option<u8>,
}
impl Default for XzDecoder {
fn default() -> Self {
Self {
inner: Xz2Decoder::new(usize::MAX.try_into().unwrap()),
skip_padding: None,
}
}
}
impl XzDecoder {
pub fn new() -> Self {
Self::default()
}
pub fn with_memlimit(memlimit: u64) -> Self {
Self {
inner: Xz2Decoder::new(memlimit),
skip_padding: None,
}
}
#[cfg(feature = "xz-parallel")]
pub fn parallel(threads: std::num::NonZeroU32, memlimit: u64) -> Self {
Self {
inner: Xz2Decoder::parallel(threads, memlimit),
skip_padding: None,
}
}
}
impl DecodeV2 for XzDecoder {
fn reinit(&mut self) -> Result<()> {
self.skip_padding = Some(4);
self.inner.reinit()
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
if let Some(ref mut count) = self.skip_padding {
while input.unwritten().first() == Some(&0) {
input.advance(1);
*count -= 1;
if *count == 0 {
*count = 4;
}
}
if input.unwritten().is_empty() {
return Ok(true);
}
// If this is non-padding then it cannot start with null bytes, so it must be invalid
// padding
if *count != 4 {
return Err(Error::new(
ErrorKind::InvalidData,
"stream padding was not a multiple of 4 bytes",
));
}
self.skip_padding = None;
}
self.inner.decode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
if self.skip_padding.is_some() {
return Ok(true);
}
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
if self.skip_padding.is_some() {
return Ok(true);
}
self.inner.finish(output)
}
}
impl DecodedSize for XzDecoder {
fn decoded_size(input: &[u8]) -> Result<u64> {
Xz2Decoder::decoded_size(input)
}
}
@@ -0,0 +1,45 @@
use crate::{EncodeV2, Xz2Encoder, Xz2FileFormat};
use compression_core::{
util::{PartialBuffer, WriteBuffer},
Level,
};
use std::io::Result;
/// Xz encoding stream
#[derive(Debug)]
pub struct XzEncoder {
inner: Xz2Encoder,
}
impl XzEncoder {
pub fn new(level: Level) -> Self {
Self {
inner: Xz2Encoder::new(Xz2FileFormat::Xz, level),
}
}
#[cfg(feature = "xz-parallel")]
pub fn parallel(threads: std::num::NonZeroU32, level: Level) -> Self {
Self {
inner: Xz2Encoder::xz_parallel(level, threads),
}
}
}
impl EncodeV2 for XzEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()> {
self.inner.encode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
+4
View File
@@ -0,0 +1,4 @@
mod decoder;
mod encoder;
pub use self::{decoder::XzDecoder, encoder::XzEncoder};
@@ -0,0 +1,108 @@
use crate::{lzma::params::LzmaDecoderParams, xz2::process_stream, DecodeV2, DecodedSize};
use compression_core::util::{PartialBuffer, WriteBuffer};
use liblzma::stream::{Action, Stream};
use std::{
convert::TryFrom,
fmt,
io::{self, Cursor},
};
/// Xz2 decoding stream
pub struct Xz2Decoder {
stream: Stream,
params: LzmaDecoderParams,
}
impl fmt::Debug for Xz2Decoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Xz2Decoder").finish_non_exhaustive()
}
}
impl TryFrom<LzmaDecoderParams> for Xz2Decoder {
type Error = liblzma::stream::Error;
fn try_from(params: LzmaDecoderParams) -> Result<Self, Self::Error> {
let stream = Stream::try_from(&params)?;
Ok(Self { stream, params })
}
}
impl Xz2Decoder {
pub fn new(mem_limit: u64) -> Self {
let params = LzmaDecoderParams::Auto {
mem_limit,
flags: 0,
};
Self::try_from(params).unwrap()
}
#[cfg(feature = "xz-parallel")]
pub fn parallel(threads: std::num::NonZeroU32, mem_limit: u64) -> Self {
use crate::lzma::params::MtStreamBuilder;
let mut builder = MtStreamBuilder::default();
builder
.threads(threads)
.timeout_ms(300)
.mem_limit_stop(mem_limit);
let params = LzmaDecoderParams::MultiThread { builder };
Self::try_from(params).unwrap()
}
}
impl DecodeV2 for Xz2Decoder {
fn reinit(&mut self) -> io::Result<()> {
*self = Self::try_from(self.params.clone())?;
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<bool> {
process_stream(&mut self.stream, input, output, Action::Run)
}
fn flush(&mut self, _output: &mut WriteBuffer<'_>) -> io::Result<bool> {
// While decoding flush is a noop
Ok(true)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
process_stream(
&mut self.stream,
&mut PartialBuffer::new(&[]),
output,
Action::Finish,
)
}
}
impl DecodedSize for Xz2Decoder {
fn decoded_size(input: &[u8]) -> io::Result<u64> {
let cursor = Cursor::new(input);
liblzma::uncompressed_size(cursor)
}
}
#[cfg(test)]
mod tests {
use std::convert::TryFrom;
use crate::{
lzma::params::{LzmaDecoderParams, LzmaFilter, LzmaFilters, LzmaOptions},
Xz2Decoder,
};
#[test]
fn test_lzma_decoder_from_params() {
let filters = LzmaFilters::default().add_filter(LzmaFilter::Lzma2(LzmaOptions::default()));
let params = LzmaDecoderParams::Raw { filters };
Xz2Decoder::try_from(params).unwrap();
}
}
@@ -0,0 +1,116 @@
use compression_core::{
util::{PartialBuffer, WriteBuffer},
Level,
};
use liblzma::stream::{Action, Check, Stream};
use std::{
convert::{TryFrom, TryInto},
fmt, io,
};
use crate::{
lzma::params::{LzmaEncoderParams, LzmaOptions},
xz2::process_stream,
EncodeV2, Xz2FileFormat,
};
/// Xz2 encoding stream
pub struct Xz2Encoder {
stream: Stream,
params: LzmaEncoderParams,
}
impl fmt::Debug for Xz2Encoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Xz2Encoder").finish_non_exhaustive()
}
}
impl TryFrom<LzmaEncoderParams> for Xz2Encoder {
type Error = liblzma::stream::Error;
fn try_from(params: LzmaEncoderParams) -> Result<Self, Self::Error> {
let stream = Stream::try_from(&params)?;
Ok(Self {
stream,
params: params.clone(),
})
}
}
fn xz2_level(level: Level) -> u32 {
match level {
Level::Fastest => 0,
Level::Best => 9,
Level::Precise(quality) => quality.try_into().unwrap_or(0).clamp(0, 9),
_ => 5,
}
}
impl Xz2Encoder {
pub fn new(format: Xz2FileFormat, level: Level) -> Self {
let preset = xz2_level(level);
let params = match format {
Xz2FileFormat::Xz => LzmaEncoderParams::Easy {
preset,
check: Check::Crc64,
},
Xz2FileFormat::Lzma => {
let options = LzmaOptions::default().preset(preset);
LzmaEncoderParams::Lzma { options }
}
};
Self::try_from(params).unwrap()
}
#[cfg(feature = "xz-parallel")]
pub fn xz_parallel(level: Level, threads: std::num::NonZeroU32) -> Self {
use crate::lzma::params::MtStreamBuilder;
let preset = xz2_level(level);
let mut builder = MtStreamBuilder::default();
builder
.threads(threads)
.timeout_ms(300)
.preset(preset)
.check(Check::Crc64);
let params = LzmaEncoderParams::MultiThread { builder };
Self::try_from(params).unwrap()
}
}
impl EncodeV2 for Xz2Encoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<()> {
process_stream(&mut self.stream, input, output, Action::Run).map(|_| ())
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
let action = match &self.params {
// Multi-threaded streams don't support SyncFlush, use FullFlush instead
#[cfg(feature = "xz-parallel")]
LzmaEncoderParams::MultiThread { builder: _ } => Action::FullFlush,
_ => Action::SyncFlush,
};
process_stream(
&mut self.stream,
&mut PartialBuffer::new(&[]),
output,
action,
)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
process_stream(
&mut self.stream,
&mut PartialBuffer::new(&[]),
output,
Action::Finish,
)
}
}
+40
View File
@@ -0,0 +1,40 @@
mod decoder;
mod encoder;
#[derive(Debug)]
pub enum Xz2FileFormat {
Xz,
Lzma,
}
pub use self::{decoder::Xz2Decoder, encoder::Xz2Encoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use liblzma::stream::{Action, Status, Stream};
use std::io;
/// Return `Ok(true)` if stream ends.
fn process_stream(
stream: &mut Stream,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
action: Action,
) -> io::Result<bool> {
let previous_in = stream.total_in() as usize;
let previous_out = stream.total_out() as usize;
// Safety: We **trust** liblzma to not write uninitialized bytes into the buffer
let status =
stream.process_uninit(input.unwritten(), unsafe { output.unwritten_mut() }, action)?;
input.advance(stream.total_in() as usize - previous_in);
// Safety: We **trust** liblzma to write bytes into the buffer properly
unsafe { output.assume_init_and_advance(stream.total_out() as usize - previous_out) };
match status {
Status::Ok => Ok(false),
Status::StreamEnd => Ok(true),
Status::GetCheck => Err(io::Error::other("Unexpected lzma integrity check")),
Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()),
}
}
@@ -0,0 +1,43 @@
use crate::{DecodeV2, FlateDecoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::io::Result;
#[derive(Debug)]
pub struct ZlibDecoder {
inner: FlateDecoder,
}
impl Default for ZlibDecoder {
fn default() -> Self {
Self {
inner: FlateDecoder::new(true),
}
}
}
impl ZlibDecoder {
pub fn new() -> Self {
Self::default()
}
}
impl DecodeV2 for ZlibDecoder {
fn reinit(&mut self) -> Result<()> {
self.inner.reinit()?;
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
self.inner.decode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
@@ -0,0 +1,38 @@
use crate::{flate::params::FlateEncoderParams, EncodeV2, FlateEncoder};
use compression_core::util::{PartialBuffer, WriteBuffer};
use std::io::Result;
#[derive(Debug)]
pub struct ZlibEncoder {
inner: FlateEncoder,
}
impl ZlibEncoder {
pub fn new(level: FlateEncoderParams) -> Self {
Self {
inner: FlateEncoder::new(level, true),
}
}
pub fn get_ref(&self) -> &FlateEncoder {
&self.inner
}
}
impl EncodeV2 for ZlibEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()> {
self.inner.encode(input, output)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.inner.finish(output)
}
}
@@ -0,0 +1,4 @@
mod decoder;
mod encoder;
pub use self::{decoder::ZlibDecoder, encoder::ZlibEncoder};
@@ -0,0 +1,105 @@
use crate::{
zstd::{params::DParameter, OperationExt},
{DecodeV2, DecodedSize},
};
use compression_core::{
unshared::Unshared,
util::{PartialBuffer, WriteBuffer},
};
use libzstd::stream::raw::Decoder;
use std::{
convert::TryInto,
io::{self, Result},
};
use zstd_safe::get_error_name;
#[derive(Debug)]
pub struct ZstdDecoder {
decoder: Unshared<Decoder<'static>>,
stream_ended: bool,
}
impl Default for ZstdDecoder {
fn default() -> Self {
Self {
decoder: Unshared::new(Decoder::new().unwrap()),
stream_ended: false,
}
}
}
impl ZstdDecoder {
pub fn new() -> Self {
Self::default()
}
pub fn new_with_params(params: &[DParameter]) -> Self {
let mut decoder = Decoder::new().unwrap();
for param in params {
decoder.set_parameter(param.as_zstd()).unwrap();
}
Self {
decoder: Unshared::new(decoder),
stream_ended: false,
}
}
pub fn new_with_dict(dictionary: &[u8]) -> io::Result<Self> {
let decoder = Decoder::with_dictionary(dictionary)?;
Ok(Self {
decoder: Unshared::new(decoder),
stream_ended: false,
})
}
}
impl DecodeV2 for ZstdDecoder {
fn reinit(&mut self) -> Result<()> {
self.decoder.reinit()?;
self.stream_ended = false;
Ok(())
}
fn decode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<bool> {
let finished = self.decoder.run(input, output)?;
if finished {
self.stream_ended = true;
}
Ok(finished)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
// Note: stream_ended is not updated here because zstd's flush only flushes
// buffered output and doesn't indicate stream completion. Stream completion
// is detected in decode() when status.remaining == 0.
self.decoder.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.decoder.finish(output)?;
if self.stream_ended {
Ok(true)
} else {
Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"zstd stream did not finish",
))
}
}
}
impl DecodedSize for ZstdDecoder {
fn decoded_size(input: &[u8]) -> Result<u64> {
zstd_safe::find_frame_compressed_size(input)
.map_err(|error_code| io::Error::other(get_error_name(error_code)))
.and_then(|size| {
size.try_into()
.map_err(|_| io::Error::from(io::ErrorKind::FileTooLarge))
})
}
}
@@ -0,0 +1,59 @@
use crate::{
zstd::{params::CParameter, OperationExt},
EncodeV2,
};
use compression_core::{
unshared::Unshared,
util::{PartialBuffer, WriteBuffer},
};
use libzstd::stream::raw::Encoder;
use std::io::{self, Result};
#[derive(Debug)]
pub struct ZstdEncoder {
encoder: Unshared<Encoder<'static>>,
}
impl ZstdEncoder {
pub fn new(level: i32) -> Self {
Self {
encoder: Unshared::new(Encoder::new(level).unwrap()),
}
}
pub fn new_with_params(level: i32, params: &[CParameter]) -> Self {
let mut encoder = Encoder::new(level).unwrap();
for param in params {
encoder.set_parameter(param.as_zstd()).unwrap();
}
Self {
encoder: Unshared::new(encoder),
}
}
pub fn new_with_dict(level: i32, dictionary: &[u8]) -> io::Result<Self> {
let encoder = Encoder::with_dictionary(level, dictionary)?;
Ok(Self {
encoder: Unshared::new(encoder),
})
}
}
impl EncodeV2 for ZstdEncoder {
fn encode(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> Result<()> {
self.encoder.run(input, output)?;
Ok(())
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.encoder.flush(output)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
self.encoder.finish(output)
}
}
+97
View File
@@ -0,0 +1,97 @@
mod decoder;
mod encoder;
pub mod params;
pub use self::{decoder::ZstdDecoder, encoder::ZstdEncoder};
use compression_core::{
unshared::Unshared,
util::{PartialBuffer, WriteBuffer},
};
use libzstd::stream::raw::{InBuffer, Operation, OutBuffer, WriteBuf};
use std::io;
#[repr(transparent)]
struct WriteBufferWrapper<'a>(WriteBuffer<'a>);
unsafe impl WriteBuf for WriteBufferWrapper<'_> {
fn as_slice(&self) -> &[u8] {
self.0.written()
}
fn capacity(&self) -> usize {
self.0.capacity()
}
fn as_mut_ptr(&mut self) -> *mut u8 {
self.0.as_mut_ptr()
}
unsafe fn filled_until(&mut self, n: usize) {
self.0.set_written_and_initialized_len(n);
}
}
trait WriteBufExt {
fn get_out_buf(&mut self) -> OutBuffer<'_, WriteBufferWrapper<'_>>;
}
impl WriteBufExt for WriteBuffer<'_> {
fn get_out_buf(&mut self) -> OutBuffer<'_, WriteBufferWrapper<'_>> {
{
use std::mem::{align_of, size_of};
assert_eq!(
size_of::<WriteBuffer<'static>>(),
size_of::<WriteBufferWrapper<'static>>()
);
assert_eq!(
align_of::<WriteBuffer<'static>>(),
align_of::<WriteBufferWrapper<'static>>()
);
}
// Pass written_len to avoid overwriting existing data in buffer.
let written_len = self.written_len();
OutBuffer::around_pos(
unsafe { &mut *(self as *mut _ as *mut WriteBufferWrapper<'_>) },
written_len,
)
}
}
trait OperationExt {
fn reinit(&mut self) -> io::Result<()>;
/// Return `true` if finished.
fn run(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<bool>;
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool>;
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool>;
}
impl<C: Operation> OperationExt for Unshared<C> {
fn reinit(&mut self) -> io::Result<()> {
self.get_mut().reinit()
}
fn run(
&mut self,
input: &mut PartialBuffer<&[u8]>,
output: &mut WriteBuffer<'_>,
) -> io::Result<bool> {
let mut in_buf = InBuffer::around(input.unwritten());
let result = self.get_mut().run(&mut in_buf, &mut output.get_out_buf());
input.advance(in_buf.pos());
Ok(result? == 0)
}
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
Ok(self.get_mut().flush(&mut output.get_out_buf())? == 0)
}
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
Ok(self.get_mut().finish(&mut output.get_out_buf(), true)? == 0)
}
}
@@ -0,0 +1,154 @@
//! This module contains zstd-specific types for async-compression.
use compression_core::Level;
/// A compression parameter for zstd. This is a stable wrapper around zstd's own `CParameter`
/// type, to abstract over different versions of the zstd library.
///
/// See the [zstd documentation](https://facebook.github.io/zstd/zstd_manual.html) for more
/// information on these parameters.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct CParameter(libzstd::stream::raw::CParameter);
impl From<CParameter> for libzstd::stream::raw::CParameter {
fn from(value: CParameter) -> Self {
value.0
}
}
impl CParameter {
pub fn quality(level: Level) -> i32 {
let (fastest, best) = libzstd::compression_level_range().into_inner();
// NOTE: zstd's "fastest" level is -131072 which can create outputs larger than inputs.
// This library chooses a "fastest" level which has a more-or-less equivalent compression
// ratio to gzip's fastest mode. We still allow precise levels to go negative.
// See discussion in https://github.com/Nullus157/async-compression/issues/352
const OUR_FASTEST: i32 = 1;
match level {
Level::Fastest => OUR_FASTEST,
Level::Best => best,
Level::Precise(quality) => quality.clamp(fastest, best),
_ => libzstd::DEFAULT_COMPRESSION_LEVEL,
}
}
/// Window size in bytes (as a power of two)
pub fn window_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::WindowLog(value))
}
/// Size of the initial probe table in 4-byte entries (as a power of two)
pub fn hash_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::HashLog(value))
}
/// Size of the multi-probe table in 4-byte entries (as a power of two)
pub fn chain_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::ChainLog(value))
}
/// Number of search attempts (as a power of two)
pub fn search_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::SearchLog(value))
}
/// Minimum size of matches searched for
pub fn min_match(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::MinMatch(value))
}
/// Strategy-dependent length modifier
pub fn target_length(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::TargetLength(value))
}
/// Enable long-distance matching mode to look for and emit long-distance references.
///
/// This increases the default window size.
pub fn enable_long_distance_matching(value: bool) -> Self {
Self(libzstd::stream::raw::CParameter::EnableLongDistanceMatching(value))
}
/// Size of the long-distance matching table (as a power of two)
pub fn ldm_hash_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::LdmHashLog(value))
}
/// Minimum size of long-distance matches searched for
pub fn ldm_min_match(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::LdmMinMatch(value))
}
/// Size of each bucket in the LDM hash table for collision resolution (as a power of two)
pub fn ldm_bucket_size_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::LdmBucketSizeLog(value))
}
/// Frequency of using the LDM hash table (as a power of two)
pub fn ldm_hash_rate_log(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::LdmHashRateLog(value))
}
/// Emit the size of the content (default: true).
pub fn content_size_flag(value: bool) -> Self {
Self(libzstd::stream::raw::CParameter::ContentSizeFlag(value))
}
/// Emit a checksum (default: false).
pub fn checksum_flag(value: bool) -> Self {
Self(libzstd::stream::raw::CParameter::ChecksumFlag(value))
}
/// Emit a dictionary ID when using a custom dictionary (default: true).
pub fn dict_id_flag(value: bool) -> Self {
Self(libzstd::stream::raw::CParameter::DictIdFlag(value))
}
/// Number of threads to spawn.
///
/// If set to 0, compression functions will block; if set to 1 or more, compression will
/// run in background threads and `flush` pushes bytes through the compressor.
///
/// # Panics
///
/// This parameter requires feature `zstdmt` to be enabled, otherwise it will cause a panic
/// when used in `ZstdEncoder::with_quality_and_params()` calls.
//
// TODO: make this a normal feature guarded fn on next breaking release
#[cfg_attr(docsrs, doc(cfg(feature = "zstdmt")))]
pub fn nb_workers(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::NbWorkers(value))
}
/// Number of bytes given to each worker.
///
/// If set to 0, zstd selects a job size based on compression parameters.
pub fn job_size(value: u32) -> Self {
Self(libzstd::stream::raw::CParameter::JobSize(value))
}
pub(crate) fn as_zstd(&self) -> libzstd::stream::raw::CParameter {
self.0
}
}
/// A decompression parameter for zstd. This is a stable wrapper around zstd's own `DParameter`
/// type, to abstract over different versions of the zstd library.
///
/// See the [zstd documentation](https://facebook.github.io/zstd/zstd_manual.html) for more
/// information on these parameters.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct DParameter(libzstd::stream::raw::DParameter);
impl DParameter {
/// Maximum window size in bytes (as a power of two)
pub fn window_log_max(value: u32) -> Self {
Self(libzstd::stream::raw::DParameter::WindowLogMax(value))
}
pub(crate) fn as_zstd(&self) -> libzstd::stream::raw::DParameter {
self.0
}
}