mirrors/v
1
0
Fork 0
mirror of https://github.com/vlang/v.git synced 2025-09-13 14:32:26 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

zstd: make the api more V like

Browse source
This commit is contained in:
Alexander Medvednikov 2025-03-22 15:02:05 +03:00
parent 308c9a51dd
commit 3285b5a67c
2 changed files with 170 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

303
vlib/compress/zstd/zstd.c.v
View file

@ -1,6 +1,5 @@
// zstd(https://github.com/facebook/zstd) is a fast real-time compression algorithm developed by Facebook.
// zstd compression/decompression
module zstd
import os
@ -9,9 +8,9 @@ import encoding.binary
#flag -I @VMODROOT/thirdparty/zstd
#include "zstd.c" // msvc can't compile multiple source files, so included
const zstd_frame_header_size_max = 18
const zstd_content_size_unknown = u64(-1)
const zstd_content_size_error = u64(-2)
const frame_header_size_max = 18
const content_size_unknown = u64(-1)
const content_size_error = u64(-2)
const buf_in_size = 1024 * 1024
const buf_out_size = 1024 * 1024
@ -29,28 +28,28 @@ fn C.ZSTD_getErrorName(usize) charptr
fn C.ZSTD_minCLevel() int
fn C.ZSTD_maxCLevel() int
fn C.ZSTD_defaultCLevel() int
fn C.ZSTD_createCCtx() &ZSTD_CCtx
fn C.ZSTD_createCCtx() &C.ZSTD_CCtx
fn C.ZSTD_freeCCtx(voidptr) usize
fn C.ZSTD_compressCCtx(voidptr, voidptr, usize, voidptr, usize, int) usize
fn C.ZSTD_createDCtx() &ZSTD_DCtx
fn C.ZSTD_createDCtx() &C.ZSTD_DCtx
fn C.ZSTD_freeDCtx(voidptr) usize
fn C.ZSTD_decompressDCtx(voidptr, voidptr, usize, voidptr, usize) usize
pub enum ZSTD_strategy {
zstd_default = 0
zstd_fast = 1
zstd_dfast = 2
zstd_greedy = 3
zstd_lazy = 4
zstd_lazy2 = 5
zstd_btlazy2 = 6
zstd_btopt = 7
zstd_btultra = 8
zstd_btultra2 = 9
// note : new strategies _might_ be added in the future. Only the order (from fast to strong) is guaranteed
// note : new strategies _might_ be added in the future. Only the order (from fast to strong) is guaranteed
pub enum Strategy {
default = 0
fast = 1
dfast = 2
greedy = 3
lazy = 4
lazy2 = 5
btlazy2 = 6
btopt = 7
btultra = 8
btultra2 = 9
}
pub enum ZSTD_cParameter {
pub enum CParameter {
// compression parameters
// Note: When compressing with a ZSTD_CDict these parameters are superseded
// by the parameters used to construct the ZSTD_CDict.
@ -66,7 +65,7 @@ pub enum ZSTD_cParameter {
// to default. Setting this will however eventually dynamically impact the compression
// parameters which have not been manually set. The manually set
// ones will 'stick'.
zstd_c_compression_level = 100
compression_level = 100
// Advanced compression parameters :
// It's possible to pin down compression parameters to some specific values.
// In which case, these values are no longer dynamically selected by the compressor
@ -79,14 +78,14 @@ pub enum ZSTD_cParameter {
// Special: value 0 means "use default windowLog".
// Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
// requires explicitly allowing such size at streaming decompression stage.
zstd_c_window_log = 101
window_log = 101
// Size of the initial probe table, as a power of 2.
// Resulting memory usage is (1 << (hashLog+2)).
// Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
// Larger tables improve compression ratio of strategies <= dFast,
// and improve speed of strategies > dFast.
// Special: value 0 means "use default hashLog".
zstd_c_hash_log = 102
hash_log = 102
// Size of the multi-probe search table, as a power of 2.
// Resulting memory usage is (1 << (chainLog+2)).
// Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
@ -95,12 +94,12 @@ pub enum ZSTD_cParameter {
// It's still useful when using "dfast" strategy,
// in which case it defines a secondary probe table.
// Special: value 0 means "use default chainLog".
zstd_c_chain_log = 103
chain_log = 103
// Number of search attempts, as a power of 2.
// More attempts result in better and slower compression.
// This parameter is useless for "fast" and "dFast" strategies.
// Special: value 0 means "use default searchLog".
zstd_c_search_log = 104
search_log = 104
// Minimum size of searched matches.
// Note that Zstandard can still find matches of smaller size,
// it just tweaks its search algorithm to look for this size and larger.
@ -109,7 +108,7 @@ pub enum ZSTD_cParameter {
// Note that currently, for all strategies < btopt, effective minimum is 4.
// , for all strategies > fast, effective maximum is 6.
// Special: value 0 means "use default minMatchLength".
zstd_c_min_match = 105
min_match = 105
// Impact of this field depends on strategy.
// For strategies btopt, btultra & btultra2:
// Length of Match considered "good enough" to stop search.
@ -118,12 +117,12 @@ pub enum ZSTD_cParameter {
// Distance between match sampling.
// Larger values make compression faster, and weaker.
// Special: value 0 means "use default targetLength".
zstd_c_target_length = 106
target_length = 106
// See ZSTD_strategy enum definition.
// The higher the value of selected strategy, the more complex it is,
// resulting in stronger and slower compression.
// Special: value 0 means "use default strategy".
zstd_c_strategy = 107
strategy = 107
// LDM mode parameters
// Enable long distance matching.
// This parameter is designed to improve compression ratio
@ -133,41 +132,41 @@ pub enum ZSTD_cParameter {
// except when expressly set to a different value.
// Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
// compression strategy >= ZSTD_btopt (== compression level 16+)
zstd_c_enable_long_distance_matching = 160
enable_long_distance_matching = 160
// Size of the table for long distance matching, as a power of 2.
// Larger values increase memory usage and compression ratio,
// but decrease compression speed.
// Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
// default: windowlog - 7.
// Special: value 0 means "automatically determine hashlog".
zstd_c_ldm_hash_log = 161
ldm_hash_log = 161
// Minimum match size for long distance matcher.
// Larger/too small values usually decrease compression ratio.
// Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
// Special: value 0 means "use default value" (default: 64).
zstd_c_ldm_min_match = 162
ldm_min_match = 162
// log size of each bucket in the ldm hash table for collision resolution.
// Larger values improve collision resolution but decrease compression speed.
// The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
// Special: value 0 means "use default value" (default: 3).
zstd_c_ldm_bucket_size_log = 163
ldm_bucket_size_log = 163
// Frequency of inserting/looking up entries into the LDM hash table.
// Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
// Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
// Larger values improve compression speed.
// Deviating far from default value will likely result in a compression ratio decrease.
// Special: value 0 means "automatically determine hashRateLog".
zstd_c_ldm_hash_rate_log = 164
ldm_hash_rate_log = 164
// frame parameters
// Content size will be written into frame header _whenever known_ (default:1)
// Content size must be known at the beginning of compression.
// This is automatically the case when using ZSTD_compress2(),
// For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize()
zstd_c_content_size_flag = 200
content_size_flag = 200
// A 32-bits checksum of content is written at end of frame (default:0)
zstd_c_checksum_flag = 201
checksum_flag = 201
// When applicable, dictionary's ID is written into frame header (default:1)
zstd_c_dict_id_flag = 202
dict_id_flag = 202
// multi-threading parameters
// These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
// Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
@ -183,13 +182,13 @@ pub enum ZSTD_cParameter {
// More workers improve speed, but also increase memory usage.
// Default value is `0`, aka "single-threaded mode" : no worker is spawned,
// compression is performed inside Caller's thread, and all invocations are blocking
zstd_c_nb_workers = 400
nb_workers = 400
// Size of a compression job. This value is enforced only when nbWorkers >= 1.
// Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
// 0 means default, which is dynamically determined based on compression parameters.
// Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
// The minimum size is automatically and transparently enforced.
zstd_c_job_size = 401
job_size = 401
// Control the overlap size, as a fraction of window size.
// The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
// It helps preserve compression ratio, while each job is compressed in parallel.
@ -202,7 +201,7 @@ pub enum ZSTD_cParameter {
// Each intermediate rank increases/decreases load size by a factor 2 :
// 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
// default value varies between 6 and 9, depending on strategy
zstd_c_overlap_log = 402
overlap_log = 402
// note : additional experimental parameters are also available
// within the experimental section of the API.
// At the time of this writing, they include :
@ -227,54 +226,55 @@ pub enum ZSTD_cParameter {
// note : never ever use experimentalParam? names directly;
// also, the enums values themselves are unstable and can still change.
//
zstd_c_experimental_param1 = 500
zstd_c_experimental_param2 = 10
zstd_c_experimental_param3 = 1000
zstd_c_experimental_param4 = 1001
zstd_c_experimental_param5 = 1002
zstd_c_experimental_param6 = 1003
zstd_c_experimental_param7 = 1004
zstd_c_experimental_param8 = 1005
zstd_c_experimental_param9 = 1006
zstd_c_experimental_param10 = 1007
zstd_c_experimental_param11 = 1008
zstd_c_experimental_param12 = 1009
zstd_c_experimental_param13 = 1010
zstd_c_experimental_param14 = 1011
zstd_c_experimental_param15 = 1012
zstd_c_experimental_param16 = 1013
zstd_c_experimental_param17 = 1014
zstd_c_experimental_param18 = 1015
zstd_c_experimental_param19 = 1016
experimental_param1 = 500
experimental_param2 = 10
experimental_param3 = 1000
experimental_param4 = 1001
experimental_param5 = 1002
experimental_param6 = 1003
experimental_param7 = 1004
experimental_param8 = 1005
experimental_param9 = 1006
experimental_param10 = 1007
experimental_param11 = 1008
experimental_param12 = 1009
experimental_param13 = 1010
experimental_param14 = 1011
experimental_param15 = 1012
experimental_param16 = 1013
experimental_param17 = 1014
experimental_param18 = 1015
experimental_param19 = 1016
}
pub struct ZSTD_bounds {
pub struct Bounds {
pub:
error usize
lower_bound int
upper_bound int
}
fn C.ZSTD_cParam_getBounds(ZSTD_cParameter) ZSTD_bounds
fn C.ZSTD_CCtx_setParameter(voidptr, ZSTD_cParameter, int) usize
fn C.ZSTD_cParam_getBounds(CParameter) Bounds
fn C.ZSTD_CCtx_setParameter(voidptr, CParameter, int) usize
fn C.ZSTD_CCtx_setPledgedSrcSize(voidptr, u64) usize
pub enum ZSTD_ResetDirective {
pub enum ResetDirective {
zstd_reset_session_only = 1
zstd_reset_parameters = 2
zstd_reset_session_and_parameters = 3
}
fn C.ZSTD_CCtx_reset(voidptr, ZSTD_ResetDirective) usize
fn C.ZSTD_CCtx_reset(voidptr, ResetDirective) usize
fn C.ZSTD_compress2(voidptr, voidptr, usize, voidptr, usize) usize
pub enum ZSTD_dParameter {
pub enum DParameter {
// Select a size limit (in power of 2) beyond which
// the streaming API will refuse to allocate memory buffer
// in order to protect the host from unreasonable memory requirements.
// This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
// By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
// Special: value 0 means "use default maximum windowLog".
zstd_d_window_log_max = 100
window_log_max = 100
// note : additional experimental parameters are also available
// within the experimental section of the API.
// At the time of this writing, they include :
@ -285,26 +285,26 @@ pub enum ZSTD_dParameter {
// zstd_d_disable_huffman_assembly
// Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
// note : never ever use experimentalParam? names directly
zstd_d_experimental_param1 = 1000
zstd_d_experimental_param2 = 1001
zstd_d_experimental_param3 = 1002
zstd_d_experimental_param4 = 1003
zstd_d_experimental_param5 = 1004
experimental_param1 = 1000
experimental_param2 = 1001
experimental_param3 = 1002
experimental_param4 = 1003
experimental_param5 = 1004
}
fn C.ZSTD_dParam_getBounds(ZSTD_dParameter) ZSTD_bounds
fn C.ZSTD_DCtx_setParameter(voidptr, ZSTD_dParameter, int) usize
fn C.ZSTD_DCtx_reset(voidptr, ZSTD_ResetDirective) usize
fn C.ZSTD_dParam_getBounds(DParameter) Bounds
fn C.ZSTD_DCtx_setParameter(voidptr, DParameter, int) usize
fn C.ZSTD_DCtx_reset(voidptr, ResetDirective) usize
// streaming compression
pub struct ZSTD_inBuffer {
pub struct InBuffer {
pub mut:
src voidptr
size usize
pos usize
}
pub struct ZSTD_outBuffer {
pub struct OutBuffer {
pub mut:
dst voidptr
size usize
@ -314,35 +314,35 @@ pub mut:
fn C.ZSTD_createCStream() voidptr
fn C.ZSTD_freeCStream(voidptr) usize
pub enum ZSTD_EndDirective {
pub enum EndDirective {
// collect more data, encoder decides when to output compressed result, for optimal compression ratio
zstd_e_continue = 0
continue = 0
// flush any data provided so far,
// it creates (at least) one new block, that can be decoded immediately on reception;
// frame will continue: any future data can still reference previously compressed data, improving compression.
// note : multithreaded compression will block to flush as much output as possible.
zstd_e_flush = 1
flush = 1
// flush any remaining data _and_ close current frame.
// note that frame is only closed after compressed data is fully flushed (return value == 0).
// After that point, any additional data starts a new frame.
// note : each frame is independent (does not reference any content from previous frame).
// note : multithreaded compression will block to flush as much output as possible.
zstd_e_end = 2
end = 2
}
fn C.ZSTD_compressStream2(voidptr, &ZSTD_outBuffer, &ZSTD_inBuffer, ZSTD_EndDirective) usize
fn C.ZSTD_compressStream2(voidptr, &OutBuffer, &InBuffer, EndDirective) usize
fn C.ZSTD_CStreamInSize() usize
fn C.ZSTD_CStreamOutSize() usize
fn C.ZSTD_initCStream(voidptr, int) usize
fn C.ZSTD_compressStream(voidptr, &ZSTD_outBuffer, &ZSTD_inBuffer) usize
fn C.ZSTD_flushStream(voidptr, &ZSTD_outBuffer) usize
fn C.ZSTD_endStream(voidptr, &ZSTD_outBuffer) usize
fn C.ZSTD_compressStream(voidptr, &OutBuffer, &InBuffer) usize
fn C.ZSTD_flushStream(voidptr, &OutBuffer) usize
fn C.ZSTD_endStream(voidptr, &OutBuffer) usize
// streaming decompression
fn C.ZSTD_createDStream() voidptr
fn C.ZSTD_freeDStream(voidptr) usize
fn C.ZSTD_initDStream(voidptr) usize
fn C.ZSTD_decompressStream(voidptr, &ZSTD_outBuffer, &ZSTD_inBuffer) usize
fn C.ZSTD_decompressStream(voidptr, &OutBuffer, &InBuffer) usize
fn C.ZSTD_DStreamInSize() usize
fn C.ZSTD_DStreamOutSize() usize
@ -367,7 +367,7 @@ pub fn get_error_name(code usize) string {
}
// check_zstd check the zstd error code, and return a error string.
pub fn check_zstd(code usize) ! {
pub fn check_error(code usize) ! {
if is_error(code) {
return error(get_error_name(code))
}
@ -391,10 +391,12 @@ pub fn default_c_level() int {
@[params]
pub struct CompressParams {
pub:
compression_level int // 1~22
nb_threads int = 1 // how many threads will be spawned to compress in parallel
// 1~22
compression_level int = default_c_level()
// how many threads will be spawned to compress in parallel
nb_threads int = 1
checksum_flag bool = true
strategy ZSTD_strategy = ZSTD_strategy.zstd_default
strategy Strategy = .default
}
// compresses an array of bytes using zstd and returns the compressed bytes in a new array
@ -402,23 +404,15 @@ pub:
// Example: compressed := zstd.compress(b)!
pub fn compress(data []u8, params CompressParams) ![]u8 {
dst_capacity := C.ZSTD_compressBound(data.len)
check_zstd(dst_capacity)!
check_error(dst_capacity)!
mut dst := []u8{len: int(dst_capacity)}
mut cctx := new_cctx()!
mut cctx := new_cctx(params)!
defer {
cctx.free_cctx()
}
check_zstd(cctx.set_parameter(.zstd_c_compression_level, params.compression_level))!
$if !(tinyc && windows) {
// TODO: tinyc on windows doesn't support multiple thread
check_zstd(cctx.set_parameter(.zstd_c_nb_workers, params.nb_threads))!
}
check_zstd(cctx.set_parameter(.zstd_c_checksum_flag, if params.checksum_flag { 1 } else { 0 }))!
check_zstd(cctx.set_parameter(.zstd_c_strategy, int(params.strategy)))!
compressed_size := C.ZSTD_compress2(cctx, dst.data, dst.len, data.data, data.len)
check_zstd(compressed_size)!
return dst[..compressed_size]
size := C.ZSTD_compress2(cctx.ctx, dst.data, dst.len, data.data, data.len)
check_error(size)!
return dst[..size]
}
@[params]
@ -431,45 +425,46 @@ pub:
// extra decompression parameters can be set by `params`
// Example: decompressed := zstd.decompress(b)!
pub fn decompress(data []u8, params DecompressParams) ![]u8 {
dst_capacity := C.ZSTD_getFrameContentSize(data.data, zstd_frame_header_size_max)
if dst_capacity == zstd_content_size_unknown {
dst_capacity := C.ZSTD_getFrameContentSize(data.data, frame_header_size_max)
if dst_capacity == content_size_unknown {
return error('The size cannot be determined, try use streaming mode to decompress data?')
} else if dst_capacity == zstd_content_size_error {
} else if dst_capacity == content_size_error {
return error('An error occurred (e.g. invalid magic number, srcSize too small)')
} else if dst_capacity == 0 {
return error('The frame is valid but empty')
}
mut dst := []u8{len: int(dst_capacity)}
decompressed_size := C.ZSTD_decompress(dst.data, dst.len, data.data, data.len)
check_zstd(decompressed_size)!
check_error(decompressed_size)!
return dst[..decompressed_size]
}
struct C.ZSTD_CCtx {}
// ZSTD_CCtx zstd compression context struct
pub type ZSTD_CCtx = C.ZSTD_CCtx
pub struct CCtx {
mut:
ctx &C.ZSTD_CCtx
}
// new_cctx create a compression context
// extra compression parameters can be set by `params`
pub fn new_cctx(params CompressParams) !&ZSTD_CCtx {
mut cctx := C.ZSTD_createCCtx()
if isnil(cctx) {
pub fn new_cctx(params CompressParams) !&CCtx {
mut ctx := C.ZSTD_createCCtx()
if isnil(ctx) {
return error('new_cctx() failed!')
}
check_zstd(cctx.set_parameter(.zstd_c_compression_level, params.compression_level))!
mut cctx := &CCtx{ctx}
cctx.set(.compression_level, params.compression_level)!
$if !(tinyc && windows) {
// TODO: tinyc on windows doesn't support multiple thread
check_zstd(cctx.set_parameter(.zstd_c_nb_workers, params.nb_threads))!
cctx.set(.nb_workers, params.nb_threads)!
}
check_zstd(cctx.set_parameter(.zstd_c_checksum_flag, if params.checksum_flag { 1 } else { 0 }))!
check_zstd(cctx.set_parameter(.zstd_c_strategy, int(params.strategy)))!
cctx.set(.checksum_flag, if params.checksum_flag { 1 } else { 0 })!
cctx.set(.strategy, int(params.strategy))!
return cctx
}
// set_parameter set compression parameter `c_param` to value `val`
pub fn (mut u ZSTD_CCtx) set_parameter(c_param ZSTD_cParameter, val int) usize {
return C.ZSTD_CCtx_setParameter(&u, c_param, val)
pub fn (mut c CCtx) set(c_param CParameter, val int) ! {
check_error(C.ZSTD_CCtx_setParameter(c.ctx, c_param, val))!
}
// compress_stream2 do stream compress on `input`, and store compressed data in `output`.
@ -477,45 +472,54 @@ pub fn (mut u ZSTD_CCtx) set_parameter(c_param ZSTD_cParameter, val int) usize {
// .zstd_e_continue => continue stream compression.
// .zstd_e_flush => flush data
// .zstd_e_end => it is the last frame
pub fn (mut u ZSTD_CCtx) compress_stream2(output &ZSTD_outBuffer, input &ZSTD_inBuffer, mode ZSTD_EndDirective) usize {
return C.ZSTD_compressStream2(&u, output, input, mode)
pub fn (mut c CCtx) compress_stream2(output &OutBuffer, input &InBuffer, mode EndDirective) !usize {
res := C.ZSTD_compressStream2(c.ctx, output, input, mode)
check_error(res)!
return res
}
// free_cctx free a compression context
pub fn (mut u ZSTD_CCtx) free_cctx() usize {
return C.ZSTD_freeCCtx(&u)
pub fn (mut c CCtx) free_cctx() usize {
return C.ZSTD_freeCCtx(c.ctx)
}
struct C.ZSTD_CCtx {}
struct C.ZSTD_DCtx {}
// ZSTD_DCtx zstd decompression context struct
pub type ZSTD_DCtx = C.ZSTD_DCtx
pub struct DCtx {
mut:
ctx &C.ZSTD_DCtx
}
// new_dctx create a decompression context
// extra decompression parameters can be set by `params`
pub fn new_dctx(params DecompressParams) !&ZSTD_DCtx {
mut dctx := C.ZSTD_createDCtx()
if isnil(dctx) {
pub fn new_dctx(params DecompressParams) !&DCtx {
mut ctx := C.ZSTD_createDCtx()
if isnil(ctx) {
return error('new_dctx() failed!')
}
check_zstd(dctx.set_parameter(.zstd_d_window_log_max, params.window_log_max))!
mut dctx := &DCtx{ctx}
dctx.set(.window_log_max, params.window_log_max)!
return dctx
}
// set_parameter set decompression parameter `d_param` to value `val`
pub fn (mut u ZSTD_DCtx) set_parameter(d_param ZSTD_dParameter, val int) usize {
return C.ZSTD_DCtx_setParameter(&u, d_param, val)
pub fn (mut d DCtx) set(d_param DParameter, val int) ! {
check_error(C.ZSTD_DCtx_setParameter(d.ctx, d_param, val))!
}
// decompress_stream do stream decompress on `input`, and store decompressed data in `output`.
// return remaining bytes in `input` stream
pub fn (mut u ZSTD_DCtx) decompress_stream(output &ZSTD_outBuffer, input &ZSTD_inBuffer) usize {
return C.ZSTD_decompressStream(&u, output, input)
pub fn (mut d DCtx) decompress_stream(output &OutBuffer, input &InBuffer) !usize {
res := C.ZSTD_decompressStream(d.ctx, output, input)
check_error(res)!
return res
}
// free_cctx free a compression context
pub fn (mut u ZSTD_DCtx) free_dctx() usize {
return C.ZSTD_freeDCtx(&u)
pub fn (mut d DCtx) free_dctx() usize {
return C.ZSTD_freeDCtx(d.ctx)
}
// store_array compress an `array`'s data, and store it to file `fname`.
@ -544,9 +548,8 @@ pub fn store_array[T](fname string, array []T, params CompressParams) ! {
}
mut buf_out := []u8{len: buf_out_size}
mut input := &ZSTD_inBuffer{}
mut output := &ZSTD_outBuffer{}
mut remaining := usize(1)
mut input := &InBuffer{}
mut output := &OutBuffer{}
// first, write the array.len to file
mut len_buf := []u8{len: 8}
binary.little_endian_put_u64(mut len_buf, u64(array.len))
@ -556,8 +559,7 @@ pub fn store_array[T](fname string, array []T, params CompressParams) ! {
output.dst = buf_out.data
output.size = buf_out_size
output.pos = 0
remaining = cctx.compress_stream2(output, input, .zstd_e_flush)
check_zstd(remaining)!
mut remaining := cctx.compress_stream2(output, input, .flush)!
fout.write(buf_out[..output.pos])!
// then, write the array.data to file
input.src = array.data
@ -566,14 +568,15 @@ pub fn store_array[T](fname string, array []T, params CompressParams) ! {
output.dst = buf_out.data
output.size = buf_out_size
output.pos = 0
remaining = 1
for remaining != 0 {
for {
output.dst = buf_out.data
output.size = buf_out_size
output.pos = 0
remaining = cctx.compress_stream2(output, input, .zstd_e_end)
check_zstd(remaining)!
remaining = cctx.compress_stream2(output, input, .end)!
fout.write(buf_out[..output.pos])!
if remaining == 0 {
break
}
}
}
@ -589,10 +592,9 @@ pub fn load_array[T](fname string, params DecompressParams) ![]T {
mut buf_in := []u8{len: buf_in_size}
mut len_buf := []u8{len: 8}
mut input := &ZSTD_inBuffer{}
mut output := &ZSTD_outBuffer{}
mut input := &InBuffer{}
mut output := &OutBuffer{}
mut last_ret := usize(0)
mut ret := usize(0)
mut last_chunk := false
// first, read the array.len from file
mut read_len := fin.read(mut buf_in)!
@ -603,26 +605,21 @@ pub fn load_array[T](fname string, params DecompressParams) ![]T {
output.dst = len_buf.data
output.size = usize(len_buf.len)
output.pos = 0
ret = dctx.decompress_stream(output, input)
check_zstd(ret)!
last_ret = dctx.decompress_stream(output, input)!
len := binary.little_endian_u64(len_buf)
// then, read the array.data from file
mut result := []T{len: int(len)}
output.dst = result.data
output.size = usize(result.len) * sizeof(T)
output.pos = 0
ret = dctx.decompress_stream(output, input)
last_ret = ret
last_ret = dctx.decompress_stream(output, input)!
for !last_chunk {
read_len = fin.read(mut buf_in)!
last_chunk = read_len < buf_in.len
// input.src = buf_in.data
input.size = usize(read_len)
input.pos = 0
for input.pos < input.size {
ret = dctx.decompress_stream(output, input)
check_zstd(ret)!
last_ret = ret
last_ret = dctx.decompress_stream(output, input)!
}
if read_len < buf_in.len {
break

40
vlib/compress/zstd/zstd_test.v
View file

@ -68,43 +68,43 @@ fn test_zstd_checksum_flag() {
fn test_zstd_deferent_strategy() {
uncompressed := 'Hello world!'.repeat(10000)
compressed_default := compress(uncompressed.bytes(), strategy: .zstd_default)!
compressed_default := compress(uncompressed.bytes(), strategy: .default)!
decompressed_default := decompress(compressed_default)!
assert decompressed_default == uncompressed.bytes()
compressed_fast := compress(uncompressed.bytes(), strategy: .zstd_fast)!
compressed_fast := compress(uncompressed.bytes(), strategy: .fast)!
decompressed_fast := decompress(compressed_fast)!
assert decompressed_fast == uncompressed.bytes()
compressed_dfast := compress(uncompressed.bytes(), strategy: .zstd_dfast)!
compressed_dfast := compress(uncompressed.bytes(), strategy: .dfast)!
decompressed_dfast := decompress(compressed_dfast)!
assert decompressed_dfast == uncompressed.bytes()
compressed_greedy := compress(uncompressed.bytes(), strategy: .zstd_greedy)!
compressed_greedy := compress(uncompressed.bytes(), strategy: .greedy)!
decompressed_greedy := decompress(compressed_greedy)!
assert decompressed_greedy == uncompressed.bytes()
compressed_lazy := compress(uncompressed.bytes(), strategy: .zstd_lazy)!
compressed_lazy := compress(uncompressed.bytes(), strategy: .lazy)!
decompressed_lazy := decompress(compressed_lazy)!
assert decompressed_lazy == uncompressed.bytes()
compressed_lazy2 := compress(uncompressed.bytes(), strategy: .zstd_lazy2)!
compressed_lazy2 := compress(uncompressed.bytes(), strategy: .lazy2)!
decompressed_lazy2 := decompress(compressed_lazy2)!
assert decompressed_lazy2 == uncompressed.bytes()
compressed_btlazy2 := compress(uncompressed.bytes(), strategy: .zstd_btlazy2)!
compressed_btlazy2 := compress(uncompressed.bytes(), strategy: .btlazy2)!
decompressed_btlazy2 := decompress(compressed_btlazy2)!
assert decompressed_btlazy2 == uncompressed.bytes()
compressed_btopt := compress(uncompressed.bytes(), strategy: .zstd_btopt)!
compressed_btopt := compress(uncompressed.bytes(), strategy: .btopt)!
decompressed_btopt := decompress(compressed_btopt)!
assert decompressed_btopt == uncompressed.bytes()
compressed_btultra := compress(uncompressed.bytes(), strategy: .zstd_btultra)!
compressed_btultra := compress(uncompressed.bytes(), strategy: .btultra)!
decompressed_btultra := decompress(compressed_btultra)!
assert decompressed_btultra == uncompressed.bytes()
compressed_btultra2 := compress(uncompressed.bytes(), strategy: .zstd_btultra2)!
compressed_btultra2 := compress(uncompressed.bytes(), strategy: .btultra2)!
decompressed_btultra2 := decompress(compressed_btultra2)!
assert decompressed_btultra2 == uncompressed.bytes()
}
@ -126,12 +126,12 @@ fn compress_file(fname string, oname string, params CompressParams) ! {
}
mut last_chunk := false
mut input := &ZSTD_inBuffer{
mut input := &InBuffer{
src: buf_in.data
size: 0
pos: 0
}
mut output := &ZSTD_outBuffer{
mut output := &OutBuffer{
dst: buf_out.data
size: 0
pos: 0
@ -140,9 +140,9 @@ fn compress_file(fname string, oname string, params CompressParams) ! {
read_len := fin.read(mut buf_in)!
last_chunk = read_len < buf_in_size
mode := if last_chunk {
ZSTD_EndDirective.zstd_e_end
EndDirective.end
} else {
ZSTD_EndDirective.zstd_e_continue
EndDirective.continue
}
mut finished := false
@ -153,8 +153,7 @@ fn compress_file(fname string, oname string, params CompressParams) ! {
output.dst = buf_out.data
output.size = buf_out_size
output.pos = 0
remaining := cctx.compress_stream2(output, input, mode)
check_zstd(remaining)!
remaining := cctx.compress_stream2(output, input, mode)!
fout.write(buf_out[..output.pos])!
finished = if last_chunk { remaining == 0 } else { input.pos == input.size }
}
@ -181,12 +180,12 @@ fn decompress_file(fname string, oname string, params DecompressParams) ! {
dctx.free_dctx()
}
mut input := &ZSTD_inBuffer{
mut input := &InBuffer{
src: buf_in.data
size: 0
pos: 0
}
mut output := &ZSTD_outBuffer{
mut output := &OutBuffer{
dst: buf_out.data
size: 0
pos: 0
@ -202,8 +201,7 @@ fn decompress_file(fname string, oname string, params DecompressParams) ! {
output.dst = buf_out.data
output.size = buf_out_size
output.pos = 0
ret := dctx.decompress_stream(output, input)
check_zstd(ret)!
ret := dctx.decompress_stream(output, input)!
fout.write(buf_out[..output.pos])!
last_ret = ret
}
@ -212,7 +210,7 @@ fn decompress_file(fname string, oname string, params DecompressParams) ! {
}
}
if last_ret != 0 {
/* The last return value from ZSTD_decompressStream did not end on a
/* The last return value from DecompressStream did not end on a
* frame, but we reached the end of the file! We assume this is an
* error, and the input was truncated.
*/