vlib: add a compress.szip module, deprecate the szip one after 2023-12-31 (#20003)

vlib/compress/szip/README.md

@@ -0,0 +1,6 @@
+## Description:
+
+`szip` is a thin wrapper around [miniz.h](https://github.com/richgel999/miniz),
+which in turn is "Single C source file zlib-replacement library,
+originally from code.google.com/p/miniz".
+It provides utility functions for reading/writing .zip files.

vlib/compress/szip/szip.c.v

@@ -0,0 +1,314 @@
+module szip
+
+import os
+
+#flag -I @VEXEROOT/thirdparty/zip
+#include "zip.c"
+
+@[params]
+pub struct ZipFolderOptions {
+	omit_empty_folders bool
+}
+
+pub struct C.zip_t {
+}
+
+type Zip = C.zip_t
+
+pub type Fn_on_extract_entry = fn (&&char, &&char) int
+
+fn C.zip_open(&char, int, char) &Zip
+
+fn C.zip_close(&Zip)
+
+fn C.zip_entry_open(&Zip, &u8) int
+
+fn C.zip_entry_openbyindex(&Zip, usize) int
+
+fn C.zip_entry_close(&Zip) int
+
+fn C.zip_entry_name(&Zip) &u8
+
+fn C.zip_entry_index(&Zip) int
+
+fn C.zip_entry_isdir(&Zip) int
+
+fn C.zip_entry_size(&Zip) u64
+
+fn C.zip_entry_crc32(&Zip) u32
+
+fn C.zip_entry_write(&Zip, voidptr, usize) int
+
+fn C.zip_entry_fwrite(&Zip, &char) int
+
+fn C.zip_entry_read(&Zip, &voidptr, &usize) int
+
+fn C.zip_entry_noallocread(&Zip, voidptr, usize) int
+
+fn C.zip_entry_fread(&Zip, &char) int
+
+fn C.zip_entries_total(&Zip) int
+
+fn C.zip_extract(&char, &char, Fn_on_extract_entry, voidptr) int
+
+fn cb_zip_extract(filename &&char, arg &&char) int {
+	return 0
+}
+
+pub enum CompressionLevel {
+	no_compression      = C.MZ_NO_COMPRESSION
+	best_speed          = C.MZ_BEST_SPEED
+	best_compression    = C.MZ_BEST_COMPRESSION
+	uber_compression    = C.MZ_UBER_COMPRESSION
+	default_level       = C.MZ_DEFAULT_LEVEL
+	default_compression = C.MZ_DEFAULT_COMPRESSION
+}
+
+// OpenMode lists the opening modes
+// .write: opens a file for reading/extracting (the file must exists).
+// .read_only: creates an empty file for writing.
+// .append: appends to an existing archive.
+pub enum OpenMode {
+	write
+	read_only
+	append
+}
+
+@[inline]
+fn (om OpenMode) to_u8() u8 {
+	return match om {
+		.write {
+			`w`
+		}
+		.read_only {
+			`r`
+		}
+		.append {
+			`a`
+		}
+	}
+}
+
+// open opens zip archive with compression level using the given mode.
+// name: the name of the zip file to open.
+// level: can be any value of the CompressionLevel enum.
+// mode: can be any value of the OpenMode enum.
+pub fn open(name string, level CompressionLevel, mode OpenMode) !&Zip {
+	if name.len == 0 {
+		return error('szip: name of file empty')
+	}
+	p_zip := unsafe { &Zip(C.zip_open(&char(name.str), int(level), char(mode.to_u8()))) }
+	if isnil(p_zip) {
+		return error('szip: cannot open/create/append new zip archive')
+	}
+	return p_zip
+}
+
+// close closes the zip archive, releases resources - always finalize.
+@[inline]
+pub fn (mut z Zip) close() {
+	C.zip_close(z)
+}
+
+// open_entry opens an entry by name in the zip archive.
+// For zip archive opened in 'w' or 'a' mode the function will append
+// a new entry. In readonly mode the function tries to locate the entry
+// in global dictionary.
+pub fn (mut zentry Zip) open_entry(name string) ! {
+	res := C.zip_entry_open(zentry, &char(name.str))
+	if res == -1 {
+		return error('szip: cannot open archive entry')
+	}
+}
+
+// open_entry_by_index opens an entry by index in the archive.
+pub fn (mut z Zip) open_entry_by_index(index int) ! {
+	res := C.zip_entry_openbyindex(z, index)
+	if res == -1 {
+		return error('szip: cannot open archive entry at index ${index}')
+	}
+}
+
+// close_entry closes a zip entry, flushes buffer and releases resources.
+@[inline]
+pub fn (mut zentry Zip) close_entry() {
+	C.zip_entry_close(zentry)
+}
+
+// name returns a local name of the current zip entry.
+// The main difference between user's entry name and local entry name
+// is optional relative path.
+// Following .ZIP File Format Specification - the path stored MUST not contain
+// a drive or device letter, or a leading slash.
+// All slashes MUST be forward slashes '/' as opposed to backwards slashes '\'
+// for compatibility with Amiga and UNIX file systems etc.
+pub fn (mut zentry Zip) name() string {
+	name := unsafe { &u8(C.zip_entry_name(zentry)) }
+	if name == 0 {
+		return ''
+	}
+	return unsafe { name.vstring() }
+}
+
+// index returns an index of the current zip entry.
+pub fn (mut zentry Zip) index() !int {
+	index := int(C.zip_entry_index(zentry))
+	if index == -1 {
+		return error('szip: cannot get current index of zip entry')
+	}
+	return index // must be check for INVALID_VALUE
+}
+
+// is_dir determines if the current zip entry is a directory entry.
+pub fn (mut zentry Zip) is_dir() !bool {
+	isdir := C.zip_entry_isdir(zentry)
+	if isdir < 0 {
+		return error('szip: cannot check entry type')
+	}
+	return isdir == 1
+}
+
+// size returns an uncompressed size of the current zip entry.
+@[inline]
+pub fn (mut zentry Zip) size() u64 {
+	return C.zip_entry_size(zentry)
+}
+
+// crc32 returns CRC-32 checksum of the current zip entry.
+@[inline]
+pub fn (mut zentry Zip) crc32() u32 {
+	return C.zip_entry_crc32(zentry)
+}
+
+// write_entry compresses an input buffer for the current zip entry.
+pub fn (mut zentry Zip) write_entry(data []u8) ! {
+	if int(data[0] & 0xff) == -1 {
+		return error('szip: cannot write entry')
+	}
+	res := C.zip_entry_write(zentry, data.data, data.len)
+	if res != 0 {
+		return error('szip: failed to write entry')
+	}
+}
+
+// create_entry compresses a file for the current zip entry.
+pub fn (mut zentry Zip) create_entry(name string) ! {
+	res := C.zip_entry_fwrite(zentry, &char(name.str))
+	if res != 0 {
+		return error('szip: failed to create entry')
+	}
+}
+
+// read_entry extracts the current zip entry into output buffer.
+// The function allocates sufficient memory for an output buffer.
+// NOTE: remember to release the memory allocated for an output buffer.
+// for large entries, please take a look at zip_entry_extract function.
+pub fn (mut zentry Zip) read_entry() !voidptr {
+	mut buf := &u8(0)
+	mut bsize := usize(0)
+	res := C.zip_entry_read(zentry, unsafe { &voidptr(&buf) }, &bsize)
+	if res == -1 {
+		return error('szip: cannot read properly data from entry')
+	}
+	return buf
+}
+
+// read_entry_buf extracts the current zip entry into user specified buffer
+pub fn (mut zentry Zip) read_entry_buf(buf voidptr, in_bsize int) !int {
+	bsize := usize(in_bsize)
+	res := C.zip_entry_noallocread(zentry, buf, bsize)
+	if res == -1 {
+		return error('szip: cannot read properly data from entry')
+	}
+	return res
+}
+
+// extract_entry extracts the current zip entry into output file.
+pub fn (mut zentry Zip) extract_entry(path string) ! {
+	res := C.zip_entry_fread(zentry, &char(path.str))
+	if res != 0 {
+		return error('szip: failed to extract entry')
+	}
+}
+
+// extract zip file to directory
+pub fn extract_zip_to_dir(file string, dir string) !bool {
+	if C.access(&char(dir.str), 0) == -1 {
+		return error('szip: cannot open directory for extracting, directory not exists')
+	}
+	res := C.zip_extract(&char(file.str), &char(dir.str), cb_zip_extract, 0)
+	return res == 0
+}
+
+// zip files (full path) to zip file
+pub fn zip_files(path_to_file []string, path_to_export_zip string) ! {
+	// open or create new zip
+	mut zip := open(path_to_export_zip, .no_compression, .write) or { panic(err) }
+
+	// add all files from the directory to the archive
+	for file in path_to_file {
+		// add file to zip
+		zip.open_entry(os.base(file)) or { panic(err) }
+		file_as_byte := os.read_bytes(file) or { panic(err) }
+		zip.write_entry(file_as_byte) or { panic(err) }
+
+		zip.close_entry()
+	}
+
+	// close zip
+	defer {
+		zip.close()
+	}
+}
+
+// zip_folder zips all entries in `folder` *recursively* to the zip file at `zip_file`.
+// Empty folders will be included, unless specified otherwise in `opt`.
+pub fn zip_folder(folder string, zip_file string, opt ZipFolderOptions) ! {
+	// get list of files from directory
+	path := folder.trim_right(os.path_separator)
+	mut files := []string{}
+	os.walk_with_context(path, &files, fn (mut files []string, file string) {
+		files << file
+	})
+
+	// open or create new zip
+	mut zip := open(zip_file, .no_compression, .write)!
+	// close zip
+	defer {
+		zip.close()
+	}
+
+	// add all files from the directory to the archive
+	for file in files {
+		is_dir := os.is_dir(file)
+		if opt.omit_empty_folders && is_dir {
+			continue
+		}
+		// strip each zip entry for the path prefix - this way
+		// all files in the archive can be made relative.
+		mut zip_file_entry := file.trim_string_left(path + os.path_separator)
+		// Normalize path on Windows \ -> /
+		$if windows {
+			zip_file_entry = zip_file_entry.replace(os.path_separator, '/')
+		}
+		if is_dir {
+			zip_file_entry += '/' // Tells the implementation that the entry is a directory
+		}
+		// add file or directory (ends with "/") to zip
+		zip.open_entry(zip_file_entry)!
+		if !is_dir {
+			file_as_byte := os.read_bytes(file)!
+			zip.write_entry(file_as_byte)!
+		}
+		zip.close_entry()
+	}
+}
+
+// total returns the number of all entries (files and directories) in the zip archive.
+pub fn (mut zentry Zip) total() !int {
+	tentry := int(C.zip_entries_total(zentry))
+	if tentry == -1 {
+		return error('szip: cannot count total entries')
+	}
+	return tentry
+}

vlib/compress/szip/szip_test.v

@@ -0,0 +1,172 @@
+import compress.szip
+import os
+
+const test_out_zip = 'v_test_zip.zip'
+const test_dir_zip = 'v_test_dir_zip.zip'
+const test_path = 'zip files'
+const test_path2 = '.zip folder'
+const test_path3 = 'test zip folder'
+const test_path3_1 = os.join_path(test_path3, '1', '1')
+const test_path3_2 = os.join_path(test_path3, '2', '1')
+const test_path3_3 = os.join_path(test_path3, '3', '1')
+const test_path3_4 = os.join_path(test_path3, '4', '1')
+const fname1 = 'file_1.txt'
+const fpath1 = os.join_path(test_path, fname1)
+const fname2 = 'file_2.txt'
+const fpath2 = os.join_path(test_path, fname2)
+const fname3 = '.New Text Document.txt'
+const fpath3 = os.join_path(test_path2, fname3)
+const fname4 = 'file.txt'
+const fpath4 = os.join_path(test_path3_1, fname4)
+const fpath5 = os.join_path(test_path3_2, fname4)
+const fpath6 = os.join_path(test_path3_4, fname4)
+
+fn cleanup() {
+	os.chdir(os.temp_dir()) or {}
+	os.rmdir_all(test_path) or {}
+	os.rmdir_all(test_path2) or {}
+	os.rmdir_all(test_path3) or {}
+	os.rm(test_out_zip) or {}
+	os.rm(test_dir_zip) or {}
+}
+
+fn testsuite_begin() {
+	cleanup()
+}
+
+fn testsuite_end() {
+	cleanup()
+}
+
+fn test_szip_create_temp_files() {
+	os.mkdir(test_path)!
+	os.mkdir(test_path2)!
+	os.write_file(fpath1, 'file one')!
+	os.write_file(fpath2, 'file two')!
+	os.write_file(fpath3, 'file three')!
+	assert os.exists(fpath1)
+	assert os.exists(fpath2)
+	assert os.exists(fpath3)
+}
+
+fn test_zipping_files() {
+	mut files := (os.ls(test_path)!).map(os.join_path(test_path, it))
+	files << (os.ls(test_path2)!).map(os.join_path(test_path2, it))
+	szip.zip_files(files, test_out_zip)!
+	assert os.exists(test_out_zip)
+	os.rm(fpath1)!
+	os.rm(fpath2)!
+	os.rm(fpath3)!
+}
+
+fn test_extract_zipped_files() {
+	szip.extract_zip_to_dir(test_out_zip, test_path)!
+	szip.extract_zip_to_dir(test_out_zip, test_path2)!
+	assert os.exists(fpath1)
+	assert os.exists(fpath2)
+	assert os.exists(fpath3)
+	assert (os.read_file(fpath1)!) == 'file one'
+	assert (os.read_file(fpath2)!) == 'file two'
+	assert (os.read_file(fpath3)!) == 'file three'
+	cleanup()
+}
+
+fn test_reading_zipping_files() {
+	n_files := 2
+	mut file_name_list := []string{}
+	for i in 0 .. n_files {
+		file_name_list << 'file_${i:02}.txt'
+	}
+
+	cleanup()
+	os.mkdir(test_path)!
+	os.mkdir(test_path2)!
+	os.write_file(fpath3, 'file three')!
+	for c, f_name in file_name_list {
+		tmp_path := os.join_path(test_path, f_name)
+		os.write_file(tmp_path, 'file ${c:02}')!
+		assert os.exists(tmp_path)
+	}
+	files := (os.ls(test_path)!).map(os.join_path(test_path, it))
+
+	szip.zip_files(files, test_out_zip)!
+	assert os.exists(test_out_zip)
+
+	mut zp := szip.open(test_out_zip, szip.CompressionLevel.no_compression, szip.OpenMode.read_only)!
+	n_entries := zp.total()!
+	assert n_entries == n_files
+
+	unsafe {
+		data_len := 'file XX'.len
+		buf_size := 32
+		buf := malloc(data_len * 2)
+
+		for _ in 0 .. n_files {
+			zp.open_entry_by_index(0)!
+			name := zp.name()
+			assert name in file_name_list
+
+			zp.read_entry_buf(buf, buf_size)!
+			buf[data_len] = 0
+			tmp_str := tos(buf, data_len)
+
+			assert tmp_str[0..4] == 'file'
+			assert tmp_str[5..7] == name[5..7]
+
+			zp.close_entry()
+		}
+
+		free(buf)
+	}
+	zp.close()
+}
+
+fn test_zip_folder() {
+	cleanup()
+	os.mkdir_all(test_path3_1)!
+	os.mkdir_all(test_path3_2)!
+	os.mkdir_all(test_path3_3)!
+	os.mkdir_all(test_path3_4)!
+	os.write_file(fpath4, '4')!
+	os.write_file(fpath5, '5')!
+	os.write_file(fpath6, '6')!
+
+	szip.zip_folder(test_path3, test_dir_zip)!
+	assert os.exists(test_dir_zip)
+
+	os.rmdir_all(test_path3)!
+	os.mkdir_all(test_path3)!
+	szip.extract_zip_to_dir(test_dir_zip, test_path3)!
+	assert os.exists(test_path3_1)
+	assert os.exists(test_path3_2)
+	assert os.exists(test_path3_3) // This is the empty dir
+	assert os.exists(test_path3_4)
+	assert (os.read_file(fpath4)!) == '4'
+	assert (os.read_file(fpath5)!) == '5'
+	assert (os.read_file(fpath6)!) == '6'
+}
+
+fn test_zip_folder_omit_empty_directories() {
+	cleanup()
+	os.mkdir_all(test_path3_1)!
+	os.mkdir_all(test_path3_2)!
+	os.mkdir_all(test_path3_3)!
+	os.mkdir_all(test_path3_4)!
+	os.write_file(fpath4, '4')!
+	os.write_file(fpath5, '5')!
+	os.write_file(fpath6, '6')!
+
+	szip.zip_folder(test_path3, test_dir_zip, omit_empty_folders: true)!
+	assert os.exists(test_dir_zip)
+
+	os.rmdir_all(test_path3)!
+	os.mkdir_all(test_path3)!
+	szip.extract_zip_to_dir(test_dir_zip, test_path3)!
+	assert os.exists(test_path3_1)
+	assert os.exists(test_path3_2)
+	assert !os.exists(test_path3_3) // This is the empty dir, should be omitted with `omit_empty_folders`
+	assert os.exists(test_path3_4)
+	assert (os.read_file(fpath4)!) == '4'
+	assert (os.read_file(fpath5)!) == '5'
+	assert (os.read_file(fpath6)!) == '6'
+}

vlib/szip/szip.c.v

@@ -1,3 +1,5 @@
+@[deprecated: 'use `compress/szip` instead']
+@[deprecated_after: '2023-12-31']
 module szip
 
 import os