From d645114f7e3cbbfa19bb47ace2671fcac75e32ca Mon Sep 17 00:00:00 2001 From: Igor Anić Date: Tue, 13 Feb 2024 22:14:40 +0100 Subject: add deflate implemented from first principles Zig deflate compression/decompression implementation. It supports compression and decompression of gzip, zlib and raw deflate format. Fixes #18062. This PR replaces current compress/gzip and compress/zlib packages. Deflate package is renamed to flate. Flate is common name for deflate/inflate where deflate is compression and inflate decompression. There are breaking change. Methods signatures are changed because of removal of the allocator, and I also unified API for all three namespaces (flate, gzip, zlib). Currently I put old packages under v1 namespace they are still available as compress/v1/gzip, compress/v1/zlib, compress/v1/deflate. Idea is to give users of the current API little time to postpone analyzing what they had to change. Although that rises question when it is safe to remove that v1 namespace. Here is current API in the compress package: ```Zig // deflate fn compressor(allocator, writer, options) !Compressor(@TypeOf(writer)) fn Compressor(comptime WriterType) type fn decompressor(allocator, reader, null) !Decompressor(@TypeOf(reader)) fn Decompressor(comptime ReaderType: type) type // gzip fn compress(allocator, writer, options) !Compress(@TypeOf(writer)) fn Compress(comptime WriterType: type) type fn decompress(allocator, reader) !Decompress(@TypeOf(reader)) fn Decompress(comptime ReaderType: type) type // zlib fn compressStream(allocator, writer, options) !CompressStream(@TypeOf(writer)) fn CompressStream(comptime WriterType: type) type fn decompressStream(allocator, reader) !DecompressStream(@TypeOf(reader)) fn DecompressStream(comptime ReaderType: type) type // xz fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) fn Decompress(comptime ReaderType: type) type // lzma fn decompress(allocator, reader) !Decompress(@TypeOf(reader)) fn Decompress(comptime ReaderType: type) type // lzma2 fn decompress(allocator, reader, writer !void // zstandard: fn DecompressStream(ReaderType, options) type fn decompressStream(allocator, reader) DecompressStream(@TypeOf(reader), .{}) struct decompress ``` The proposed naming convention: - Compressor/Decompressor for functions which return type, like Reader/Writer/GeneralPurposeAllocator - compressor/compressor for functions which are initializers for that type, like reader/writer/allocator - compress/decompress for one shot operations, accepts reader/writer pair, like read/write/alloc ```Zig /// Compress from reader and write compressed data to the writer. fn compress(reader: anytype, writer: anytype, options: Options) !void /// Create Compressor which outputs the writer. fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) /// Compressor type fn Compressor(comptime WriterType: type) type /// Decompress from reader and write plain data to the writer. fn decompress(reader: anytype, writer: anytype) !void /// Create Decompressor which reads from reader. fn decompressor(reader: anytype) Decompressor(@TypeOf(reader) /// Decompressor type fn Decompressor(comptime ReaderType: type) type ``` Comparing this implementation with the one we currently have in Zig's standard library (std). Std is roughly 1.2-1.4 times slower in decompression, and 1.1-1.2 times slower in compression. Compressed sizes are pretty much same in both cases. More resutls in [this](https://github.com/ianic/flate) repo. This library uses static allocations for all structures, doesn't require allocator. That makes sense especially for deflate where all structures, internal buffers are allocated to the full size. Little less for inflate where we std version uses less memory by not preallocating to theoretical max size array which are usually not fully used. For deflate this library allocates 395K while std 779K. For inflate this library allocates 74.5K while std around 36K. Inflate difference is because we here use 64K history instead of 32K in std. If merged existing usage of compress gzip/zlib/deflate need some changes. Here is example with necessary changes in comments: ```Zig const std = @import("std"); // To get this file: // wget -nc -O war_and_peace.txt https://www.gutenberg.org/ebooks/2600.txt.utf-8 const data = @embedFile("war_and_peace.txt"); pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer std.debug.assert(gpa.deinit() == .ok); const allocator = gpa.allocator(); try oldDeflate(allocator); try new(std.compress.flate, allocator); try oldZlib(allocator); try new(std.compress.zlib, allocator); try oldGzip(allocator); try new(std.compress.gzip, allocator); } pub fn new(comptime pkg: type, allocator: std.mem.Allocator) !void { var buf = std.ArrayList(u8).init(allocator); defer buf.deinit(); // Compressor var cmp = try pkg.compressor(buf.writer(), .{}); _ = try cmp.write(data); try cmp.finish(); var fbs = std.io.fixedBufferStream(buf.items); // Decompressor var dcp = pkg.decompressor(fbs.reader()); const plain = try dcp.reader().readAllAlloc(allocator, std.math.maxInt(usize)); defer allocator.free(plain); try std.testing.expectEqualSlices(u8, data, plain); } pub fn oldDeflate(allocator: std.mem.Allocator) !void { const deflate = std.compress.v1.deflate; // Compressor var buf = std.ArrayList(u8).init(allocator); defer buf.deinit(); // Remove allocator // Rename deflate -> flate var cmp = try deflate.compressor(allocator, buf.writer(), .{}); _ = try cmp.write(data); try cmp.close(); // Rename to finish cmp.deinit(); // Remove // Decompressor var fbs = std.io.fixedBufferStream(buf.items); // Remove allocator and last param // Rename deflate -> flate // Remove try var dcp = try deflate.decompressor(allocator, fbs.reader(), null); defer dcp.deinit(); // Remove const plain = try dcp.reader().readAllAlloc(allocator, std.math.maxInt(usize)); defer allocator.free(plain); try std.testing.expectEqualSlices(u8, data, plain); } pub fn oldZlib(allocator: std.mem.Allocator) !void { const zlib = std.compress.v1.zlib; var buf = std.ArrayList(u8).init(allocator); defer buf.deinit(); // Compressor // Rename compressStream => compressor // Remove allocator var cmp = try zlib.compressStream(allocator, buf.writer(), .{}); _ = try cmp.write(data); try cmp.finish(); cmp.deinit(); // Remove var fbs = std.io.fixedBufferStream(buf.items); // Decompressor // decompressStream => decompressor // Remove allocator // Remove try var dcp = try zlib.decompressStream(allocator, fbs.reader()); defer dcp.deinit(); // Remove const plain = try dcp.reader().readAllAlloc(allocator, std.math.maxInt(usize)); defer allocator.free(plain); try std.testing.expectEqualSlices(u8, data, plain); } pub fn oldGzip(allocator: std.mem.Allocator) !void { const gzip = std.compress.v1.gzip; var buf = std.ArrayList(u8).init(allocator); defer buf.deinit(); // Compressor // Rename compress => compressor // Remove allocator var cmp = try gzip.compress(allocator, buf.writer(), .{}); _ = try cmp.write(data); try cmp.close(); // Rename to finisho cmp.deinit(); // Remove var fbs = std.io.fixedBufferStream(buf.items); // Decompressor // Rename decompress => decompressor // Remove allocator // Remove try var dcp = try gzip.decompress(allocator, fbs.reader()); defer dcp.deinit(); // Remove const plain = try dcp.reader().readAllAlloc(allocator, std.math.maxInt(usize)); defer allocator.free(plain); try std.testing.expectEqualSlices(u8, data, plain); } ``` --- src/Package/Fetch.zig | 7 ++++++- src/Package/Fetch/git.zig | 9 +++------ src/link/Elf/Object.zig | 4 +--- src/objcopy.zig | 3 +-- 4 files changed, 11 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index fb9d7c823c..ed3c6b099f 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1099,7 +1099,12 @@ fn unpackResource( switch (file_type) { .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), - .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), + .@"tar.gz" => { + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + var dcp = std.compress.gzip.decompressor(br.reader()); + try unpackTarball(f, tmp_directory.handle, dcp.reader()); + }, .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper), .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index df5332d41d..abbb031948 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -1115,8 +1115,7 @@ fn indexPackFirstPass( const entry_header = try EntryHeader.read(entry_crc32_reader.reader()); switch (entry_header) { inline .commit, .tree, .blob, .tag => |object, tag| { - var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); - defer entry_decompress_stream.deinit(); + var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); const entry_writer = entry_hashed_writer.writer(); @@ -1135,8 +1134,7 @@ fn indexPackFirstPass( }); }, inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); - defer entry_decompress_stream.deinit(); + var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); @@ -1257,8 +1255,7 @@ fn resolveDeltaChain( fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader()); - defer decompress_stream.deinit(); + var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); const data = try allocator.alloc(u8, alloc_size); errdefer allocator.free(data); try decompress_stream.reader().readNoEof(data); diff --git a/src/link/Elf/Object.zig b/src/link/Elf/Object.zig index 395e6680a1..b29de3fc59 100644 --- a/src/link/Elf/Object.zig +++ b/src/link/Elf/Object.zig @@ -902,9 +902,7 @@ pub fn codeDecompressAlloc(self: Object, elf_file: *Elf, atom_index: Atom.Index) switch (chdr.ch_type) { .ZLIB => { var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]); - var zlib_stream = std.compress.zlib.decompressStream(gpa, stream.reader()) catch - return error.InputOutput; - defer zlib_stream.deinit(); + var zlib_stream = std.compress.zlib.decompressor(stream.reader()); const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow; const decomp = try gpa.alloc(u8, size); const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput; diff --git a/src/objcopy.zig b/src/objcopy.zig index 8ea0eaac59..81638d4af0 100644 --- a/src/objcopy.zig +++ b/src/objcopy.zig @@ -1298,8 +1298,7 @@ const ElfFileHelper = struct { try compressed_stream.writer().writeAll(prefix); { - var compressor = try std.compress.zlib.compressStream(allocator, compressed_stream.writer(), .{}); - defer compressor.deinit(); + var compressor = try std.compress.zlib.compressor(compressed_stream.writer(), .{}); var buf: [8000]u8 = undefined; while (true) { -- cgit v1.2.3 From cb3fe277bbbdcf432530cd0a1f3d05d3d924675b Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Wed, 14 Feb 2024 07:37:16 +0100 Subject: x86_64: fix crash loading a packed value from a spilled pointer Unblocks #18923 --- src/arch/x86_64/CodeGen.zig | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index e547b797ac..33b8623183 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -7276,7 +7276,20 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn else => |vector_index| @intFromEnum(vector_index) * val_bit_size, }; if (ptr_bit_off % 8 == 0) { - try self.load(dst_mcv, ptr_ty, ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); + { + const mat_ptr_mcv: MCValue = switch (ptr_mcv) { + .immediate, .register, .register_offset, .lea_frame => ptr_mcv, + else => .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, + }; + const mat_ptr_lock = switch (mat_ptr_mcv) { + .register => |mat_ptr_reg| self.register_manager.lockReg(mat_ptr_reg), + else => null, + }; + defer if (mat_ptr_lock) |lock| self.register_manager.unlockReg(lock); + + try self.load(dst_mcv, ptr_ty, mat_ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); + } + if (val_abi_size * 8 > val_bit_size) { if (dst_mcv.isRegister()) { try self.truncateRegister(val_ty, dst_mcv.getReg().?); -- cgit v1.2.3