diff options
Diffstat (limited to 'lib/std')
55 files changed, 2069 insertions, 1689 deletions
diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig new file mode 100644 index 0000000000..ae9b1783be --- /dev/null +++ b/lib/std/Progress.zig @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! This API non-allocating, non-fallible, and thread-safe. +//! The tradeoff is that users of this API must provide the storage +//! for each `Progress.Node`. +//! +//! Initialize the struct directly, overriding these fields as desired: +//! * `refresh_rate_ms` +//! * `initial_delay_ms` + +const std = @import("std"); +const windows = std.os.windows; +const testing = std.testing; +const assert = std.debug.assert; +const Progress = @This(); + +/// `null` if the current node (and its children) should +/// not print on update() +terminal: ?std.fs.File = undefined, + +/// Whether the terminal supports ANSI escape codes. +supports_ansi_escape_codes: bool = false, + +root: Node = undefined, + +/// Keeps track of how much time has passed since the beginning. +/// Used to compare with `initial_delay_ms` and `refresh_rate_ms`. +timer: std.time.Timer = undefined, + +/// When the previous refresh was written to the terminal. +/// Used to compare with `refresh_rate_ms`. +prev_refresh_timestamp: u64 = undefined, + +/// This buffer represents the maximum number of bytes written to the terminal +/// with each refresh. +output_buffer: [100]u8 = undefined, + +/// How many nanoseconds between writing updates to the terminal. +refresh_rate_ns: u64 = 50 * std.time.ns_per_ms, + +/// How many nanoseconds to keep the output hidden +initial_delay_ns: u64 = 500 * std.time.ns_per_ms, + +done: bool = true, + +/// Protects the `refresh` function, as well as `node.recently_updated_child`. +/// Without this, callsites would call `Node.end` and then free `Node` memory +/// while it was still being accessed by the `refresh` function. +update_lock: std.Mutex = .{}, + +/// Keeps track of how many columns in the terminal have been output, so that +/// we can move the cursor back later. +columns_written: usize = undefined, + +/// Represents one unit of progress. Each node can have children nodes, or +/// one can use integers with `update`. +pub const Node = struct { + context: *Progress, + parent: ?*Node, + name: []const u8, + /// Must be handled atomically to be thread-safe. + recently_updated_child: ?*Node = null, + /// Must be handled atomically to be thread-safe. 0 means null. + unprotected_estimated_total_items: usize, + /// Must be handled atomically to be thread-safe. + unprotected_completed_items: usize, + + /// Create a new child progress node. Thread-safe. + /// Call `Node.end` when done. + /// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this + /// API to set `self.parent.recently_updated_child` with the return value. + /// Until that is fixed you probably want to call `activate` on the return value. + /// Passing 0 for `estimated_total_items` means unknown. + pub fn start(self: *Node, name: []const u8, estimated_total_items: usize) Node { + return Node{ + .context = self.context, + .parent = self, + .name = name, + .unprotected_estimated_total_items = estimated_total_items, + .unprotected_completed_items = 0, + }; + } + + /// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe. + pub fn completeOne(self: *Node) void { + self.activate(); + _ = @atomicRmw(usize, &self.unprotected_completed_items, .Add, 1, .Monotonic); + self.context.maybeRefresh(); + } + + /// Finish a started `Node`. Thread-safe. + pub fn end(self: *Node) void { + self.context.maybeRefresh(); + if (self.parent) |parent| { + { + const held = self.context.update_lock.acquire(); + defer held.release(); + _ = @cmpxchgStrong(?*Node, &parent.recently_updated_child, self, null, .Monotonic, .Monotonic); + } + parent.completeOne(); + } else { + self.context.done = true; + self.context.refresh(); + } + } + + /// Tell the parent node that this node is actively being worked on. Thread-safe. + pub fn activate(self: *Node) void { + if (self.parent) |parent| { + @atomicStore(?*Node, &parent.recently_updated_child, self, .Release); + } + } + + /// Thread-safe. 0 means unknown. + pub fn setEstimatedTotalItems(self: *Node, count: usize) void { + @atomicStore(usize, &self.unprotected_estimated_total_items, count, .Monotonic); + } + + /// Thread-safe. + pub fn setCompletedItems(self: *Node, completed_items: usize) void { + @atomicStore(usize, &self.unprotected_completed_items, completed_items, .Monotonic); + } +}; + +/// Create a new progress node. +/// Call `Node.end` when done. +/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this +/// API to return Progress rather than accept it as a parameter. +/// `estimated_total_items` value of 0 means unknown. +pub fn start(self: *Progress, name: []const u8, estimated_total_items: usize) !*Node { + const stderr = std.io.getStdErr(); + self.terminal = null; + if (stderr.supportsAnsiEscapeCodes()) { + self.terminal = stderr; + self.supports_ansi_escape_codes = true; + } else if (std.builtin.os.tag == .windows and stderr.isTty()) { + self.terminal = stderr; + } + self.root = Node{ + .context = self, + .parent = null, + .name = name, + .unprotected_estimated_total_items = estimated_total_items, + .unprotected_completed_items = 0, + }; + self.columns_written = 0; + self.prev_refresh_timestamp = 0; + self.timer = try std.time.Timer.start(); + self.done = false; + return &self.root; +} + +/// Updates the terminal if enough time has passed since last update. Thread-safe. +pub fn maybeRefresh(self: *Progress) void { + const now = self.timer.read(); + if (now < self.initial_delay_ns) return; + const held = self.update_lock.tryAcquire() orelse return; + defer held.release(); + if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return; + return self.refreshWithHeldLock(); +} + +/// Updates the terminal and resets `self.next_refresh_timestamp`. Thread-safe. +pub fn refresh(self: *Progress) void { + const held = self.update_lock.tryAcquire() orelse return; + defer held.release(); + + return self.refreshWithHeldLock(); +} + +fn refreshWithHeldLock(self: *Progress) void { + const file = self.terminal orelse return; + + const prev_columns_written = self.columns_written; + var end: usize = 0; + if (self.columns_written > 0) { + // restore the cursor position by moving the cursor + // `columns_written` cells to the left, then clear the rest of the + // line + if (self.supports_ansi_escape_codes) { + end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{d}D", .{self.columns_written}) catch unreachable).len; + end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len; + } else if (std.builtin.os.tag == .windows) winapi: { + var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined; + if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE) + unreachable; + + var cursor_pos = windows.COORD{ + .X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written), + .Y = info.dwCursorPosition.Y, + }; + + if (cursor_pos.X < 0) + cursor_pos.X = 0; + + const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X); + + var written: windows.DWORD = undefined; + if (windows.kernel32.FillConsoleOutputAttribute( + file.handle, + info.wAttributes, + fill_chars, + cursor_pos, + &written, + ) != windows.TRUE) { + // Stop trying to write to this file. + self.terminal = null; + break :winapi; + } + if (windows.kernel32.FillConsoleOutputCharacterA( + file.handle, + ' ', + fill_chars, + cursor_pos, + &written, + ) != windows.TRUE) unreachable; + + if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE) + unreachable; + } else unreachable; + + self.columns_written = 0; + } + + if (!self.done) { + var need_ellipse = false; + var maybe_node: ?*Node = &self.root; + while (maybe_node) |node| { + if (need_ellipse) { + self.bufWrite(&end, "... ", .{}); + } + need_ellipse = false; + const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic); + const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic); + if (node.name.len != 0 or eti > 0) { + if (node.name.len != 0) { + self.bufWrite(&end, "{s}", .{node.name}); + need_ellipse = true; + } + if (eti > 0) { + if (need_ellipse) self.bufWrite(&end, " ", .{}); + self.bufWrite(&end, "[{d}/{d}] ", .{ completed_items + 1, eti }); + need_ellipse = false; + } else if (completed_items != 0) { + if (need_ellipse) self.bufWrite(&end, " ", .{}); + self.bufWrite(&end, "[{d}] ", .{completed_items + 1}); + need_ellipse = false; + } + } + maybe_node = @atomicLoad(?*Node, &node.recently_updated_child, .Acquire); + } + if (need_ellipse) { + self.bufWrite(&end, "... ", .{}); + } + } + + _ = file.write(self.output_buffer[0..end]) catch |e| { + // Stop trying to write to this file once it errors. + self.terminal = null; + }; + self.prev_refresh_timestamp = self.timer.read(); +} + +pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void { + const file = self.terminal orelse return; + self.refresh(); + file.outStream().print(format, args) catch { + self.terminal = null; + return; + }; + self.columns_written = 0; +} + +fn bufWrite(self: *Progress, end: *usize, comptime format: []const u8, args: anytype) void { + if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) |written| { + const amt = written.len; + end.* += amt; + self.columns_written += amt; + } else |err| switch (err) { + error.NoSpaceLeft => { + self.columns_written += self.output_buffer.len - end.*; + end.* = self.output_buffer.len; + }, + } + const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11; + const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end; + if (end.* > max_end) { + const suffix = "... "; + self.columns_written = self.columns_written - (end.* - max_end) + suffix.len; + std.mem.copy(u8, self.output_buffer[max_end..], suffix); + end.* = max_end + suffix.len; + } +} + +test "basic functionality" { + var disable = true; + if (disable) { + // This test is disabled because it uses time.sleep() and is therefore slow. It also + // prints bogus progress data to stderr. + return error.SkipZigTest; + } + var progress = Progress{}; + const root_node = try progress.start("", 100); + defer root_node.end(); + + const sub_task_names = [_][]const u8{ + "reticulating splines", + "adjusting shoes", + "climbing towers", + "pouring juice", + }; + var next_sub_task: usize = 0; + + var i: usize = 0; + while (i < 100) : (i += 1) { + var node = root_node.start(sub_task_names[next_sub_task], 5); + node.activate(); + next_sub_task = (next_sub_task + 1) % sub_task_names.len; + + node.completeOne(); + std.time.sleep(5 * std.time.ns_per_ms); + node.completeOne(); + node.completeOne(); + std.time.sleep(5 * std.time.ns_per_ms); + node.completeOne(); + node.completeOne(); + std.time.sleep(5 * std.time.ns_per_ms); + + node.end(); + + std.time.sleep(5 * std.time.ns_per_ms); + } + { + var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", 0); + node.activate(); + std.time.sleep(10 * std.time.ns_per_ms); + progress.refresh(); + std.time.sleep(10 * std.time.ns_per_ms); + node.end(); + } +} diff --git a/lib/std/auto_reset_event.zig b/lib/std/auto_reset_event.zig index 7e13dc1aba..3c7e65e362 100644 --- a/lib/std/auto_reset_event.zig +++ b/lib/std/auto_reset_event.zig @@ -11,33 +11,33 @@ const assert = std.debug.assert; /// Similar to std.ResetEvent but on `set()` it also (atomically) does `reset()`. /// Unlike std.ResetEvent, `wait()` can only be called by one thread (MPSC-like). pub const AutoResetEvent = struct { - // AutoResetEvent has 3 possible states: - // - UNSET: the AutoResetEvent is currently unset - // - SET: the AutoResetEvent was notified before a wait() was called - // - <std.ResetEvent pointer>: there is an active waiter waiting for a notification. - // - // When attempting to wait: - // if the event is unset, it registers a ResetEvent pointer to be notified when the event is set - // if the event is already set, then it consumes the notification and resets the event. - // - // When attempting to notify: - // if the event is unset, then we set the event - // if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent - // - // This ensures that the event is automatically reset after a wait() has been issued - // and avoids the race condition when using std.ResetEvent in the following scenario: - // thread 1 | thread 2 - // std.ResetEvent.wait() | - // | std.ResetEvent.set() - // | std.ResetEvent.set() - // std.ResetEvent.reset() | - // std.ResetEvent.wait() | (missed the second .set() notification above) + /// AutoResetEvent has 3 possible states: + /// - UNSET: the AutoResetEvent is currently unset + /// - SET: the AutoResetEvent was notified before a wait() was called + /// - <std.ResetEvent pointer>: there is an active waiter waiting for a notification. + /// + /// When attempting to wait: + /// if the event is unset, it registers a ResetEvent pointer to be notified when the event is set + /// if the event is already set, then it consumes the notification and resets the event. + /// + /// When attempting to notify: + /// if the event is unset, then we set the event + /// if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent + /// + /// This ensures that the event is automatically reset after a wait() has been issued + /// and avoids the race condition when using std.ResetEvent in the following scenario: + /// thread 1 | thread 2 + /// std.ResetEvent.wait() | + /// | std.ResetEvent.set() + /// | std.ResetEvent.set() + /// std.ResetEvent.reset() | + /// std.ResetEvent.wait() | (missed the second .set() notification above) state: usize = UNSET, const UNSET = 0; const SET = 1; - // the minimum alignment for the `*std.ResetEvent` created by wait*() + /// the minimum alignment for the `*std.ResetEvent` created by wait*() const event_align = std.math.max(@alignOf(std.ResetEvent), 2); pub fn wait(self: *AutoResetEvent) void { diff --git a/lib/std/build.zig b/lib/std/build.zig index dacfaf5f75..8d6db459ea 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -509,7 +509,26 @@ pub const Builder = struct { return null; }, }, - .Float => panic("TODO float options to build script", .{}), + .Float => switch (entry.value.value) { + .Flag => { + warn("Expected -D{} to be a float, but received a boolean.\n", .{name}); + self.markInvalidUserInput(); + return null; + }, + .Scalar => |s| { + const n = std.fmt.parseFloat(T, s) catch |err| { + warn("Expected -D{} to be a float of type {}.\n", .{ name, @typeName(T) }); + self.markInvalidUserInput(); + return null; + }; + return n; + }, + .List => { + warn("Expected -D{} to be a float, but received a list.\n", .{name}); + self.markInvalidUserInput(); + return null; + }, + }, .Enum => switch (entry.value.value) { .Flag => { warn("Expected -D{} to be a string, but received a boolean.\n", .{name}); diff --git a/lib/std/c.zig b/lib/std/c.zig index 5ebbb9dd22..aae3f383d1 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -12,6 +12,7 @@ pub const Token = tokenizer.Token; pub const Tokenizer = tokenizer.Tokenizer; pub const parse = @import("c/parse.zig").parse; pub const ast = @import("c/ast.zig"); +pub const builtins = @import("c/builtins.zig"); test "" { _ = tokenizer; @@ -200,7 +201,7 @@ pub usingnamespace switch (builtin.os.tag) { pub extern "c" fn gettimeofday(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; pub extern "c" fn nanosleep(rqtp: *const timespec, rmtp: ?*timespec) c_int; pub extern "c" fn sched_yield() c_int; - pub extern "c" fn sigaction(sig: c_int, noalias act: *const Sigaction, noalias oact: ?*Sigaction) c_int; + pub extern "c" fn sigaction(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int; pub extern "c" fn sigprocmask(how: c_int, noalias set: ?*const sigset_t, noalias oset: ?*sigset_t) c_int; pub extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int; pub extern "c" fn stat(noalias path: [*:0]const u8, noalias buf: *libc_stat) c_int; @@ -215,7 +216,7 @@ pub usingnamespace switch (builtin.os.tag) { pub extern "c" fn gettimeofday(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; pub extern "c" fn nanosleep(rqtp: *const timespec, rmtp: ?*timespec) c_int; pub extern "c" fn sched_yield() c_int; - pub extern "c" fn sigaction(sig: c_int, noalias act: *const Sigaction, noalias oact: ?*Sigaction) c_int; + pub extern "c" fn sigaction(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int; pub extern "c" fn sigprocmask(how: c_int, noalias set: ?*const sigset_t, noalias oset: ?*sigset_t) c_int; pub extern "c" fn stat(noalias path: [*:0]const u8, noalias buf: *libc_stat) c_int; }, @@ -227,7 +228,7 @@ pub usingnamespace switch (builtin.os.tag) { pub extern "c" fn gettimeofday(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; pub extern "c" fn nanosleep(rqtp: *const timespec, rmtp: ?*timespec) c_int; pub extern "c" fn sched_yield() c_int; - pub extern "c" fn sigaction(sig: c_int, noalias act: *const Sigaction, noalias oact: ?*Sigaction) c_int; + pub extern "c" fn sigaction(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int; pub extern "c" fn sigprocmask(how: c_int, noalias set: ?*const sigset_t, noalias oset: ?*sigset_t) c_int; pub extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int; pub extern "c" fn stat(noalias path: [*:0]const u8, noalias buf: *libc_stat) c_int; @@ -264,6 +265,11 @@ pub extern "c" fn pthread_attr_setguardsize(attr: *pthread_attr_t, guardsize: us pub extern "c" fn pthread_attr_destroy(attr: *pthread_attr_t) c_int; pub extern "c" fn pthread_self() pthread_t; pub extern "c" fn pthread_join(thread: pthread_t, arg_return: ?*?*c_void) c_int; +pub extern "c" fn pthread_atfork( + prepare: ?fn () callconv(.C) void, + parent: ?fn () callconv(.C) void, + child: ?fn () callconv(.C) void, +) c_int; pub extern "c" fn kqueue() c_int; pub extern "c" fn kevent( @@ -336,6 +342,8 @@ pub extern "c" fn prctl(option: c_int, ...) c_int; pub extern "c" fn getrlimit(resource: rlimit_resource, rlim: *rlimit) c_int; pub extern "c" fn setrlimit(resource: rlimit_resource, rlim: *const rlimit) c_int; +pub extern "c" fn fmemopen(noalias buf: ?*c_void, size: usize, noalias mode: [*:0]const u8) ?*FILE; + pub const max_align_t = if (std.Target.current.abi == .msvc) f64 else if (std.Target.current.isDarwin()) diff --git a/lib/std/c/builtins.zig b/lib/std/c/builtins.zig new file mode 100644 index 0000000000..3dd6e23b91 --- /dev/null +++ b/lib/std/c/builtins.zig @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +const std = @import("std"); + +pub inline fn __builtin_bswap16(val: u16) callconv(.C) u16 { return @byteSwap(u16, val); } +pub inline fn __builtin_bswap32(val: u32) callconv(.C) u32 { return @byteSwap(u32, val); } +pub inline fn __builtin_bswap64(val: u64) callconv(.C) u64 { return @byteSwap(u64, val); } + +pub inline fn __builtin_signbit(val: f64) callconv(.C) c_int { return @boolToInt(std.math.signbit(val)); } +pub inline fn __builtin_signbitf(val: f32) callconv(.C) c_int { return @boolToInt(std.math.signbit(val)); } + +pub inline fn __builtin_popcount(val: c_uint) callconv(.C) c_int { + // popcount of a c_uint will never exceed the capacity of a c_int + @setRuntimeSafety(false); + return @bitCast(c_int, @as(c_uint, @popCount(c_uint, val))); +} +pub inline fn __builtin_ctz(val: c_uint) callconv(.C) c_int { + // Returns the number of trailing 0-bits in val, starting at the least significant bit position. + // In C if `val` is 0, the result is undefined; in zig it's the number of bits in a c_uint + @setRuntimeSafety(false); + return @bitCast(c_int, @as(c_uint, @ctz(c_uint, val))); +} +pub inline fn __builtin_clz(val: c_uint) callconv(.C) c_int { + // Returns the number of leading 0-bits in x, starting at the most significant bit position. + // In C if `val` is 0, the result is undefined; in zig it's the number of bits in a c_uint + @setRuntimeSafety(false); + return @bitCast(c_int, @as(c_uint, @clz(c_uint, val))); +} + +pub inline fn __builtin_sqrt(val: f64) callconv(.C) f64 { return @sqrt(val); } +pub inline fn __builtin_sqrtf(val: f32) callconv(.C) f32 { return @sqrt(val); } + +pub inline fn __builtin_sin(val: f64) callconv(.C) f64 { return @sin(val); } +pub inline fn __builtin_sinf(val: f32) callconv(.C) f32 { return @sin(val); } +pub inline fn __builtin_cos(val: f64) callconv(.C) f64 { return @cos(val); } +pub inline fn __builtin_cosf(val: f32) callconv(.C) f32 { return @cos(val); } + +pub inline fn __builtin_exp(val: f64) callconv(.C) f64 { return @exp(val); } +pub inline fn __builtin_expf(val: f32) callconv(.C) f32 { return @exp(val); } +pub inline fn __builtin_exp2(val: f64) callconv(.C) f64 { return @exp2(val); } +pub inline fn __builtin_exp2f(val: f32) callconv(.C) f32 { return @exp2(val); } +pub inline fn __builtin_log(val: f64) callconv(.C) f64 { return @log(val); } +pub inline fn __builtin_logf(val: f32) callconv(.C) f32 { return @log(val); } +pub inline fn __builtin_log2(val: f64) callconv(.C) f64 { return @log2(val); } +pub inline fn __builtin_log2f(val: f32) callconv(.C) f32 { return @log2(val); } +pub inline fn __builtin_log10(val: f64) callconv(.C) f64 { return @log10(val); } +pub inline fn __builtin_log10f(val: f32) callconv(.C) f32 { return @log10(val); } + +// Standard C Library bug: The absolute value of the most negative integer remains negative. +pub inline fn __builtin_abs(val: c_int) callconv(.C) c_int { return std.math.absInt(val) catch std.math.minInt(c_int); } +pub inline fn __builtin_fabs(val: f64) callconv(.C) f64 { return @fabs(val); } +pub inline fn __builtin_fabsf(val: f32) callconv(.C) f32 { return @fabs(val); } + +pub inline fn __builtin_floor(val: f64) callconv(.C) f64 { return @floor(val); } +pub inline fn __builtin_floorf(val: f32) callconv(.C) f32 { return @floor(val); } +pub inline fn __builtin_ceil(val: f64) callconv(.C) f64 { return @ceil(val); } +pub inline fn __builtin_ceilf(val: f32) callconv(.C) f32 { return @ceil(val); } +pub inline fn __builtin_trunc(val: f64) callconv(.C) f64 { return @trunc(val); } +pub inline fn __builtin_truncf(val: f32) callconv(.C) f32 { return @trunc(val); } +pub inline fn __builtin_round(val: f64) callconv(.C) f64 { return @round(val); } +pub inline fn __builtin_roundf(val: f32) callconv(.C) f32 { return @round(val); } + +pub inline fn __builtin_strlen(s: [*c]const u8) callconv(.C) usize { return std.mem.lenZ(s); } +pub inline fn __builtin_strcmp(s1: [*c]const u8, s2: [*c]const u8) callconv(.C) c_int { + return @as(c_int, std.cstr.cmp(s1, s2)); +} + +pub inline fn __builtin_object_size(ptr: ?*const c_void, ty: c_int) callconv(.C) usize { + // clang semantics match gcc's: https://gcc.gnu.org/onlinedocs/gcc/Object-Size-Checking.html + // If it is not possible to determine which objects ptr points to at compile time, + // __builtin_object_size should return (size_t) -1 for type 0 or 1 and (size_t) 0 + // for type 2 or 3. + if (ty == 0 or ty == 1) return @bitCast(usize, -@as(c_long, 1)); + if (ty == 2 or ty == 3) return 0; + unreachable; +} + +pub inline fn __builtin___memset_chk( + dst: ?*c_void, + val: c_int, + len: usize, + remaining: usize, +) callconv(.C) ?*c_void { + if (len > remaining) @panic("std.c.builtins.memset_chk called with len > remaining"); + return __builtin_memset(dst, val, len); +} + +pub inline fn __builtin_memset(dst: ?*c_void, val: c_int, len: usize) callconv(.C) ?*c_void { + const dst_cast = @ptrCast([*c]u8, dst); + @memset(dst_cast, @bitCast(u8, @truncate(i8, val)), len); + return dst; +} + +pub inline fn __builtin___memcpy_chk( + noalias dst: ?*c_void, + noalias src: ?*const c_void, + len: usize, + remaining: usize, +) callconv(.C) ?*c_void { + if (len > remaining) @panic("std.c.builtins.memcpy_chk called with len > remaining"); + return __builtin_memcpy(dst, src, len); +} + +pub inline fn __builtin_memcpy( + noalias dst: ?*c_void, + noalias src: ?*const c_void, + len: usize, +) callconv(.C) ?*c_void { + const dst_cast = @ptrCast([*c]u8, dst); + const src_cast = @ptrCast([*c]const u8, src); + + @memcpy(dst_cast, src_cast, len); + return dst; +} diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index 21124d1030..97a25617ef 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -106,6 +106,12 @@ pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *con pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; pub extern "c" fn malloc_usable_size(?*const c_void) usize; +pub extern "c" fn madvise( + addr: *align(std.mem.page_size) c_void, + length: usize, + advice: c_uint, +) c_int; + pub const pthread_attr_t = extern struct { __size: [56]u8, __align: c_long, diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 6eb934473f..e3581cde96 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -134,8 +134,10 @@ pub const nacl = struct { pub const utils = @import("crypto/utils.zig"); +/// This is a thread-local, cryptographically secure pseudo random number generator. +pub const random = &@import("crypto/tlcsprng.zig").interface; + const std = @import("std.zig"); -pub const randomBytes = std.os.getrandom; test "crypto" { inline for (std.meta.declarations(@This())) |decl| { @@ -178,6 +180,13 @@ test "crypto" { _ = @import("crypto/25519/ristretto255.zig"); } +test "CSPRNG" { + const a = random.int(u64); + const b = random.int(u64); + const c = random.int(u64); + std.testing.expect(a ^ b ^ c != 0); +} + test "issue #4532: no index out of bounds" { const types = [_]type{ hash.Md5, diff --git a/lib/std/crypto/25519/ed25519.zig b/lib/std/crypto/25519/ed25519.zig index 842b08d706..7f90ba584c 100644 --- a/lib/std/crypto/25519/ed25519.zig +++ b/lib/std/crypto/25519/ed25519.zig @@ -43,7 +43,7 @@ pub const Ed25519 = struct { pub fn create(seed: ?[seed_length]u8) !KeyPair { const ss = seed orelse ss: { var random_seed: [seed_length]u8 = undefined; - try crypto.randomBytes(&random_seed); + crypto.random.bytes(&random_seed); break :ss random_seed; }; var az: [Sha512.digest_length]u8 = undefined; @@ -179,7 +179,7 @@ pub const Ed25519 = struct { var z_batch: [count]Curve.scalar.CompressedScalar = undefined; for (z_batch) |*z| { - try std.crypto.randomBytes(z[0..16]); + std.crypto.random.bytes(z[0..16]); mem.set(u8, z[16..], 0); } @@ -232,8 +232,8 @@ test "ed25519 batch verification" { const key_pair = try Ed25519.KeyPair.create(null); var msg1: [32]u8 = undefined; var msg2: [32]u8 = undefined; - try std.crypto.randomBytes(&msg1); - try std.crypto.randomBytes(&msg2); + std.crypto.random.bytes(&msg1); + std.crypto.random.bytes(&msg2); const sig1 = try Ed25519.sign(&msg1, key_pair, null); const sig2 = try Ed25519.sign(&msg2, key_pair, null); var signature_batch = [_]Ed25519.BatchElement{ diff --git a/lib/std/crypto/25519/edwards25519.zig b/lib/std/crypto/25519/edwards25519.zig index 008a4535b3..d64f06c421 100644 --- a/lib/std/crypto/25519/edwards25519.zig +++ b/lib/std/crypto/25519/edwards25519.zig @@ -484,8 +484,8 @@ test "edwards25519 packing/unpacking" { test "edwards25519 point addition/substraction" { var s1: [32]u8 = undefined; var s2: [32]u8 = undefined; - try std.crypto.randomBytes(&s1); - try std.crypto.randomBytes(&s2); + std.crypto.random.bytes(&s1); + std.crypto.random.bytes(&s2); const p = try Edwards25519.basePoint.clampedMul(s1); const q = try Edwards25519.basePoint.clampedMul(s2); const r = p.add(q).add(q).sub(q).sub(q); diff --git a/lib/std/crypto/25519/x25519.zig b/lib/std/crypto/25519/x25519.zig index 3b3ff551fe..0bf55d52fc 100644 --- a/lib/std/crypto/25519/x25519.zig +++ b/lib/std/crypto/25519/x25519.zig @@ -34,7 +34,7 @@ pub const X25519 = struct { pub fn create(seed: ?[seed_length]u8) !KeyPair { const sk = seed orelse sk: { var random_seed: [seed_length]u8 = undefined; - try crypto.randomBytes(&random_seed); + crypto.random.bytes(&random_seed); break :sk random_seed; }; var kp: KeyPair = undefined; diff --git a/lib/std/crypto/bcrypt.zig b/lib/std/crypto/bcrypt.zig index 4cec59961b..6813495d25 100644 --- a/lib/std/crypto/bcrypt.zig +++ b/lib/std/crypto/bcrypt.zig @@ -262,7 +262,7 @@ fn strHashInternal(password: []const u8, rounds_log: u6, salt: [salt_length]u8) /// and then use the resulting hash as the password parameter for bcrypt. pub fn strHash(password: []const u8, rounds_log: u6) ![hash_length]u8 { var salt: [salt_length]u8 = undefined; - try crypto.randomBytes(&salt); + crypto.random.bytes(&salt); return strHashInternal(password, rounds_log, salt); } @@ -283,7 +283,7 @@ pub fn strVerify(h: [hash_length]u8, password: []const u8) BcryptError!void { test "bcrypt codec" { var salt: [salt_length]u8 = undefined; - try crypto.randomBytes(&salt); + crypto.random.bytes(&salt); var salt_str: [salt_str_length]u8 = undefined; Codec.encode(salt_str[0..], salt[0..]); var salt2: [salt_length]u8 = undefined; diff --git a/lib/std/crypto/gimli.zig b/lib/std/crypto/gimli.zig index 78ab88b9cf..f21bc1008a 100644 --- a/lib/std/crypto/gimli.zig +++ b/lib/std/crypto/gimli.zig @@ -229,18 +229,17 @@ pub const Hash = struct { const buf = self.state.toSlice(); var in = data; while (in.len > 0) { - var left = State.RATE - self.buf_off; - if (left == 0) { - self.state.permute(); - self.buf_off = 0; - left = State.RATE; - } + const left = State.RATE - self.buf_off; const ps = math.min(in.len, left); for (buf[self.buf_off .. self.buf_off + ps]) |*p, i| { p.* ^= in[i]; } self.buf_off += ps; in = in[ps..]; + if (self.buf_off == State.RATE) { + self.state.permute(); + self.buf_off = 0; + } } } @@ -277,6 +276,22 @@ test "hash" { htest.assertEqual("1C9A03DC6A5DDC5444CFC6F4B154CFF5CF081633B2CEA4D7D0AE7CCFED5AAA44", &md); } +test "hash test vector 17" { + var msg: [32 / 2]u8 = undefined; + try std.fmt.hexToBytes(&msg, "000102030405060708090A0B0C0D0E0F"); + var md: [32]u8 = undefined; + hash(&md, &msg, .{}); + htest.assertEqual("404C130AF1B9023A7908200919F690FFBB756D5176E056FFDE320016A37C7282", &md); +} + +test "hash test vector 33" { + var msg: [32]u8 = undefined; + try std.fmt.hexToBytes(&msg, "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F"); + var md: [32]u8 = undefined; + hash(&md, &msg, .{}); + htest.assertEqual("A8F4FA28708BDA7EFB4C1914CA4AFA9E475B82D588D36504F87DBB0ED9AB3C4B", &md); +} + pub const Aead = struct { pub const tag_length = State.RATE; pub const nonce_length = 16; diff --git a/lib/std/crypto/salsa20.zig b/lib/std/crypto/salsa20.zig index dd3e4fe99b..8122e9b25c 100644 --- a/lib/std/crypto/salsa20.zig +++ b/lib/std/crypto/salsa20.zig @@ -571,9 +571,9 @@ test "xsalsa20poly1305" { var key: [XSalsa20Poly1305.key_length]u8 = undefined; var nonce: [XSalsa20Poly1305.nonce_length]u8 = undefined; var tag: [XSalsa20Poly1305.tag_length]u8 = undefined; - try crypto.randomBytes(&msg); - try crypto.randomBytes(&key); - try crypto.randomBytes(&nonce); + crypto.random.bytes(&msg); + crypto.random.bytes(&key); + crypto.random.bytes(&nonce); XSalsa20Poly1305.encrypt(c[0..], &tag, msg[0..], "ad", nonce, key); try XSalsa20Poly1305.decrypt(msg2[0..], c[0..], tag, "ad", nonce, key); @@ -585,9 +585,9 @@ test "xsalsa20poly1305 secretbox" { var key: [XSalsa20Poly1305.key_length]u8 = undefined; var nonce: [Box.nonce_length]u8 = undefined; var boxed: [msg.len + Box.tag_length]u8 = undefined; - try crypto.randomBytes(&msg); - try crypto.randomBytes(&key); - try crypto.randomBytes(&nonce); + crypto.random.bytes(&msg); + crypto.random.bytes(&key); + crypto.random.bytes(&nonce); SecretBox.seal(boxed[0..], msg[0..], nonce, key); try SecretBox.open(msg2[0..], boxed[0..], nonce, key); @@ -598,8 +598,8 @@ test "xsalsa20poly1305 box" { var msg2: [msg.len]u8 = undefined; var nonce: [Box.nonce_length]u8 = undefined; var boxed: [msg.len + Box.tag_length]u8 = undefined; - try crypto.randomBytes(&msg); - try crypto.randomBytes(&nonce); + crypto.random.bytes(&msg); + crypto.random.bytes(&nonce); var kp1 = try Box.KeyPair.create(null); var kp2 = try Box.KeyPair.create(null); @@ -611,7 +611,7 @@ test "xsalsa20poly1305 sealedbox" { var msg: [100]u8 = undefined; var msg2: [msg.len]u8 = undefined; var boxed: [msg.len + SealedBox.seal_length]u8 = undefined; - try crypto.randomBytes(&msg); + crypto.random.bytes(&msg); var kp = try Box.KeyPair.create(null); try SealedBox.seal(boxed[0..], msg[0..], kp.public_key); diff --git a/lib/std/crypto/tlcsprng.zig b/lib/std/crypto/tlcsprng.zig new file mode 100644 index 0000000000..384216a81b --- /dev/null +++ b/lib/std/crypto/tlcsprng.zig @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! Thread-local cryptographically secure pseudo-random number generator. +//! This file has public declarations that are intended to be used internally +//! by the standard library; this namespace is not intended to be exposed +//! directly to standard library users. + +const std = @import("std"); +const root = @import("root"); +const mem = std.mem; + +/// We use this as a layer of indirection because global const pointers cannot +/// point to thread-local variables. +pub var interface = std.rand.Random{ .fillFn = tlsCsprngFill }; + +const os_has_fork = switch (std.Target.current.os.tag) { + .dragonfly, + .freebsd, + .ios, + .kfreebsd, + .linux, + .macos, + .netbsd, + .openbsd, + .solaris, + .tvos, + .watchos, + => true, + + else => false, +}; +const os_has_arc4random = std.builtin.link_libc and @hasDecl(std.c, "arc4random_buf"); +const want_fork_safety = os_has_fork and !os_has_arc4random and + (std.meta.globalOption("crypto_fork_safety", bool) orelse true); +const maybe_have_wipe_on_fork = std.Target.current.os.isAtLeast(.linux, .{ + .major = 4, + .minor = 14, +}) orelse true; + +const WipeMe = struct { + init_state: enum { uninitialized, initialized, failed }, + gimli: std.crypto.core.Gimli, +}; +const wipe_align = if (maybe_have_wipe_on_fork) mem.page_size else @alignOf(WipeMe); + +threadlocal var wipe_me: WipeMe align(wipe_align) = .{ + .gimli = undefined, + .init_state = .uninitialized, +}; + +fn tlsCsprngFill(_: *const std.rand.Random, buffer: []u8) void { + if (std.builtin.link_libc and @hasDecl(std.c, "arc4random_buf")) { + // arc4random is already a thread-local CSPRNG. + return std.c.arc4random_buf(buffer.ptr, buffer.len); + } + // Allow applications to decide they would prefer to have every call to + // std.crypto.random always make an OS syscall, rather than rely on an + // application implementation of a CSPRNG. + if (comptime std.meta.globalOption("crypto_always_getrandom", bool) orelse false) { + return fillWithOsEntropy(buffer); + } + switch (wipe_me.init_state) { + .uninitialized => { + if (want_fork_safety) { + if (maybe_have_wipe_on_fork) { + if (std.os.madvise( + @ptrCast([*]align(mem.page_size) u8, &wipe_me), + @sizeOf(@TypeOf(wipe_me)), + std.os.MADV_WIPEONFORK, + )) |_| { + return initAndFill(buffer); + } else |_| if (std.Thread.use_pthreads) { + return setupPthreadAtforkAndFill(buffer); + } else { + // Since we failed to set up fork safety, we fall back to always + // calling getrandom every time. + wipe_me.init_state = .failed; + return fillWithOsEntropy(buffer); + } + } else if (std.Thread.use_pthreads) { + return setupPthreadAtforkAndFill(buffer); + } else { + // We have no mechanism to provide fork safety, but we want fork safety, + // so we fall back to calling getrandom every time. + wipe_me.init_state = .failed; + return fillWithOsEntropy(buffer); + } + } else { + return initAndFill(buffer); + } + }, + .initialized => { + return fillWithCsprng(buffer); + }, + .failed => { + if (want_fork_safety) { + return fillWithOsEntropy(buffer); + } else { + unreachable; + } + }, + } +} + +fn setupPthreadAtforkAndFill(buffer: []u8) void { + const failed = std.c.pthread_atfork(null, null, childAtForkHandler) != 0; + if (failed) { + wipe_me.init_state = .failed; + return fillWithOsEntropy(buffer); + } else { + return initAndFill(buffer); + } +} + +fn childAtForkHandler() callconv(.C) void { + const wipe_slice = @ptrCast([*]u8, &wipe_me)[0..@sizeOf(@TypeOf(wipe_me))]; + std.crypto.utils.secureZero(u8, wipe_slice); +} + +fn fillWithCsprng(buffer: []u8) void { + if (buffer.len != 0) { + wipe_me.gimli.squeeze(buffer); + } else { + wipe_me.gimli.permute(); + } + mem.set(u8, wipe_me.gimli.toSlice()[0..std.crypto.core.Gimli.RATE], 0); +} + +fn fillWithOsEntropy(buffer: []u8) void { + std.os.getrandom(buffer) catch @panic("getrandom() failed to provide entropy"); +} + +fn initAndFill(buffer: []u8) void { + var seed: [std.crypto.core.Gimli.BLOCKBYTES]u8 = undefined; + // Because we panic on getrandom() failing, we provide the opportunity + // to override the default seed function. This also makes + // `std.crypto.random` available on freestanding targets, provided that + // the `cryptoRandomSeed` function is provided. + if (@hasDecl(root, "cryptoRandomSeed")) { + root.cryptoRandomSeed(&seed); + } else { + fillWithOsEntropy(&seed); + } + + wipe_me.gimli = std.crypto.core.Gimli.init(seed); + + // This is at the end so that accidental recursive dependencies result + // in stack overflows instead of invalid random data. + wipe_me.init_state = .initialized; + + return fillWithCsprng(buffer); +} diff --git a/lib/std/crypto/utils.zig b/lib/std/crypto/utils.zig index 33ad6360f6..08271ac9f4 100644 --- a/lib/std/crypto/utils.zig +++ b/lib/std/crypto/utils.zig @@ -51,8 +51,8 @@ pub fn secureZero(comptime T: type, s: []T) void { test "crypto.utils.timingSafeEql" { var a: [100]u8 = undefined; var b: [100]u8 = undefined; - try std.crypto.randomBytes(a[0..]); - try std.crypto.randomBytes(b[0..]); + std.crypto.random.bytes(a[0..]); + std.crypto.random.bytes(b[0..]); testing.expect(!timingSafeEql([100]u8, a, b)); mem.copy(u8, a[0..], b[0..]); testing.expect(timingSafeEql([100]u8, a, b)); @@ -61,8 +61,8 @@ test "crypto.utils.timingSafeEql" { test "crypto.utils.timingSafeEql (vectors)" { var a: [100]u8 = undefined; var b: [100]u8 = undefined; - try std.crypto.randomBytes(a[0..]); - try std.crypto.randomBytes(b[0..]); + std.crypto.random.bytes(a[0..]); + std.crypto.random.bytes(b[0..]); const v1: std.meta.Vector(100, u8) = a; const v2: std.meta.Vector(100, u8) = b; testing.expect(!timingSafeEql(std.meta.Vector(100, u8), v1, v2)); diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 4f0f44d1b7..7284237cb2 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -1696,6 +1696,7 @@ fn getDebugInfoAllocator() *mem.Allocator { pub const have_segfault_handling_support = switch (builtin.os.tag) { .linux, .netbsd => true, .windows => true, + .freebsd, .openbsd => @hasDecl(os, "ucontext_t"), else => false, }; pub const enable_segfault_handler: bool = if (@hasDecl(root, "enable_segfault_handler")) @@ -1721,7 +1722,7 @@ pub fn attachSegfaultHandler() void { return; } var act = os.Sigaction{ - .sigaction = handleSegfaultLinux, + .handler = .{ .sigaction = handleSegfaultLinux }, .mask = os.empty_sigset, .flags = (os.SA_SIGINFO | os.SA_RESTART | os.SA_RESETHAND), }; @@ -1740,7 +1741,7 @@ fn resetSegfaultHandler() void { return; } var act = os.Sigaction{ - .sigaction = os.SIG_DFL, + .handler = .{ .sigaction = os.SIG_DFL }, .mask = os.empty_sigset, .flags = 0, }; @@ -1757,7 +1758,9 @@ fn handleSegfaultLinux(sig: i32, info: *const os.siginfo_t, ctx_ptr: ?*const c_v const addr = switch (builtin.os.tag) { .linux => @ptrToInt(info.fields.sigfault.addr), + .freebsd => @ptrToInt(info.addr), .netbsd => @ptrToInt(info.info.reason.fault.addr), + .openbsd => @ptrToInt(info.data.fault.addr), else => unreachable, }; @@ -1781,8 +1784,18 @@ fn handleSegfaultLinux(sig: i32, info: *const os.siginfo_t, ctx_ptr: ?*const c_v }, .x86_64 => { const ctx = @ptrCast(*const os.ucontext_t, @alignCast(@alignOf(os.ucontext_t), ctx_ptr)); - const ip = @intCast(usize, ctx.mcontext.gregs[os.REG_RIP]); - const bp = @intCast(usize, ctx.mcontext.gregs[os.REG_RBP]); + const ip = switch (builtin.os.tag) { + .linux, .netbsd => @intCast(usize, ctx.mcontext.gregs[os.REG_RIP]), + .freebsd => @intCast(usize, ctx.mcontext.rip), + .openbsd => @intCast(usize, ctx.sc_rip), + else => unreachable, + }; + const bp = switch (builtin.os.tag) { + .linux, .netbsd => @intCast(usize, ctx.mcontext.gregs[os.REG_RBP]), + .openbsd => @intCast(usize, ctx.sc_rbp), + .freebsd => @intCast(usize, ctx.mcontext.rbp), + else => unreachable, + }; dumpStackTraceFromBase(bp, ip); }, .arm => { diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 8b949a57f1..4c346a2c89 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -82,7 +82,7 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path: mem.copy(u8, tmp_path[0..], dirname); tmp_path[dirname.len] = path.sep; while (true) { - try crypto.randomBytes(rand_buf[0..]); + crypto.random.bytes(rand_buf[0..]); base64_encoder.encode(tmp_path[dirname.len + 1 ..], &rand_buf); if (cwd().symLink(existing_path, tmp_path, .{})) { @@ -157,7 +157,7 @@ pub const AtomicFile = struct { tmp_path_buf[base64.Base64Encoder.calcSize(RANDOM_BYTES)] = 0; while (true) { - try crypto.randomBytes(rand_buf[0..]); + crypto.random.bytes(rand_buf[0..]); base64_encoder.encode(&tmp_path_buf, &rand_buf); const file = dir.createFile( diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index e97be007ab..469215e2b3 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -698,7 +698,8 @@ pub const File = struct { error.FastOpenAlreadyInProgress, error.MessageTooBig, error.FileDescriptorNotASocket, - error.AddressFamilyNotSupported, + error.NetworkUnreachable, + error.NetworkSubsystemFailed, => return self.writeFileAllUnseekable(in_file, args), else => |e| return e, diff --git a/lib/std/heap.zig b/lib/std/heap.zig index a0484d4fdc..fddef10cd3 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -789,7 +789,7 @@ pub fn stackFallback(comptime size: usize, fallback_allocator: *Allocator) Stack .fallback_allocator = fallback_allocator, .fixed_buffer_allocator = undefined, .allocator = Allocator{ - .allocFn = StackFallbackAllocator(size).realloc, + .allocFn = StackFallbackAllocator(size).alloc, .resizeFn = StackFallbackAllocator(size).resize, }, }; @@ -815,25 +815,25 @@ pub fn StackFallbackAllocator(comptime size: usize) type { ptr_align: u29, len_align: u29, return_address: usize, - ) error{OutOfMemory}![*]u8 { + ) error{OutOfMemory}![]u8 { const self = @fieldParentPtr(Self, "allocator", allocator); - return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align) catch - return fallback_allocator.alloc(len, ptr_align); + return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator.allocator, len, ptr_align, len_align, return_address) catch + return self.fallback_allocator.allocFn(self.fallback_allocator, len, ptr_align, len_align, return_address); } fn resize( - self: *Allocator, + allocator: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, return_address: usize, - ) error{OutOfMemory}!void { + ) error{OutOfMemory}!usize { const self = @fieldParentPtr(Self, "allocator", allocator); if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) { - try self.fixed_buffer_allocator.resize(buf, new_len); + return FixedBufferAllocator.resize(&self.fixed_buffer_allocator.allocator, buf, buf_align, new_len, len_align, return_address); } else { - try self.fallback_allocator.resize(buf, new_len); + return self.fallback_allocator.resizeFn(self.fallback_allocator, buf, buf_align, new_len, len_align, return_address); } } }; @@ -970,6 +970,16 @@ test "FixedBufferAllocator.reset" { testing.expect(y.* == Y); } +test "StackFallbackAllocator" { + const fallback_allocator = page_allocator; + var stack_allocator = stackFallback(4096, fallback_allocator); + + try testAllocator(stack_allocator.get()); + try testAllocatorAligned(stack_allocator.get()); + try testAllocatorLargeAlignment(stack_allocator.get()); + try testAllocatorAlignedShrink(stack_allocator.get()); +} + test "FixedBufferAllocator Reuse memory on realloc" { var small_fixed_buffer: [10]u8 = undefined; // check if we re-use the memory diff --git a/lib/std/io.zig b/lib/std/io.zig index 3f02128a6c..103c443dd6 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -143,6 +143,8 @@ pub const cOutStream = cWriter; pub const CountingWriter = @import("io/counting_writer.zig").CountingWriter; pub const countingWriter = @import("io/counting_writer.zig").countingWriter; +pub const CountingReader = @import("io/counting_reader.zig").CountingReader; +pub const countingReader = @import("io/counting_reader.zig").countingReader; /// Deprecated: use `CountingWriter` pub const CountingOutStream = CountingWriter; /// Deprecated: use `countingWriter` @@ -178,14 +180,6 @@ pub const changeDetectionStream = @import("io/change_detection_stream.zig").chan pub const FindByteOutStream = @import("io/find_byte_out_stream.zig").FindByteOutStream; pub const findByteOutStream = @import("io/find_byte_out_stream.zig").findByteOutStream; -pub const Packing = @import("io/serialization.zig").Packing; - -pub const Serializer = @import("io/serialization.zig").Serializer; -pub const serializer = @import("io/serialization.zig").serializer; - -pub const Deserializer = @import("io/serialization.zig").Deserializer; -pub const deserializer = @import("io/serialization.zig").deserializer; - pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; pub const StreamSource = @import("io/stream_source.zig").StreamSource; @@ -220,7 +214,6 @@ test "" { _ = @import("io/writer.zig"); _ = @import("io/peek_stream.zig"); _ = @import("io/seekable_stream.zig"); - _ = @import("io/serialization.zig"); _ = @import("io/stream_source.zig"); _ = @import("io/test.zig"); } diff --git a/lib/std/io/counting_reader.zig b/lib/std/io/counting_reader.zig new file mode 100644 index 0000000000..09a624952f --- /dev/null +++ b/lib/std/io/counting_reader.zig @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +const std = @import("../std.zig"); +const io = std.io; +const testing = std.testing; + +/// A Reader that counts how many bytes has been read from it. +pub fn CountingReader(comptime ReaderType: anytype) type { + return struct { + child_reader: ReaderType, + bytes_read: u64 = 0, + + pub const Error = ReaderType.Error; + pub const Reader = io.Reader(*@This(), Error, read); + + pub fn read(self: *@This(), buf: []u8) Error!usize { + const amt = try self.child_reader.read(buf); + self.bytes_read += amt; + return amt; + } + + pub fn reader(self: *@This()) Reader { + return .{ .context = self }; + } + }; +} + +pub fn countingReader(reader: anytype) CountingReader(@TypeOf(reader)) { + return .{ .child_reader = reader, }; +} + +test "io.CountingReader" { + const bytes = "yay" ** 100; + var fbs = io.fixedBufferStream(bytes); + + var counting_stream = countingReader(fbs.reader()); + const stream = counting_stream.reader(); + + //read and discard all bytes + while(stream.readByte()) |_| {} else |err| { + testing.expect(err == error.EndOfStream); + } + + testing.expect(counting_stream.bytes_read == bytes.len); +}
\ No newline at end of file diff --git a/lib/std/io/serialization.zig b/lib/std/io/serialization.zig deleted file mode 100644 index 3fbc203242..0000000000 --- a/lib/std/io/serialization.zig +++ /dev/null @@ -1,616 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2015-2020 Zig Contributors -// This file is part of [zig](https://ziglang.org/), which is MIT licensed. -// The MIT license requires this copyright notice to be included in all copies -// and substantial portions of the software. -const std = @import("../std.zig"); -const builtin = std.builtin; -const io = std.io; -const assert = std.debug.assert; -const math = std.math; -const meta = std.meta; -const trait = meta.trait; -const testing = std.testing; - -pub const Packing = enum { - /// Pack data to byte alignment - Byte, - - /// Pack data to bit alignment - Bit, -}; - -/// Creates a deserializer that deserializes types from any stream. -/// If `is_packed` is true, the data stream is treated as bit-packed, -/// otherwise data is expected to be packed to the smallest byte. -/// Types may implement a custom deserialization routine with a -/// function named `deserialize` in the form of: -/// ``` -/// pub fn deserialize(self: *Self, deserializer: anytype) !void -/// ``` -/// which will be called when the deserializer is used to deserialize -/// that type. It will pass a pointer to the type instance to deserialize -/// into and a pointer to the deserializer struct. -pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime ReaderType: type) type { - return struct { - in_stream: if (packing == .Bit) io.BitReader(endian, ReaderType) else ReaderType, - - const Self = @This(); - - pub fn init(in_stream: ReaderType) Self { - return Self{ - .in_stream = switch (packing) { - .Bit => io.bitReader(endian, in_stream), - .Byte => in_stream, - }, - }; - } - - pub fn alignToByte(self: *Self) void { - if (packing == .Byte) return; - self.in_stream.alignToByte(); - } - - //@BUG: inferred error issue. See: #1386 - fn deserializeInt(self: *Self, comptime T: type) (ReaderType.Error || error{EndOfStream})!T { - comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); - - const u8_bit_count = 8; - const t_bit_count = comptime meta.bitCount(T); - - const U = std.meta.Int(.unsigned, t_bit_count); - const Log2U = math.Log2Int(U); - const int_size = (t_bit_count + 7) / 8; - - if (packing == .Bit) { - const result = try self.in_stream.readBitsNoEof(U, t_bit_count); - return @bitCast(T, result); - } - - var buffer: [int_size]u8 = undefined; - const read_size = try self.in_stream.read(buffer[0..]); - if (read_size < int_size) return error.EndOfStream; - - if (int_size == 1) { - if (t_bit_count == 8) return @bitCast(T, buffer[0]); - const PossiblySignedByte = std.meta.Int(@typeInfo(T).Int.signedness, 8); - return @truncate(T, @bitCast(PossiblySignedByte, buffer[0])); - } - - var result = @as(U, 0); - for (buffer) |byte, i| { - switch (endian) { - .Big => { - result = (result << u8_bit_count) | byte; - }, - .Little => { - result |= @as(U, byte) << @intCast(Log2U, u8_bit_count * i); - }, - } - } - - return @bitCast(T, result); - } - - /// Deserializes and returns data of the specified type from the stream - pub fn deserialize(self: *Self, comptime T: type) !T { - var value: T = undefined; - try self.deserializeInto(&value); - return value; - } - - /// Deserializes data into the type pointed to by `ptr` - pub fn deserializeInto(self: *Self, ptr: anytype) !void { - const T = @TypeOf(ptr); - comptime assert(trait.is(.Pointer)(T)); - - if (comptime trait.isSlice(T) or comptime trait.isPtrTo(.Array)(T)) { - for (ptr) |*v| - try self.deserializeInto(v); - return; - } - - comptime assert(trait.isSingleItemPtr(T)); - - const C = comptime meta.Child(T); - const child_type_id = @typeInfo(C); - - //custom deserializer: fn(self: *Self, deserializer: anytype) !void - if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self); - - if (comptime trait.isPacked(C) and packing != .Bit) { - var packed_deserializer = deserializer(endian, .Bit, self.in_stream); - return packed_deserializer.deserializeInto(ptr); - } - - switch (child_type_id) { - .Void => return, - .Bool => ptr.* = (try self.deserializeInt(u1)) > 0, - .Float, .Int => ptr.* = try self.deserializeInt(C), - .Struct => { - const info = @typeInfo(C).Struct; - - inline for (info.fields) |*field_info| { - const name = field_info.name; - const FieldType = field_info.field_type; - - if (FieldType == void or FieldType == u0) continue; - - //it doesn't make any sense to read pointers - if (comptime trait.is(.Pointer)(FieldType)) { - @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++ - @typeName(C) ++ " because it " ++ "is of pointer-type " ++ - @typeName(FieldType) ++ "."); - } - - try self.deserializeInto(&@field(ptr, name)); - } - }, - .Union => { - const info = @typeInfo(C).Union; - if (info.tag_type) |TagType| { - //we avoid duplicate iteration over the enum tags - // by getting the int directly and casting it without - // safety. If it is bad, it will be caught anyway. - const TagInt = @TagType(TagType); - const tag = try self.deserializeInt(TagInt); - - inline for (info.fields) |field_info| { - if (@enumToInt(@field(TagType, field_info.name)) == tag) { - const name = field_info.name; - const FieldType = field_info.field_type; - ptr.* = @unionInit(C, name, undefined); - try self.deserializeInto(&@field(ptr, name)); - return; - } - } - //This is reachable if the enum data is bad - return error.InvalidEnumTag; - } - @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++ - " because it is an untagged union. Use a custom deserialize()."); - }, - .Optional => { - const OC = comptime meta.Child(C); - const exists = (try self.deserializeInt(u1)) > 0; - if (!exists) { - ptr.* = null; - return; - } - - ptr.* = @as(OC, undefined); //make it non-null so the following .? is guaranteed safe - const val_ptr = &ptr.*.?; - try self.deserializeInto(val_ptr); - }, - .Enum => { - var value = try self.deserializeInt(@TagType(C)); - ptr.* = try meta.intToEnum(C, value); - }, - else => { - @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented)."); - }, - } - } - }; -} - -pub fn deserializer( - comptime endian: builtin.Endian, - comptime packing: Packing, - in_stream: anytype, -) Deserializer(endian, packing, @TypeOf(in_stream)) { - return Deserializer(endian, packing, @TypeOf(in_stream)).init(in_stream); -} - -/// Creates a serializer that serializes types to any stream. -/// If `is_packed` is true, the data will be bit-packed into the stream. -/// Note that the you must call `serializer.flush()` when you are done -/// writing bit-packed data in order ensure any unwritten bits are committed. -/// If `is_packed` is false, data is packed to the smallest byte. In the case -/// of packed structs, the struct will written bit-packed and with the specified -/// endianess, after which data will resume being written at the next byte boundary. -/// Types may implement a custom serialization routine with a -/// function named `serialize` in the form of: -/// ``` -/// pub fn serialize(self: Self, serializer: anytype) !void -/// ``` -/// which will be called when the serializer is used to serialize that type. It will -/// pass a const pointer to the type instance to be serialized and a pointer -/// to the serializer struct. -pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime OutStreamType: type) type { - return struct { - out_stream: if (packing == .Bit) io.BitOutStream(endian, OutStreamType) else OutStreamType, - - const Self = @This(); - pub const Error = OutStreamType.Error; - - pub fn init(out_stream: OutStreamType) Self { - return Self{ - .out_stream = switch (packing) { - .Bit => io.bitOutStream(endian, out_stream), - .Byte => out_stream, - }, - }; - } - - /// Flushes any unwritten bits to the stream - pub fn flush(self: *Self) Error!void { - if (packing == .Bit) return self.out_stream.flushBits(); - } - - fn serializeInt(self: *Self, value: anytype) Error!void { - const T = @TypeOf(value); - comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); - - const t_bit_count = comptime meta.bitCount(T); - const u8_bit_count = comptime meta.bitCount(u8); - - const U = std.meta.Int(.unsigned, t_bit_count); - const Log2U = math.Log2Int(U); - const int_size = (t_bit_count + 7) / 8; - - const u_value = @bitCast(U, value); - - if (packing == .Bit) return self.out_stream.writeBits(u_value, t_bit_count); - - var buffer: [int_size]u8 = undefined; - if (int_size == 1) buffer[0] = u_value; - - for (buffer) |*byte, i| { - const idx = switch (endian) { - .Big => int_size - i - 1, - .Little => i, - }; - const shift = @intCast(Log2U, idx * u8_bit_count); - const v = u_value >> shift; - byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v); - } - - try self.out_stream.writeAll(&buffer); - } - - /// Serializes the passed value into the stream - pub fn serialize(self: *Self, value: anytype) Error!void { - const T = comptime @TypeOf(value); - - if (comptime trait.isIndexable(T)) { - for (value) |v| - try self.serialize(v); - return; - } - - //custom serializer: fn(self: Self, serializer: anytype) !void - if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self); - - if (comptime trait.isPacked(T) and packing != .Bit) { - var packed_serializer = Serializer(endian, .Bit, OutStreamType).init(self.out_stream); - try packed_serializer.serialize(value); - try packed_serializer.flush(); - return; - } - - switch (@typeInfo(T)) { - .Void => return, - .Bool => try self.serializeInt(@as(u1, @boolToInt(value))), - .Float, .Int => try self.serializeInt(value), - .Struct => { - const info = @typeInfo(T); - - inline for (info.Struct.fields) |*field_info| { - const name = field_info.name; - const FieldType = field_info.field_type; - - if (FieldType == void or FieldType == u0) continue; - - //It doesn't make sense to write pointers - if (comptime trait.is(.Pointer)(FieldType)) { - @compileError("Will not " ++ "serialize field " ++ name ++ - " of struct " ++ @typeName(T) ++ " because it " ++ - "is of pointer-type " ++ @typeName(FieldType) ++ "."); - } - try self.serialize(@field(value, name)); - } - }, - .Union => { - const info = @typeInfo(T).Union; - if (info.tag_type) |TagType| { - const active_tag = meta.activeTag(value); - try self.serialize(active_tag); - //This inline loop is necessary because active_tag is a runtime - // value, but @field requires a comptime value. Our alternative - // is to check each field for a match - inline for (info.fields) |field_info| { - if (@field(TagType, field_info.name) == active_tag) { - const name = field_info.name; - const FieldType = field_info.field_type; - try self.serialize(@field(value, name)); - return; - } - } - unreachable; - } - @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++ - " because it is an untagged union. Use a custom serialize()."); - }, - .Optional => { - if (value == null) { - try self.serializeInt(@as(u1, @boolToInt(false))); - return; - } - try self.serializeInt(@as(u1, @boolToInt(true))); - - const OC = comptime meta.Child(T); - const val_ptr = &value.?; - try self.serialize(val_ptr.*); - }, - .Enum => { - try self.serializeInt(@enumToInt(value)); - }, - else => @compileError("Cannot serialize " ++ @tagName(@typeInfo(T)) ++ " types (unimplemented)."), - } - } - }; -} - -pub fn serializer( - comptime endian: builtin.Endian, - comptime packing: Packing, - out_stream: anytype, -) Serializer(endian, packing, @TypeOf(out_stream)) { - return Serializer(endian, packing, @TypeOf(out_stream)).init(out_stream); -} - -fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - @setEvalBranchQuota(1500); - //@NOTE: if this test is taking too long, reduce the maximum tested bitsize - const max_test_bitsize = 128; - - const total_bytes = comptime blk: { - var bytes = 0; - comptime var i = 0; - while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0); - break :blk bytes * 2; - }; - - var data_mem: [total_bytes]u8 = undefined; - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - comptime var i = 0; - inline while (i <= max_test_bitsize) : (i += 1) { - const U = std.meta.Int(.unsigned, i); - const S = std.meta.Int(.signed, i); - try _serializer.serializeInt(@as(U, i)); - if (i != 0) try _serializer.serializeInt(@as(S, -1)) else try _serializer.serialize(@as(S, 0)); - } - try _serializer.flush(); - - i = 0; - inline while (i <= max_test_bitsize) : (i += 1) { - const U = std.meta.Int(.unsigned, i); - const S = std.meta.Int(.signed, i); - const x = try _deserializer.deserializeInt(U); - const y = try _deserializer.deserializeInt(S); - testing.expect(x == @as(U, i)); - if (i != 0) testing.expect(y == @as(S, -1)) else testing.expect(y == 0); - } - - const u8_bit_count = comptime meta.bitCount(u8); - //0 + 1 + 2 + ... n = (n * (n + 1)) / 2 - //and we have each for unsigned and signed, so * 2 - const total_bits = (max_test_bitsize * (max_test_bitsize + 1)); - const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); - const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; - - testing.expect(in.pos == if (packing == .Bit) total_packed_bytes else total_bytes); - - //Verify that empty error set works with serializer. - //deserializer is covered by FixedBufferStream - var null_serializer = io.serializer(endian, packing, std.io.null_out_stream); - try null_serializer.serialize(data_mem[0..]); - try null_serializer.flush(); -} - -test "Serializer/Deserializer Int" { - try testIntSerializerDeserializer(.Big, .Byte); - try testIntSerializerDeserializer(.Little, .Byte); - // TODO these tests are disabled due to tripping an LLVM assertion - // https://github.com/ziglang/zig/issues/2019 - //try testIntSerializerDeserializer(builtin.Endian.Big, true); - //try testIntSerializerDeserializer(builtin.Endian.Little, true); -} - -fn testIntSerializerDeserializerInfNaN( - comptime endian: builtin.Endian, - comptime packing: io.Packing, -) !void { - const mem_size = (16 * 2 + 32 * 2 + 64 * 2 + 128 * 2) / comptime meta.bitCount(u8); - var data_mem: [mem_size]u8 = undefined; - - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - //@TODO: isInf/isNan not currently implemented for f128. - try _serializer.serialize(std.math.nan(f16)); - try _serializer.serialize(std.math.inf(f16)); - try _serializer.serialize(std.math.nan(f32)); - try _serializer.serialize(std.math.inf(f32)); - try _serializer.serialize(std.math.nan(f64)); - try _serializer.serialize(std.math.inf(f64)); - //try serializer.serialize(std.math.nan(f128)); - //try serializer.serialize(std.math.inf(f128)); - const nan_check_f16 = try _deserializer.deserialize(f16); - const inf_check_f16 = try _deserializer.deserialize(f16); - const nan_check_f32 = try _deserializer.deserialize(f32); - _deserializer.alignToByte(); - const inf_check_f32 = try _deserializer.deserialize(f32); - const nan_check_f64 = try _deserializer.deserialize(f64); - const inf_check_f64 = try _deserializer.deserialize(f64); - //const nan_check_f128 = try deserializer.deserialize(f128); - //const inf_check_f128 = try deserializer.deserialize(f128); - testing.expect(std.math.isNan(nan_check_f16)); - testing.expect(std.math.isInf(inf_check_f16)); - testing.expect(std.math.isNan(nan_check_f32)); - testing.expect(std.math.isInf(inf_check_f32)); - testing.expect(std.math.isNan(nan_check_f64)); - testing.expect(std.math.isInf(inf_check_f64)); - //expect(std.math.isNan(nan_check_f128)); - //expect(std.math.isInf(inf_check_f128)); -} - -test "Serializer/Deserializer Int: Inf/NaN" { - try testIntSerializerDeserializerInfNaN(.Big, .Byte); - try testIntSerializerDeserializerInfNaN(.Little, .Byte); - try testIntSerializerDeserializerInfNaN(.Big, .Bit); - try testIntSerializerDeserializerInfNaN(.Little, .Bit); -} - -fn testAlternateSerializer(self: anytype, _serializer: anytype) !void { - try _serializer.serialize(self.f_f16); -} - -fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - const ColorType = enum(u4) { - RGB8 = 1, - RA16 = 2, - R32 = 3, - }; - - const TagAlign = union(enum(u32)) { - A: u8, - B: u8, - C: u8, - }; - - const Color = union(ColorType) { - RGB8: struct { - r: u8, - g: u8, - b: u8, - a: u8, - }, - RA16: struct { - r: u16, - a: u16, - }, - R32: u32, - }; - - const PackedStruct = packed struct { - f_i3: i3, - f_u2: u2, - }; - - //to test custom serialization - const Custom = struct { - f_f16: f16, - f_unused_u32: u32, - - pub fn deserialize(self: *@This(), _deserializer: anytype) !void { - try _deserializer.deserializeInto(&self.f_f16); - self.f_unused_u32 = 47; - } - - pub const serialize = testAlternateSerializer; - }; - - const MyStruct = struct { - f_i3: i3, - f_u8: u8, - f_tag_align: TagAlign, - f_u24: u24, - f_i19: i19, - f_void: void, - f_f32: f32, - f_f128: f128, - f_packed_0: PackedStruct, - f_i7arr: [10]i7, - f_of64n: ?f64, - f_of64v: ?f64, - f_color_type: ColorType, - f_packed_1: PackedStruct, - f_custom: Custom, - f_color: Color, - }; - - const my_inst = MyStruct{ - .f_i3 = -1, - .f_u8 = 8, - .f_tag_align = TagAlign{ .B = 148 }, - .f_u24 = 24, - .f_i19 = 19, - .f_void = {}, - .f_f32 = 32.32, - .f_f128 = 128.128, - .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 }, - .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, - .f_of64n = null, - .f_of64v = 64.64, - .f_color_type = ColorType.R32, - .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 }, - .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 }, - .f_color = Color{ .R32 = 123822 }, - }; - - var data_mem: [@sizeOf(MyStruct)]u8 = undefined; - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - try _serializer.serialize(my_inst); - - const my_copy = try _deserializer.deserialize(MyStruct); - testing.expect(meta.eql(my_copy, my_inst)); -} - -test "Serializer/Deserializer generic" { - try testSerializerDeserializer(builtin.Endian.Big, .Byte); - try testSerializerDeserializer(builtin.Endian.Little, .Byte); - try testSerializerDeserializer(builtin.Endian.Big, .Bit); - try testSerializerDeserializer(builtin.Endian.Little, .Bit); -} - -fn testBadData(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - const E = enum(u14) { - One = 1, - Two = 2, - }; - - const A = struct { - e: E, - }; - - const C = union(E) { - One: u14, - Two: f16, - }; - - var data_mem: [4]u8 = undefined; - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - try _serializer.serialize(@as(u14, 3)); - testing.expectError(error.InvalidEnumTag, _deserializer.deserialize(A)); - out.pos = 0; - try _serializer.serialize(@as(u14, 3)); - try _serializer.serialize(@as(u14, 88)); - testing.expectError(error.InvalidEnumTag, _deserializer.deserialize(C)); -} - -test "Deserializer bad data" { - try testBadData(.Big, .Byte); - try testBadData(.Little, .Byte); - try testBadData(.Big, .Bit); - try testBadData(.Little, .Bit); -} diff --git a/lib/std/macho.zig b/lib/std/macho.zig index ec0d23cd92..016590e36b 100644 --- a/lib/std/macho.zig +++ b/lib/std/macho.zig @@ -1257,6 +1257,33 @@ pub const VM_PROT_WRITE: vm_prot_t = 0x2; /// VM execute permission pub const VM_PROT_EXECUTE: vm_prot_t = 0x4; +pub const BIND_TYPE_POINTER: u8 = 1; +pub const BIND_TYPE_TEXT_ABSOLUTE32: u8 = 2; +pub const BIND_TYPE_TEXT_PCREL32: u8 = 3; + +pub const BIND_SPECIAL_DYLIB_SELF: i8 = 0; +pub const BIND_SPECIAL_DYLIB_MAIN_EXECUTABLE: i8 = -1; +pub const BIND_SPECIAL_DYLIB_FLAT_LOOKUP: i8 = -2; + +pub const BIND_SYMBOL_FLAGS_WEAK_IMPORT: u8 = 0x1; +pub const BIND_SYMBOL_FLAGS_NON_WEAK_DEFINITION: u8 = 0x8; + +pub const BIND_OPCODE_MASK: u8 = 0xf0; +pub const BIND_IMMEDIATE_MASK: u8 = 0x0f; +pub const BIND_OPCODE_DONE: u8 = 0x00; +pub const BIND_OPCODE_SET_DYLIB_ORDINAL_IMM: u8 = 0x10; +pub const BIND_OPCODE_SET_DYLIB_ORDINAL_ULEB: u8 = 0x20; +pub const BIND_OPCODE_SET_DYLIB_SPECIAL_IMM: u8 = 0x30; +pub const BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM: u8 = 0x40; +pub const BIND_OPCODE_SET_TYPE_IMM: u8 = 0x50; +pub const BIND_OPCODE_SET_ADDEND_SLEB: u8 = 0x60; +pub const BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB: u8 = 0x70; +pub const BIND_OPCODE_ADD_ADDR_ULEB: 0x80; +pub const BIND_OPCODE_DO_BIND: u8 = 0x90; +pub const BIND_OPCODE_DO_BIND_ADD_ADDR_ULEB: u8 = 0xa0; +pub const BIND_OPCODE_DO_BIND_ADD_ADDR_IMM_SCALED: u8 = 0xb0; +pub const BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB: u8 = xc0; + pub const reloc_type_x86_64 = packed enum(u4) { /// for absolute addresses X86_64_RELOC_UNSIGNED = 0, diff --git a/lib/std/math.zig b/lib/std/math.zig index 4fc9eb40e6..a51cac6e7d 100644 --- a/lib/std/math.zig +++ b/lib/std/math.zig @@ -1169,7 +1169,7 @@ pub const Order = enum { return switch (self) { .lt => .gt, .eq => .eq, - .gt => .gt, + .gt => .lt, }; } @@ -1266,6 +1266,29 @@ test "compare between signed and unsigned" { testing.expect(compare(@as(u8, 1), .eq, @as(u8, 1))); } +test "order" { + testing.expect(order(0, 0) == .eq); + testing.expect(order(1, 0) == .gt); + testing.expect(order(-1, 0) == .lt); +} + +test "order.invert" { + testing.expect(Order.invert(order(0, 0)) == .eq); + testing.expect(Order.invert(order(1, 0)) == .lt); + testing.expect(Order.invert(order(-1, 0)) == .gt); +} + +test "order.compare" { + testing.expect(order(-1, 0).compare(.lt)); + testing.expect(order(-1, 0).compare(.lte)); + testing.expect(order(0, 0).compare(.lte)); + testing.expect(order(0, 0).compare(.eq)); + testing.expect(order(0, 0).compare(.gte)); + testing.expect(order(1, 0).compare(.gte)); + testing.expect(order(1, 0).compare(.gt)); + testing.expect(order(1, 0).compare(.neq)); +} + test "math.comptime" { comptime const v = sin(@as(f32, 1)) + ln(@as(f32, 5)); testing.expect(v == sin(@as(f32, 1)) + ln(@as(f32, 5))); diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 2cf7f6de81..d51a2744b3 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -9,6 +9,7 @@ const debug = std.debug; const mem = std.mem; const math = std.math; const testing = std.testing; +const root = @import("root"); pub const trait = @import("meta/trait.zig"); pub const TrailerFlags = @import("meta/trailer_flags.zig").TrailerFlags; @@ -1085,3 +1086,10 @@ test "Tuple" { TupleTester.assertTuple(.{ u32, f16 }, Tuple(&[_]type{ u32, f16 })); TupleTester.assertTuple(.{ u32, f16, []const u8, void }, Tuple(&[_]type{ u32, f16, []const u8, void })); } + +/// TODO: https://github.com/ziglang/zig/issues/425 +pub fn globalOption(comptime name: []const u8, comptime T: type) ?T { + if (!@hasDecl(root, name)) + return null; + return @as(T, @field(root, name)); +} diff --git a/lib/std/os.zig b/lib/std/os.zig index e3afe90e5d..5a57fed5cf 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4592,7 +4592,7 @@ pub fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) SigaltstackError!void { } /// Examine and change a signal action. -pub fn sigaction(sig: u6, act: *const Sigaction, oact: ?*Sigaction) void { +pub fn sigaction(sig: u6, act: ?*const Sigaction, oact: ?*Sigaction) void { switch (errno(system.sigaction(sig, act, oact))) { 0 => return, EFAULT => unreachable, @@ -4774,9 +4774,33 @@ pub const SendError = error{ BrokenPipe, FileDescriptorNotASocket, - AddressFamilyNotSupported, + + /// Network is unreachable. + NetworkUnreachable, + + /// The local network interface used to reach the destination is down. + NetworkSubsystemFailed, } || UnexpectedError; +pub const SendToError = SendError || error{ + /// The passed address didn't have the correct address family in its sa_family field. + AddressFamilyNotSupported, + + /// Returned when socket is AF_UNIX and the given path has a symlink loop. + SymLinkLoop, + + /// Returned when socket is AF_UNIX and the given path length exceeds `MAX_PATH_BYTES` bytes. + NameTooLong, + + /// Returned when socket is AF_UNIX and the given path does not point to an existing file. + FileNotFound, + NotDir, + + /// The socket is not connected (connection-oriented sockets only). + SocketNotConnected, + AddressNotAvailable, +}; + /// Transmit a message to another socket. /// /// The `sendto` call may be used only when the socket is in a connected state (so that the intended @@ -4810,19 +4834,31 @@ pub fn sendto( flags: u32, dest_addr: ?*const sockaddr, addrlen: socklen_t, -) SendError!usize { +) SendToError!usize { while (true) { const rc = system.sendto(sockfd, buf.ptr, buf.len, flags, dest_addr, addrlen); if (builtin.os.tag == .windows) { if (rc == windows.ws2_32.SOCKET_ERROR) { switch (windows.ws2_32.WSAGetLastError()) { .WSAEACCES => return error.AccessDenied, + .WSAEADDRNOTAVAIL => return error.AddressNotAvailable, .WSAECONNRESET => return error.ConnectionResetByPeer, .WSAEMSGSIZE => return error.MessageTooBig, .WSAENOBUFS => return error.SystemResources, .WSAENOTSOCK => return error.FileDescriptorNotASocket, .WSAEAFNOSUPPORT => return error.AddressFamilyNotSupported, - // TODO: handle more errors + .WSAEDESTADDRREQ => unreachable, // A destination address is required. + .WSAEFAULT => unreachable, // The lpBuffers, lpTo, lpOverlapped, lpNumberOfBytesSent, or lpCompletionRoutine parameters are not part of the user address space, or the lpTo parameter is too small. + .WSAEHOSTUNREACH => return error.NetworkUnreachable, + // TODO: WSAEINPROGRESS, WSAEINTR + .WSAEINVAL => unreachable, + .WSAENETDOWN => return error.NetworkSubsystemFailed, + .WSAENETRESET => return error.ConnectionResetByPeer, + .WSAENETUNREACH => return error.NetworkUnreachable, + .WSAENOTCONN => return error.SocketNotConnected, + .WSAESHUTDOWN => unreachable, // The socket has been shut down; it is not possible to WSASendTo on a socket after shutdown has been invoked with how set to SD_SEND or SD_BOTH. + .WSAEWOULDBLOCK => return error.WouldBlock, + .WSANOTINITIALISED => unreachable, // A successful WSAStartup call must occur before using this function. else => |err| return windows.unexpectedWSAError(err), } } else { @@ -4845,11 +4881,18 @@ pub fn sendto( EMSGSIZE => return error.MessageTooBig, ENOBUFS => return error.SystemResources, ENOMEM => return error.SystemResources, - ENOTCONN => unreachable, // The socket is not connected, and no target has been given. ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. EOPNOTSUPP => unreachable, // Some bit in the flags argument is inappropriate for the socket type. EPIPE => return error.BrokenPipe, EAFNOSUPPORT => return error.AddressFamilyNotSupported, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + EHOSTUNREACH => return error.NetworkUnreachable, + ENETUNREACH => return error.NetworkUnreachable, + ENOTCONN => return error.SocketNotConnected, + ENETDOWN => return error.NetworkSubsystemFailed, else => |err| return unexpectedErrno(err), } } @@ -4881,7 +4924,16 @@ pub fn send( buf: []const u8, flags: u32, ) SendError!usize { - return sendto(sockfd, buf, flags, null, 0); + return sendto(sockfd, buf, flags, null, 0) catch |err| switch (err) { + error.AddressFamilyNotSupported => unreachable, + error.SymLinkLoop => unreachable, + error.NameTooLong => unreachable, + error.FileNotFound => unreachable, + error.NotDir => unreachable, + error.NetworkUnreachable => unreachable, + error.AddressNotAvailable => unreachable, + else => |e| return e, + }; } pub const SendFileError = PReadError || WriteError || SendError; @@ -5803,3 +5855,51 @@ pub fn setrlimit(resource: rlimit_resource, limits: rlimit) SetrlimitError!void else => |err| return unexpectedErrno(err), } } + +pub const MadviseError = error{ + /// advice is MADV_REMOVE, but the specified address range is not a shared writable mapping. + AccessDenied, + /// advice is MADV_HWPOISON, but the caller does not have the CAP_SYS_ADMIN capability. + PermissionDenied, + /// A kernel resource was temporarily unavailable. + SystemResources, + /// One of the following: + /// * addr is not page-aligned or length is negative + /// * advice is not valid + /// * advice is MADV_DONTNEED or MADV_REMOVE and the specified address range + /// includes locked, Huge TLB pages, or VM_PFNMAP pages. + /// * advice is MADV_MERGEABLE or MADV_UNMERGEABLE, but the kernel was not + /// configured with CONFIG_KSM. + /// * advice is MADV_FREE or MADV_WIPEONFORK but the specified address range + /// includes file, Huge TLB, MAP_SHARED, or VM_PFNMAP ranges. + InvalidSyscall, + /// (for MADV_WILLNEED) Paging in this area would exceed the process's + /// maximum resident set size. + WouldExceedMaximumResidentSetSize, + /// One of the following: + /// * (for MADV_WILLNEED) Not enough memory: paging in failed. + /// * Addresses in the specified range are not currently mapped, or + /// are outside the address space of the process. + OutOfMemory, + /// The madvise syscall is not available on this version and configuration + /// of the Linux kernel. + MadviseUnavailable, + /// The operating system returned an undocumented error code. + Unexpected, +}; + +/// Give advice about use of memory. +/// This syscall is optional and is sometimes configured to be disabled. +pub fn madvise(ptr: [*]align(mem.page_size) u8, length: usize, advice: u32) MadviseError!void { + switch (errno(system.madvise(ptr, length, advice))) { + 0 => return, + EACCES => return error.AccessDenied, + EAGAIN => return error.SystemResources, + EBADF => unreachable, // The map exists, but the area maps something that isn't a file. + EINVAL => return error.InvalidSyscall, + EIO => return error.WouldExceedMaximumResidentSetSize, + ENOMEM => return error.OutOfMemory, + ENOSYS => return error.MadviseUnavailable, + else => |err| return unexpectedErrno(err), + } +} diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index 83ca09a9e1..8bd40ed9a3 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -124,13 +124,40 @@ pub const timespec = extern struct { }; pub const sigset_t = u32; -pub const empty_sigset = sigset_t(0); +pub const empty_sigset: sigset_t = 0; + +pub const SIG_ERR = @intToPtr(?Sigaction.sigaction_fn, maxInt(usize)); +pub const SIG_DFL = @intToPtr(?Sigaction.sigaction_fn, 0); +pub const SIG_IGN = @intToPtr(?Sigaction.sigaction_fn, 1); +pub const SIG_HOLD = @intToPtr(?Sigaction.sigaction_fn, 5); + +pub const siginfo_t = extern struct { + signo: c_int, + errno: c_int, + code: c_int, + pid: pid_t, + uid: uid_t, + status: c_int, + addr: *c_void, + value: extern union { + int: c_int, + ptr: *c_void, + }, + si_band: c_long, + _pad: [7]c_ulong, +}; /// Renamed from `sigaction` to `Sigaction` to avoid conflict with function name. pub const Sigaction = extern struct { - handler: fn (c_int) callconv(.C) void, - sa_mask: sigset_t, - sa_flags: c_int, + pub const handler_fn = fn (c_int) callconv(.C) void; + pub const sigaction_fn = fn (c_int, *const siginfo_t, ?*const c_void) callconv(.C) void; + + handler: extern union { + handler: ?handler_fn, + sigaction: ?sigaction_fn, + }, + mask: sigset_t, + flags: c_uint, }; pub const dirent = extern struct { diff --git a/lib/std/os/bits/dragonfly.zig b/lib/std/os/bits/dragonfly.zig index 2fd9e39c7b..61b6b9f363 100644 --- a/lib/std/os/bits/dragonfly.zig +++ b/lib/std/os/bits/dragonfly.zig @@ -530,12 +530,16 @@ pub const sigset_t = extern struct { }; pub const sig_atomic_t = c_int; pub const Sigaction = extern struct { - __sigaction_u: extern union { - __sa_handler: ?fn (c_int) callconv(.C) void, - __sa_sigaction: ?fn (c_int, [*c]siginfo_t, ?*c_void) callconv(.C) void, + pub const handler_fn = fn (c_int) callconv(.C) void; + pub const sigaction_fn = fn (c_int, *const siginfo_t, ?*const c_void) callconv(.C) void; + + /// signal handler + handler: extern union { + handler: ?handler_fn, + sigaction: ?sigaction_fn, }, - sa_flags: c_int, - sa_mask: sigset_t, + flags: c_uint, + mask: sigset_t, }; pub const sig_t = [*c]fn (c_int) callconv(.C) void; diff --git a/lib/std/os/bits/freebsd.zig b/lib/std/os/bits/freebsd.zig index 5dce94bd90..08713bb2e4 100644 --- a/lib/std/os/bits/freebsd.zig +++ b/lib/std/os/bits/freebsd.zig @@ -4,6 +4,7 @@ // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. const std = @import("../../std.zig"); +const builtin = std.builtin; const maxInt = std.math.maxInt; // See https://svnweb.freebsd.org/base/head/sys/sys/_types.h?view=co @@ -736,23 +737,61 @@ pub const winsize = extern struct { const NSIG = 32; -pub const SIG_ERR = @intToPtr(fn (i32) callconv(.C) void, maxInt(usize)); -pub const SIG_DFL = @intToPtr(fn (i32) callconv(.C) void, 0); -pub const SIG_IGN = @intToPtr(fn (i32) callconv(.C) void, 1); +pub const SIG_ERR = @intToPtr(?Sigaction.sigaction_fn, maxInt(usize)); +pub const SIG_DFL = @intToPtr(?Sigaction.sigaction_fn, 0); +pub const SIG_IGN = @intToPtr(?Sigaction.sigaction_fn, 1); /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = extern struct { + pub const handler_fn = fn (c_int) callconv(.C) void; + pub const sigaction_fn = fn (c_int, *const siginfo_t, ?*const c_void) callconv(.C) void; + /// signal handler - __sigaction_u: extern union { - __sa_handler: fn (i32) callconv(.C) void, - __sa_sigaction: fn (i32, *__siginfo, usize) callconv(.C) void, + handler: extern union { + handler: ?handler_fn, + sigaction: ?sigaction_fn, }, /// see signal options - sa_flags: u32, + flags: c_uint, /// signal mask to apply - sa_mask: sigset_t, + mask: sigset_t, +}; + +pub const siginfo_t = extern struct { + signo: c_int, + errno: c_int, + code: c_int, + pid: pid_t, + uid: uid_t, + status: c_int, + addr: ?*c_void, + value: sigval, + reason: extern union { + fault: extern struct { + trapno: c_int, + }, + timer: extern struct { + timerid: c_int, + overrun: c_int, + }, + mesgq: extern struct { + mqd: c_int, + }, + poll: extern struct { + band: c_long, + }, + spare: extern struct { + spare1: c_long, + spare2: [7]c_int, + }, + }, +}; + +pub const sigval = extern union { + int: c_int, + ptr: ?*c_void, }; pub const _SIG_WORDS = 4; @@ -775,6 +814,55 @@ pub const sigset_t = extern struct { __bits: [_SIG_WORDS]u32, }; +pub const empty_sigset = sigset_t{ .__bits = [_]u32{0} ** _SIG_WORDS }; + +pub usingnamespace switch (builtin.arch) { + .x86_64 => struct { + pub const ucontext_t = extern struct { + sigmask: sigset_t, + mcontext: mcontext_t, + link: ?*ucontext_t, + stack: stack_t, + flags: c_int, + __spare__: [4]c_int, + }; + + /// XXX x86_64 specific + pub const mcontext_t = extern struct { + onstack: u64, + rdi: u64, + rsi: u64, + rdx: u64, + rcx: u64, + r8: u64, + r9: u64, + rax: u64, + rbx: u64, + rbp: u64, + r10: u64, + r11: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, + trapno: u32, + fs: u16, + gs: u16, + addr: u64, + flags: u32, + es: u16, + ds: u16, + err: u64, + rip: u64, + cs: u64, + rflags: u64, + rsp: u64, + ss: u64, + }; + }, + else => struct {}, +}; + pub const EPERM = 1; // Operation not permitted pub const ENOENT = 2; // No such file or directory pub const ESRCH = 3; // No such process diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index 6952ab7e0e..2bcfc89ecf 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -864,27 +864,38 @@ pub const sigset_t = [1024 / 32]u32; pub const all_mask: sigset_t = [_]u32{0xffffffff} ** sigset_t.len; pub const app_mask: sigset_t = [2]u32{ 0xfffffffc, 0x7fffffff } ++ [_]u32{0xffffffff} ** 30; -pub const k_sigaction = if (is_mips) - extern struct { - flags: usize, - sigaction: ?fn (i32, *siginfo_t, ?*c_void) callconv(.C) void, - mask: [4]u32, +pub const k_sigaction = switch (builtin.arch) { + .mips, .mipsel => extern struct { + flags: c_uint, + handler: ?fn (c_int) callconv(.C) void, + mask: [4]c_ulong, restorer: fn () callconv(.C) void, - } -else - extern struct { - sigaction: ?fn (i32, *siginfo_t, ?*c_void) callconv(.C) void, - flags: usize, + }, + .mips64, .mips64el => extern struct { + flags: c_uint, + handler: ?fn (c_int) callconv(.C) void, + mask: [2]c_ulong, restorer: fn () callconv(.C) void, - mask: [2]u32, - }; + }, + else => extern struct { + handler: ?fn (c_int) callconv(.C) void, + flags: c_ulong, + restorer: fn () callconv(.C) void, + mask: [2]c_uint, + }, +}; /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = extern struct { - pub const sigaction_fn = fn (i32, *siginfo_t, ?*c_void) callconv(.C) void; - sigaction: ?sigaction_fn, + pub const handler_fn = fn (c_int) callconv(.C) void; + pub const sigaction_fn = fn (c_int, *const siginfo_t, ?*const c_void) callconv(.C) void; + + handler: extern union { + handler: ?handler_fn, + sigaction: ?sigaction_fn, + }, mask: sigset_t, - flags: u32, + flags: c_uint, restorer: ?fn () callconv(.C) void = null, }; @@ -1826,6 +1837,39 @@ pub const tcflag_t = u32; pub const NCCS = 32; +pub const B0 = 0o0000000; +pub const B50 = 0o0000001; +pub const B75 = 0o0000002; +pub const B110 = 0o0000003; +pub const B134 = 0o0000004; +pub const B150 = 0o0000005; +pub const B200 = 0o0000006; +pub const B300 = 0o0000007; +pub const B600 = 0o0000010; +pub const B1200 = 0o0000011; +pub const B1800 = 0o0000012; +pub const B2400 = 0o0000013; +pub const B4800 = 0o0000014; +pub const B9600 = 0o0000015; +pub const B19200 = 0o0000016; +pub const B38400 = 0o0000017; +pub const BOTHER = 0o0010000; +pub const B57600 = 0o0010001; +pub const B115200 = 0o0010002; +pub const B230400 = 0o0010003; +pub const B460800 = 0o0010004; +pub const B500000 = 0o0010005; +pub const B576000 = 0o0010006; +pub const B921600 = 0o0010007; +pub const B1000000 = 0o0010010; +pub const B1152000 = 0o0010011; +pub const B1500000 = 0o0010012; +pub const B2000000 = 0o0010013; +pub const B2500000 = 0o0010014; +pub const B3000000 = 0o0010015; +pub const B3500000 = 0o0010016; +pub const B4000000 = 0o0010017; + pub const IGNBRK = 1; pub const BRKINT = 2; pub const IGNPAR = 4; @@ -2001,3 +2045,25 @@ pub const rlimit = extern struct { /// Hard limit max: rlim_t, }; + +pub const MADV_NORMAL = 0; +pub const MADV_RANDOM = 1; +pub const MADV_SEQUENTIAL = 2; +pub const MADV_WILLNEED = 3; +pub const MADV_DONTNEED = 4; +pub const MADV_FREE = 8; +pub const MADV_REMOVE = 9; +pub const MADV_DONTFORK = 10; +pub const MADV_DOFORK = 11; +pub const MADV_MERGEABLE = 12; +pub const MADV_UNMERGEABLE = 13; +pub const MADV_HUGEPAGE = 14; +pub const MADV_NOHUGEPAGE = 15; +pub const MADV_DONTDUMP = 16; +pub const MADV_DODUMP = 17; +pub const MADV_WIPEONFORK = 18; +pub const MADV_KEEPONFORK = 19; +pub const MADV_COLD = 20; +pub const MADV_PAGEOUT = 21; +pub const MADV_HWPOISON = 100; +pub const MADV_SOFT_OFFLINE = 101; diff --git a/lib/std/os/bits/linux/arm-eabi.zig b/lib/std/os/bits/linux/arm-eabi.zig index 5539199c73..d961e7ff4d 100644 --- a/lib/std/os/bits/linux/arm-eabi.zig +++ b/lib/std/os/bits/linux/arm-eabi.zig @@ -407,8 +407,11 @@ pub const SYS = extern enum(usize) { fspick = 433, pidfd_open = 434, clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, breakpoint = 0x0f0001, cacheflush = 0x0f0002, diff --git a/lib/std/os/bits/linux/arm64.zig b/lib/std/os/bits/linux/arm64.zig index 2f677ca537..71a98e49b7 100644 --- a/lib/std/os/bits/linux/arm64.zig +++ b/lib/std/os/bits/linux/arm64.zig @@ -308,8 +308,11 @@ pub const SYS = extern enum(usize) { fspick = 433, pidfd_open = 434, clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, _, }; diff --git a/lib/std/os/bits/linux/i386.zig b/lib/std/os/bits/linux/i386.zig index b8c7221e9e..5e78e5c357 100644 --- a/lib/std/os/bits/linux/i386.zig +++ b/lib/std/os/bits/linux/i386.zig @@ -441,8 +441,13 @@ pub const SYS = extern enum(usize) { fsconfig = 431, fsmount = 432, fspick = 433, + pidfd_open = 434, + clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, _, }; diff --git a/lib/std/os/bits/linux/powerpc64.zig b/lib/std/os/bits/linux/powerpc64.zig index 89619ebe16..98fd77997c 100644 --- a/lib/std/os/bits/linux/powerpc64.zig +++ b/lib/std/os/bits/linux/powerpc64.zig @@ -404,8 +404,11 @@ pub const SYS = extern enum(usize) { fspick = 433, pidfd_open = 434, clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, _, }; diff --git a/lib/std/os/bits/linux/riscv64.zig b/lib/std/os/bits/linux/riscv64.zig index bd4b45f95f..a597f4ff0b 100644 --- a/lib/std/os/bits/linux/riscv64.zig +++ b/lib/std/os/bits/linux/riscv64.zig @@ -305,8 +305,11 @@ pub const SYS = extern enum(usize) { fspick = 433, pidfd_open = 434, clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, _, }; diff --git a/lib/std/os/bits/linux/sparc64.zig b/lib/std/os/bits/linux/sparc64.zig index 1ce17b3a01..e2e34c39e4 100644 --- a/lib/std/os/bits/linux/sparc64.zig +++ b/lib/std/os/bits/linux/sparc64.zig @@ -381,8 +381,12 @@ pub const SYS = extern enum(usize) { fsmount = 432, fspick = 433, pidfd_open = 434, + clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, _, }; diff --git a/lib/std/os/bits/linux/x86_64.zig b/lib/std/os/bits/linux/x86_64.zig index 19e52536e8..d02f557690 100644 --- a/lib/std/os/bits/linux/x86_64.zig +++ b/lib/std/os/bits/linux/x86_64.zig @@ -370,8 +370,11 @@ pub const SYS = extern enum(usize) { fspick = 433, pidfd_open = 434, clone3 = 435, + close_range = 436, openat2 = 437, pidfd_getfd = 438, + faccessat2 = 439, + process_madvise = 440, _, }; diff --git a/lib/std/os/bits/netbsd.zig b/lib/std/os/bits/netbsd.zig index 0f4b6b60fa..be25284b73 100644 --- a/lib/std/os/bits/netbsd.zig +++ b/lib/std/os/bits/netbsd.zig @@ -716,13 +716,18 @@ pub const SIG_IGN = @intToPtr(?Sigaction.sigaction_fn, 1); /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = extern struct { - pub const sigaction_fn = fn (i32, *siginfo_t, ?*c_void) callconv(.C) void; + pub const handler_fn = fn (c_int) callconv(.C) void; + pub const sigaction_fn = fn (c_int, *const siginfo_t, ?*const c_void) callconv(.C) void; + /// signal handler - sigaction: ?sigaction_fn, + handler: extern union { + handler: ?handler_fn, + sigaction: ?sigaction_fn, + }, /// signal mask to apply mask: sigset_t, /// signal options - flags: u32, + flags: c_uint, }; pub const sigval_t = extern union { @@ -800,6 +805,10 @@ pub const sigset_t = extern struct { __bits: [_SIG_WORDS]u32, }; +pub const SIG_ERR = @intToPtr(?Sigaction.sigaction_fn, maxInt(usize)); +pub const SIG_DFL = @intToPtr(?Sigaction.sigaction_fn, 0); +pub const SIG_IGN = @intToPtr(?Sigaction.sigaction_fn, 1); + pub const empty_sigset = sigset_t{ .__bits = [_]u32{0} ** _SIG_WORDS }; // XXX x86_64 specific diff --git a/lib/std/os/bits/openbsd.zig b/lib/std/os/bits/openbsd.zig index c85a476e02..c84a6de01a 100644 --- a/lib/std/os/bits/openbsd.zig +++ b/lib/std/os/bits/openbsd.zig @@ -61,18 +61,18 @@ pub const Flock = extern struct { l_start: off_t, l_len: off_t, l_pid: pid_t, - l_type: i16, - l_whence: i16, + l_type: c_short, + l_whence: c_short, }; pub const addrinfo = extern struct { - flags: i32, - family: i32, - socktype: i32, - protocol: i32, + flags: c_int, + family: c_int, + socktype: c_int, + protocol: c_int, addrlen: socklen_t, - canonname: ?[*:0]u8, addr: ?*sockaddr, + canonname: ?[*:0]u8, next: ?*addrinfo, }; @@ -135,7 +135,7 @@ pub const msghdr = extern struct { msg_iov: [*]iovec, /// # elements in msg_iov - msg_iovlen: i32, + msg_iovlen: c_uint, /// ancillary data msg_control: ?*c_void, @@ -144,7 +144,7 @@ pub const msghdr = extern struct { msg_controllen: socklen_t, /// flags on received message - msg_flags: i32, + msg_flags: c_int, }; pub const msghdr_const = extern struct { @@ -158,7 +158,7 @@ pub const msghdr_const = extern struct { msg_iov: [*]iovec_const, /// # elements in msg_iov - msg_iovlen: i32, + msg_iovlen: c_uint, /// ancillary data msg_control: ?*c_void, @@ -167,7 +167,7 @@ pub const msghdr_const = extern struct { msg_controllen: socklen_t, /// flags on received message - msg_flags: i32, + msg_flags: c_int, }; pub const libc_stat = extern struct { @@ -739,10 +739,10 @@ pub fn WIFSIGNALED(s: u32) bool { } pub const winsize = extern struct { - ws_row: u16, - ws_col: u16, - ws_xpixel: u16, - ws_ypixel: u16, + ws_row: c_ushort, + ws_col: c_ushort, + ws_xpixel: c_ushort, + ws_ypixel: c_ushort, }; const NSIG = 33; @@ -750,16 +750,23 @@ const NSIG = 33; pub const SIG_ERR = @intToPtr(?Sigaction.sigaction_fn, maxInt(usize)); pub const SIG_DFL = @intToPtr(?Sigaction.sigaction_fn, 0); pub const SIG_IGN = @intToPtr(?Sigaction.sigaction_fn, 1); +pub const SIG_CATCH = @intToPtr(?Sigaction.sigaction_fn, 2); +pub const SIG_HOLD = @intToPtr(?Sigaction.sigaction_fn, 3); /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = extern struct { - pub const sigaction_fn = fn (c_int, *siginfo_t, ?*c_void) callconv(.C) void; + pub const handler_fn = fn (c_int) callconv(.C) void; + pub const sigaction_fn = fn (c_int, *const siginfo_t, ?*const c_void) callconv(.C) void; + /// signal handler - sigaction: ?sigaction_fn, + handler: extern union { + handler: ?handler_fn, + sigaction: ?sigaction_fn, + }, /// signal mask to apply mask: sigset_t, /// signal options - flags: c_int, + flags: c_uint, }; pub const sigval = extern union { @@ -767,33 +774,97 @@ pub const sigval = extern union { ptr: ?*c_void, }; -pub const siginfo_t = extern union { - pad: [128]u8, - info: _ksiginfo, -}; - -pub const _ksiginfo = extern struct { +pub const siginfo_t = extern struct { signo: c_int, code: c_int, errno: c_int, data: extern union { proc: extern struct { pid: pid_t, - uid: uid_t, - value: sigval, - utime: clock_t, - stime: clock_t, - status: c_int, + pdata: extern union { + kill: extern struct { + uid: uid_t, + value: sigval, + }, + cld: extern struct { + utime: clock_t, + stime: clock_t, + status: c_int, + }, + }, }, fault: extern struct { addr: ?*c_void, trapno: c_int, }, - } align(@sizeOf(usize)), + __pad: [128 - 3 * @sizeOf(c_int)]u8, + }, +}; + +comptime { + if (@sizeOf(usize) == 4) + std.debug.assert(@sizeOf(siginfo_t) == 128) + else + // Take into account the padding between errno and data fields. + std.debug.assert(@sizeOf(siginfo_t) == 136); +} + +pub usingnamespace switch (builtin.arch) { + .x86_64 => struct { + pub const ucontext_t = extern struct { + sc_rdi: c_long, + sc_rsi: c_long, + sc_rdx: c_long, + sc_rcx: c_long, + sc_r8: c_long, + sc_r9: c_long, + sc_r10: c_long, + sc_r11: c_long, + sc_r12: c_long, + sc_r13: c_long, + sc_r14: c_long, + sc_r15: c_long, + sc_rbp: c_long, + sc_rbx: c_long, + sc_rax: c_long, + sc_gs: c_long, + sc_fs: c_long, + sc_es: c_long, + sc_ds: c_long, + sc_trapno: c_long, + sc_err: c_long, + sc_rip: c_long, + sc_cs: c_long, + sc_rflags: c_long, + sc_rsp: c_long, + sc_ss: c_long, + + sc_fpstate: fxsave64, + __sc_unused: c_int, + sc_mask: c_int, + sc_cookie: c_long, + }; + + pub const fxsave64 = packed struct { + fx_fcw: u16, + fx_fsw: u16, + fx_ftw: u8, + fx_unused1: u8, + fx_fop: u16, + fx_rip: u64, + fx_rdp: u64, + fx_mxcsr: u32, + fx_mxcsr_mask: u32, + fx_st: [8][2]u64, + fx_xmm: [16][2]u64, + fx_unused3: [96]u8, + }; + }, + else => struct {}, }; pub const sigset_t = c_uint; -pub const empty_sigset = sigset_t(0); +pub const empty_sigset: sigset_t = 0; pub const EPERM = 1; // Operation not permitted pub const ENOENT = 2; // No such file or directory diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index b669ed2436..d3fffd231e 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -745,33 +745,33 @@ pub fn setregid(rgid: gid_t, egid: gid_t) usize { pub fn getuid() uid_t { if (@hasField(SYS, "getuid32")) { - return @as(uid_t, syscall0(.getuid32)); + return @intCast(uid_t, syscall0(.getuid32)); } else { - return @as(uid_t, syscall0(.getuid)); + return @intCast(uid_t, syscall0(.getuid)); } } pub fn getgid() gid_t { if (@hasField(SYS, "getgid32")) { - return @as(gid_t, syscall0(.getgid32)); + return @intCast(gid_t, syscall0(.getgid32)); } else { - return @as(gid_t, syscall0(.getgid)); + return @intCast(gid_t, syscall0(.getgid)); } } pub fn geteuid() uid_t { if (@hasField(SYS, "geteuid32")) { - return @as(uid_t, syscall0(.geteuid32)); + return @intCast(uid_t, syscall0(.geteuid32)); } else { - return @as(uid_t, syscall0(.geteuid)); + return @intCast(uid_t, syscall0(.geteuid)); } } pub fn getegid() gid_t { if (@hasField(SYS, "getegid32")) { - return @as(gid_t, syscall0(.getegid32)); + return @intCast(gid_t, syscall0(.getegid32)); } else { - return @as(gid_t, syscall0(.getegid)); + return @intCast(gid_t, syscall0(.getegid)); } } @@ -857,35 +857,42 @@ pub fn sigprocmask(flags: u32, noalias set: ?*const sigset_t, noalias oldset: ?* return syscall4(.rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG / 8); } -pub fn sigaction(sig: u6, noalias act: *const Sigaction, noalias oact: ?*Sigaction) usize { +pub fn sigaction(sig: u6, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) usize { assert(sig >= 1); assert(sig != SIGKILL); assert(sig != SIGSTOP); - const restorer_fn = if ((act.flags & SA_SIGINFO) != 0) restore_rt else restore; - var ksa = k_sigaction{ - .sigaction = act.sigaction, - .flags = act.flags | SA_RESTORER, - .mask = undefined, - .restorer = @ptrCast(fn () callconv(.C) void, restorer_fn), - }; - var ksa_old: k_sigaction = undefined; - const ksa_mask_size = @sizeOf(@TypeOf(ksa_old.mask)); - @memcpy(@ptrCast([*]u8, &ksa.mask), @ptrCast([*]const u8, &act.mask), ksa_mask_size); + var ksa: k_sigaction = undefined; + var oldksa: k_sigaction = undefined; + const mask_size = @sizeOf(@TypeOf(ksa.mask)); + + if (act) |new| { + const restorer_fn = if ((new.flags & SA_SIGINFO) != 0) restore_rt else restore; + ksa = k_sigaction{ + .handler = new.handler.handler, + .flags = new.flags | SA_RESTORER, + .mask = undefined, + .restorer = @ptrCast(fn () callconv(.C) void, restorer_fn), + }; + @memcpy(@ptrCast([*]u8, &ksa.mask), @ptrCast([*]const u8, &new.mask), mask_size); + } + + const ksa_arg = if (act != null) @ptrToInt(&ksa) else 0; + const oldksa_arg = if (oact != null) @ptrToInt(&oldksa) else 0; + const result = switch (builtin.arch) { // The sparc version of rt_sigaction needs the restorer function to be passed as an argument too. - .sparc, .sparcv9 => syscall5(.rt_sigaction, sig, @ptrToInt(&ksa), @ptrToInt(&ksa_old), @ptrToInt(ksa.restorer), ksa_mask_size), - else => syscall4(.rt_sigaction, sig, @ptrToInt(&ksa), @ptrToInt(&ksa_old), ksa_mask_size), + .sparc, .sparcv9 => syscall5(.rt_sigaction, sig, ksa_arg, oldksa_arg, @ptrToInt(ksa.restorer), mask_size), + else => syscall4(.rt_sigaction, sig, ksa_arg, oldksa_arg, mask_size), }; - const err = getErrno(result); - if (err != 0) { - return result; - } + if (getErrno(result) != 0) return result; + if (oact) |old| { - old.sigaction = ksa_old.sigaction; - old.flags = @truncate(u32, ksa_old.flags); - @memcpy(@ptrCast([*]u8, &old.mask), @ptrCast([*]const u8, &ksa_old.mask), ksa_mask_size); + old.handler.handler = oldksa.handler; + old.flags = @truncate(c_uint, oldksa.flags); + @memcpy(@ptrCast([*]u8, &old.mask), @ptrCast([*]const u8, &oldksa.mask), mask_size); } + return 0; } @@ -1344,6 +1351,10 @@ pub fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: ?*const rlimit, ); } +pub fn madvise(address: [*]u8, len: usize, advice: u32) usize { + return syscall3(.madvise, @ptrToInt(address), len, advice); +} + test "" { if (builtin.os.tag == .linux) { _ = @import("linux/test.zig"); diff --git a/lib/std/os/linux/test.zig b/lib/std/os/linux/test.zig index 7b3840288a..0077fbcee4 100644 --- a/lib/std/os/linux/test.zig +++ b/lib/std/os/linux/test.zig @@ -9,6 +9,7 @@ const linux = std.os.linux; const mem = std.mem; const elf = std.elf; const expect = std.testing.expect; +const expectEqual = std.testing.expectEqual; const fs = std.fs; test "fallocate" { @@ -99,3 +100,11 @@ test "statx" { expect(@bitCast(u64, @as(i64, stat_buf.blksize)) == statx_buf.blksize); expect(@bitCast(u64, @as(i64, stat_buf.blocks)) == statx_buf.blocks); } + +test "user and group ids" { + if (builtin.link_libc) return error.SkipZigTest; + expectEqual(linux.getauxval(elf.AT_UID), linux.getuid()); + expectEqual(linux.getauxval(elf.AT_GID), linux.getgid()); + expectEqual(linux.getauxval(elf.AT_EUID), linux.geteuid()); + expectEqual(linux.getauxval(elf.AT_EGID), linux.getegid()); +} diff --git a/lib/std/os/linux/tls.zig b/lib/std/os/linux/tls.zig index f6c339bc2c..6d61e60e95 100644 --- a/lib/std/os/linux/tls.zig +++ b/lib/std/os/linux/tls.zig @@ -327,32 +327,43 @@ pub fn prepareTLS(area: []u8) usize { if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset; } -var main_thread_tls_buffer: [256]u8 = undefined; +// The main motivation for the size chosen here is this is how much ends up being +// requested for the thread local variables of the std.crypto.random implementation. +// I'm not sure why it ends up being so much; the struct itself is only 64 bytes. +// I think it has to do with being page aligned and LLVM or LLD is not smart enough +// to lay out the TLS data in a space conserving way. Anyway I think it's fine +// because it's less than 3 pages of memory, and putting it in the ELF like this +// is equivalent to moving the mmap call below into the kernel, avoiding syscall +// overhead. +var main_thread_tls_buffer: [0x2100]u8 align(mem.page_size) = undefined; pub fn initStaticTLS() void { initTLS(); - const alloc_tls_area: []u8 = blk: { - const full_alloc_size = tls_image.alloc_size + tls_image.alloc_align - 1; - + const tls_area = blk: { // Fast path for the common case where the TLS data is really small, - // avoid an allocation and use our local buffer - if (full_alloc_size < main_thread_tls_buffer.len) - break :blk main_thread_tls_buffer[0..]; + // avoid an allocation and use our local buffer. + if (tls_image.alloc_align <= mem.page_size and + tls_image.alloc_size <= main_thread_tls_buffer.len) + { + break :blk main_thread_tls_buffer[0..tls_image.alloc_size]; + } - break :blk os.mmap( + const alloc_tls_area = os.mmap( null, - full_alloc_size, + tls_image.alloc_size + tls_image.alloc_align - 1, os.PROT_READ | os.PROT_WRITE, os.MAP_PRIVATE | os.MAP_ANONYMOUS, -1, 0, ) catch os.abort(); - }; - // Make sure the slice is correctly aligned - const start = @ptrToInt(alloc_tls_area.ptr) & (tls_image.alloc_align - 1); - const tls_area = alloc_tls_area[start .. start + tls_image.alloc_size]; + // Make sure the slice is correctly aligned. + const begin_addr = @ptrToInt(alloc_tls_area.ptr); + const begin_aligned_addr = mem.alignForward(begin_addr, tls_image.alloc_align); + const start = begin_aligned_addr - begin_addr; + break :blk alloc_tls_area[start .. start + tls_image.alloc_size]; + }; const tp_value = prepareTLS(tls_area); setThreadPointer(tp_value); diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 81f9922d13..8ad172679b 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -646,3 +646,41 @@ test "shutdown socket" { }; os.closeSocket(sock); } + +var signal_test_failed = true; + +test "sigaction" { + if (builtin.os.tag == .wasi or builtin.os.tag == .windows) + return error.SkipZigTest; + + // https://github.com/ziglang/zig/issues/7427 + if (builtin.os.tag == .linux and builtin.arch == .i386) + return error.SkipZigTest; + + const S = struct { + fn handler(sig: i32, info: *const os.siginfo_t, ctx_ptr: ?*const c_void) callconv(.C) void { + // Check that we received the correct signal. + if (sig == os.SIGUSR1 and sig == info.signo) + signal_test_failed = false; + } + }; + + var sa = os.Sigaction{ + .handler = .{ .sigaction = S.handler }, + .mask = os.empty_sigset, + .flags = os.SA_SIGINFO | os.SA_RESETHAND, + }; + var old_sa: os.Sigaction = undefined; + // Install the new signal handler. + os.sigaction(os.SIGUSR1, &sa, null); + // Check that we can read it back correctly. + os.sigaction(os.SIGUSR1, null, &old_sa); + testing.expectEqual(S.handler, old_sa.handler.sigaction.?); + testing.expect((old_sa.flags & os.SA_SIGINFO) != 0); + // Invoke the handler. + try os.raise(os.SIGUSR1); + testing.expect(signal_test_failed == false); + // Check if the handler has been correctly reset to SIG_DFL + os.sigaction(os.SIGUSR1, null, &old_sa); + testing.expectEqual(os.SIG_DFL, old_sa.handler.sigaction); +} diff --git a/lib/std/progress.zig b/lib/std/progress.zig deleted file mode 100644 index 82f2801fa1..0000000000 --- a/lib/std/progress.zig +++ /dev/null @@ -1,310 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2015-2020 Zig Contributors -// This file is part of [zig](https://ziglang.org/), which is MIT licensed. -// The MIT license requires this copyright notice to be included in all copies -// and substantial portions of the software. -const std = @import("std"); -const windows = std.os.windows; -const testing = std.testing; -const assert = std.debug.assert; - -/// This API is non-allocating and non-fallible. The tradeoff is that users of -/// this API must provide the storage for each `Progress.Node`. -/// Initialize the struct directly, overriding these fields as desired: -/// * `refresh_rate_ms` -/// * `initial_delay_ms` -pub const Progress = struct { - /// `null` if the current node (and its children) should - /// not print on update() - terminal: ?std.fs.File = undefined, - - /// Whether the terminal supports ANSI escape codes. - supports_ansi_escape_codes: bool = false, - - root: Node = undefined, - - /// Keeps track of how much time has passed since the beginning. - /// Used to compare with `initial_delay_ms` and `refresh_rate_ms`. - timer: std.time.Timer = undefined, - - /// When the previous refresh was written to the terminal. - /// Used to compare with `refresh_rate_ms`. - prev_refresh_timestamp: u64 = undefined, - - /// This buffer represents the maximum number of bytes written to the terminal - /// with each refresh. - output_buffer: [100]u8 = undefined, - - /// How many nanoseconds between writing updates to the terminal. - refresh_rate_ns: u64 = 50 * std.time.ns_per_ms, - - /// How many nanoseconds to keep the output hidden - initial_delay_ns: u64 = 500 * std.time.ns_per_ms, - - done: bool = true, - - /// Keeps track of how many columns in the terminal have been output, so that - /// we can move the cursor back later. - columns_written: usize = undefined, - - /// Represents one unit of progress. Each node can have children nodes, or - /// one can use integers with `update`. - pub const Node = struct { - context: *Progress, - parent: ?*Node, - completed_items: usize, - name: []const u8, - recently_updated_child: ?*Node = null, - - /// This field may be updated freely. - estimated_total_items: ?usize, - - /// Create a new child progress node. - /// Call `Node.end` when done. - /// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this - /// API to set `self.parent.recently_updated_child` with the return value. - /// Until that is fixed you probably want to call `activate` on the return value. - pub fn start(self: *Node, name: []const u8, estimated_total_items: ?usize) Node { - return Node{ - .context = self.context, - .parent = self, - .completed_items = 0, - .name = name, - .estimated_total_items = estimated_total_items, - }; - } - - /// This is the same as calling `start` and then `end` on the returned `Node`. - pub fn completeOne(self: *Node) void { - if (self.parent) |parent| parent.recently_updated_child = self; - self.completed_items += 1; - self.context.maybeRefresh(); - } - - pub fn end(self: *Node) void { - self.context.maybeRefresh(); - if (self.parent) |parent| { - if (parent.recently_updated_child) |parent_child| { - if (parent_child == self) { - parent.recently_updated_child = null; - } - } - parent.completeOne(); - } else { - self.context.done = true; - self.context.refresh(); - } - } - - /// Tell the parent node that this node is actively being worked on. - pub fn activate(self: *Node) void { - if (self.parent) |parent| parent.recently_updated_child = self; - } - }; - - /// Create a new progress node. - /// Call `Node.end` when done. - /// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this - /// API to return Progress rather than accept it as a parameter. - pub fn start(self: *Progress, name: []const u8, estimated_total_items: ?usize) !*Node { - const stderr = std.io.getStdErr(); - self.terminal = null; - if (stderr.supportsAnsiEscapeCodes()) { - self.terminal = stderr; - self.supports_ansi_escape_codes = true; - } else if (std.builtin.os.tag == .windows and stderr.isTty()) { - self.terminal = stderr; - } - self.root = Node{ - .context = self, - .parent = null, - .completed_items = 0, - .name = name, - .estimated_total_items = estimated_total_items, - }; - self.columns_written = 0; - self.prev_refresh_timestamp = 0; - self.timer = try std.time.Timer.start(); - self.done = false; - return &self.root; - } - - /// Updates the terminal if enough time has passed since last update. - pub fn maybeRefresh(self: *Progress) void { - const now = self.timer.read(); - if (now < self.initial_delay_ns) return; - if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return; - self.refresh(); - } - - /// Updates the terminal and resets `self.next_refresh_timestamp`. - pub fn refresh(self: *Progress) void { - const file = self.terminal orelse return; - - const prev_columns_written = self.columns_written; - var end: usize = 0; - if (self.columns_written > 0) { - // restore the cursor position by moving the cursor - // `columns_written` cells to the left, then clear the rest of the - // line - if (self.supports_ansi_escape_codes) { - end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{}D", .{self.columns_written}) catch unreachable).len; - end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len; - } else if (std.builtin.os.tag == .windows) winapi: { - var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined; - if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE) - unreachable; - - var cursor_pos = windows.COORD{ - .X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written), - .Y = info.dwCursorPosition.Y, - }; - - if (cursor_pos.X < 0) - cursor_pos.X = 0; - - const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X); - - var written: windows.DWORD = undefined; - if (windows.kernel32.FillConsoleOutputAttribute( - file.handle, - info.wAttributes, - fill_chars, - cursor_pos, - &written, - ) != windows.TRUE) { - // Stop trying to write to this file. - self.terminal = null; - break :winapi; - } - if (windows.kernel32.FillConsoleOutputCharacterA( - file.handle, - ' ', - fill_chars, - cursor_pos, - &written, - ) != windows.TRUE) unreachable; - - if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE) - unreachable; - } else unreachable; - - self.columns_written = 0; - } - - if (!self.done) { - var need_ellipse = false; - var maybe_node: ?*Node = &self.root; - while (maybe_node) |node| { - if (need_ellipse) { - self.bufWrite(&end, "... ", .{}); - } - need_ellipse = false; - if (node.name.len != 0 or node.estimated_total_items != null) { - if (node.name.len != 0) { - self.bufWrite(&end, "{}", .{node.name}); - need_ellipse = true; - } - if (node.estimated_total_items) |total| { - if (need_ellipse) self.bufWrite(&end, " ", .{}); - self.bufWrite(&end, "[{}/{}] ", .{ node.completed_items + 1, total }); - need_ellipse = false; - } else if (node.completed_items != 0) { - if (need_ellipse) self.bufWrite(&end, " ", .{}); - self.bufWrite(&end, "[{}] ", .{node.completed_items + 1}); - need_ellipse = false; - } - } - maybe_node = node.recently_updated_child; - } - if (need_ellipse) { - self.bufWrite(&end, "... ", .{}); - } - } - - _ = file.write(self.output_buffer[0..end]) catch |e| { - // Stop trying to write to this file once it errors. - self.terminal = null; - }; - self.prev_refresh_timestamp = self.timer.read(); - } - - pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void { - const file = self.terminal orelse return; - self.refresh(); - file.outStream().print(format, args) catch { - self.terminal = null; - return; - }; - self.columns_written = 0; - } - - fn bufWrite(self: *Progress, end: *usize, comptime format: []const u8, args: anytype) void { - if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) |written| { - const amt = written.len; - end.* += amt; - self.columns_written += amt; - } else |err| switch (err) { - error.NoSpaceLeft => { - self.columns_written += self.output_buffer.len - end.*; - end.* = self.output_buffer.len; - }, - } - const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11; - const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end; - if (end.* > max_end) { - const suffix = "... "; - self.columns_written = self.columns_written - (end.* - max_end) + suffix.len; - std.mem.copy(u8, self.output_buffer[max_end..], suffix); - end.* = max_end + suffix.len; - } - } -}; - -test "basic functionality" { - var disable = true; - if (disable) { - // This test is disabled because it uses time.sleep() and is therefore slow. It also - // prints bogus progress data to stderr. - return error.SkipZigTest; - } - var progress = Progress{}; - const root_node = try progress.start("", 100); - defer root_node.end(); - - const sub_task_names = [_][]const u8{ - "reticulating splines", - "adjusting shoes", - "climbing towers", - "pouring juice", - }; - var next_sub_task: usize = 0; - - var i: usize = 0; - while (i < 100) : (i += 1) { - var node = root_node.start(sub_task_names[next_sub_task], 5); - node.activate(); - next_sub_task = (next_sub_task + 1) % sub_task_names.len; - - node.completeOne(); - std.time.sleep(5 * std.time.ns_per_ms); - node.completeOne(); - node.completeOne(); - std.time.sleep(5 * std.time.ns_per_ms); - node.completeOne(); - node.completeOne(); - std.time.sleep(5 * std.time.ns_per_ms); - - node.end(); - - std.time.sleep(5 * std.time.ns_per_ms); - } - { - var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", null); - node.activate(); - std.time.sleep(10 * std.time.ns_per_ms); - progress.refresh(); - std.time.sleep(10 * std.time.ns_per_ms); - node.end(); - } -} diff --git a/lib/std/rand.zig b/lib/std/rand.zig index a201968a5f..681c67f22d 100644 --- a/lib/std/rand.zig +++ b/lib/std/rand.zig @@ -4,19 +4,11 @@ // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -//! The engines provided here should be initialized from an external source. For now, randomBytes -//! from the crypto package is the most suitable. Be sure to use a CSPRNG when required, otherwise using -//! a normal PRNG will be faster and use substantially less stack space. -//! -//! ``` -//! var buf: [8]u8 = undefined; -//! try std.crypto.randomBytes(buf[0..]); -//! const seed = mem.readIntLittle(u64, buf[0..8]); -//! -//! var r = DefaultPrng.init(seed); -//! -//! const s = r.random.int(u64); -//! ``` +//! The engines provided here should be initialized from an external source. +//! For a thread-local cryptographically secure pseudo random number generator, +//! use `std.crypto.random`. +//! Be sure to use a CSPRNG when required, otherwise using a normal PRNG will +//! be faster and use substantially less stack space. //! //! TODO(tiehuis): Benchmark these against other reference implementations. @@ -36,6 +28,12 @@ pub const DefaultPrng = Xoroshiro128; /// Cryptographically secure random numbers. pub const DefaultCsprng = Gimli; +pub const Isaac64 = @import("rand/Isaac64.zig"); +pub const Gimli = @import("rand/Gimli.zig"); +pub const Pcg = @import("rand/Pcg.zig"); +pub const Xoroshiro128 = @import("rand/Xoroshiro128.zig"); +pub const Sfc64 = @import("rand/Sfc64.zig"); + pub const Random = struct { fillFn: fn (r: *Random, buf: []u8) void, @@ -491,7 +489,7 @@ test "Random Biased" { // // The number of cycles is thus limited to 64-bits regardless of the engine, but this // is still plenty for practical purposes. -const SplitMix64 = struct { +pub const SplitMix64 = struct { s: u64, pub fn init(seed: u64) SplitMix64 { @@ -525,557 +523,6 @@ test "splitmix64 sequence" { } } -// PCG32 - http://www.pcg-random.org/ -// -// PRNG -pub const Pcg = struct { - const default_multiplier = 6364136223846793005; - - random: Random, - - s: u64, - i: u64, - - pub fn init(init_s: u64) Pcg { - var pcg = Pcg{ - .random = Random{ .fillFn = fill }, - .s = undefined, - .i = undefined, - }; - - pcg.seed(init_s); - return pcg; - } - - fn next(self: *Pcg) u32 { - const l = self.s; - self.s = l *% default_multiplier +% (self.i | 1); - - const xor_s = @truncate(u32, ((l >> 18) ^ l) >> 27); - const rot = @intCast(u32, l >> 59); - - return (xor_s >> @intCast(u5, rot)) | (xor_s << @intCast(u5, (0 -% rot) & 31)); - } - - fn seed(self: *Pcg, init_s: u64) void { - // Pcg requires 128-bits of seed. - var gen = SplitMix64.init(init_s); - self.seedTwo(gen.next(), gen.next()); - } - - fn seedTwo(self: *Pcg, init_s: u64, init_i: u64) void { - self.s = 0; - self.i = (init_s << 1) | 1; - self.s = self.s *% default_multiplier +% self.i; - self.s +%= init_i; - self.s = self.s *% default_multiplier +% self.i; - } - - fn fill(r: *Random, buf: []u8) void { - const self = @fieldParentPtr(Pcg, "random", r); - - var i: usize = 0; - const aligned_len = buf.len - (buf.len & 7); - - // Complete 4 byte segments. - while (i < aligned_len) : (i += 4) { - var n = self.next(); - comptime var j: usize = 0; - inline while (j < 4) : (j += 1) { - buf[i + j] = @truncate(u8, n); - n >>= 8; - } - } - - // Remaining. (cuts the stream) - if (i != buf.len) { - var n = self.next(); - while (i < buf.len) : (i += 1) { - buf[i] = @truncate(u8, n); - n >>= 4; - } - } - } -}; - -test "pcg sequence" { - var r = Pcg.init(0); - const s0: u64 = 0x9394bf54ce5d79de; - const s1: u64 = 0x84e9c579ef59bbf7; - r.seedTwo(s0, s1); - - const seq = [_]u32{ - 2881561918, - 3063928540, - 1199791034, - 2487695858, - 1479648952, - 3247963454, - }; - - for (seq) |s| { - expect(s == r.next()); - } -} - -// Xoroshiro128+ - http://xoroshiro.di.unimi.it/ -// -// PRNG -pub const Xoroshiro128 = struct { - random: Random, - - s: [2]u64, - - pub fn init(init_s: u64) Xoroshiro128 { - var x = Xoroshiro128{ - .random = Random{ .fillFn = fill }, - .s = undefined, - }; - - x.seed(init_s); - return x; - } - - fn next(self: *Xoroshiro128) u64 { - const s0 = self.s[0]; - var s1 = self.s[1]; - const r = s0 +% s1; - - s1 ^= s0; - self.s[0] = math.rotl(u64, s0, @as(u8, 55)) ^ s1 ^ (s1 << 14); - self.s[1] = math.rotl(u64, s1, @as(u8, 36)); - - return r; - } - - // Skip 2^64 places ahead in the sequence - fn jump(self: *Xoroshiro128) void { - var s0: u64 = 0; - var s1: u64 = 0; - - const table = [_]u64{ - 0xbeac0467eba5facb, - 0xd86b048b86aa9922, - }; - - inline for (table) |entry| { - var b: usize = 0; - while (b < 64) : (b += 1) { - if ((entry & (@as(u64, 1) << @intCast(u6, b))) != 0) { - s0 ^= self.s[0]; - s1 ^= self.s[1]; - } - _ = self.next(); - } - } - - self.s[0] = s0; - self.s[1] = s1; - } - - pub fn seed(self: *Xoroshiro128, init_s: u64) void { - // Xoroshiro requires 128-bits of seed. - var gen = SplitMix64.init(init_s); - - self.s[0] = gen.next(); - self.s[1] = gen.next(); - } - - fn fill(r: *Random, buf: []u8) void { - const self = @fieldParentPtr(Xoroshiro128, "random", r); - - var i: usize = 0; - const aligned_len = buf.len - (buf.len & 7); - - // Complete 8 byte segments. - while (i < aligned_len) : (i += 8) { - var n = self.next(); - comptime var j: usize = 0; - inline while (j < 8) : (j += 1) { - buf[i + j] = @truncate(u8, n); - n >>= 8; - } - } - - // Remaining. (cuts the stream) - if (i != buf.len) { - var n = self.next(); - while (i < buf.len) : (i += 1) { - buf[i] = @truncate(u8, n); - n >>= 8; - } - } - } -}; - -test "xoroshiro sequence" { - var r = Xoroshiro128.init(0); - r.s[0] = 0xaeecf86f7878dd75; - r.s[1] = 0x01cd153642e72622; - - const seq1 = [_]u64{ - 0xb0ba0da5bb600397, - 0x18a08afde614dccc, - 0xa2635b956a31b929, - 0xabe633c971efa045, - 0x9ac19f9706ca3cac, - 0xf62b426578c1e3fb, - }; - - for (seq1) |s| { - expect(s == r.next()); - } - - r.jump(); - - const seq2 = [_]u64{ - 0x95344a13556d3e22, - 0xb4fb32dafa4d00df, - 0xb2011d9ccdcfe2dd, - 0x05679a9b2119b908, - 0xa860a1da7c9cd8a0, - 0x658a96efe3f86550, - }; - - for (seq2) |s| { - expect(s == r.next()); - } -} - -// Gimli -// -// CSPRNG -pub const Gimli = struct { - random: Random, - state: std.crypto.core.Gimli, - - pub const secret_seed_length = 32; - - /// The seed must be uniform, secret and `secret_seed_length` bytes long. - /// It can be generated using `std.crypto.randomBytes()`. - pub fn init(secret_seed: [secret_seed_length]u8) Gimli { - var initial_state: [std.crypto.core.Gimli.BLOCKBYTES]u8 = undefined; - mem.copy(u8, initial_state[0..secret_seed_length], &secret_seed); - mem.set(u8, initial_state[secret_seed_length..], 0); - var self = Gimli{ - .random = Random{ .fillFn = fill }, - .state = std.crypto.core.Gimli.init(initial_state), - }; - return self; - } - - fn fill(r: *Random, buf: []u8) void { - const self = @fieldParentPtr(Gimli, "random", r); - - if (buf.len != 0) { - self.state.squeeze(buf); - } else { - self.state.permute(); - } - mem.set(u8, self.state.toSlice()[0..std.crypto.core.Gimli.RATE], 0); - } -}; - -// ISAAC64 - http://www.burtleburtle.net/bob/rand/isaacafa.html -// -// Follows the general idea of the implementation from here with a few shortcuts. -// https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html -pub const Isaac64 = struct { - random: Random, - - r: [256]u64, - m: [256]u64, - a: u64, - b: u64, - c: u64, - i: usize, - - pub fn init(init_s: u64) Isaac64 { - var isaac = Isaac64{ - .random = Random{ .fillFn = fill }, - .r = undefined, - .m = undefined, - .a = undefined, - .b = undefined, - .c = undefined, - .i = undefined, - }; - - // seed == 0 => same result as the unseeded reference implementation - isaac.seed(init_s, 1); - return isaac; - } - - fn step(self: *Isaac64, mix: u64, base: usize, comptime m1: usize, comptime m2: usize) void { - const x = self.m[base + m1]; - self.a = mix +% self.m[base + m2]; - - const y = self.a +% self.b +% self.m[@intCast(usize, (x >> 3) % self.m.len)]; - self.m[base + m1] = y; - - self.b = x +% self.m[@intCast(usize, (y >> 11) % self.m.len)]; - self.r[self.r.len - 1 - base - m1] = self.b; - } - - fn refill(self: *Isaac64) void { - const midpoint = self.r.len / 2; - - self.c +%= 1; - self.b +%= self.c; - - { - var i: usize = 0; - while (i < midpoint) : (i += 4) { - self.step(~(self.a ^ (self.a << 21)), i + 0, 0, midpoint); - self.step(self.a ^ (self.a >> 5), i + 1, 0, midpoint); - self.step(self.a ^ (self.a << 12), i + 2, 0, midpoint); - self.step(self.a ^ (self.a >> 33), i + 3, 0, midpoint); - } - } - - { - var i: usize = 0; - while (i < midpoint) : (i += 4) { - self.step(~(self.a ^ (self.a << 21)), i + 0, midpoint, 0); - self.step(self.a ^ (self.a >> 5), i + 1, midpoint, 0); - self.step(self.a ^ (self.a << 12), i + 2, midpoint, 0); - self.step(self.a ^ (self.a >> 33), i + 3, midpoint, 0); - } - } - - self.i = 0; - } - - fn next(self: *Isaac64) u64 { - if (self.i >= self.r.len) { - self.refill(); - } - - const value = self.r[self.i]; - self.i += 1; - return value; - } - - fn seed(self: *Isaac64, init_s: u64, comptime rounds: usize) void { - // We ignore the multi-pass requirement since we don't currently expose full access to - // seeding the self.m array completely. - mem.set(u64, self.m[0..], 0); - self.m[0] = init_s; - - // prescrambled golden ratio constants - var a = [_]u64{ - 0x647c4677a2884b7c, - 0xb9f8b322c73ac862, - 0x8c0ea5053d4712a0, - 0xb29b2e824a595524, - 0x82f053db8355e0ce, - 0x48fe4a0fa5a09315, - 0xae985bf2cbfc89ed, - 0x98f5704f6c44c0ab, - }; - - comptime var i: usize = 0; - inline while (i < rounds) : (i += 1) { - var j: usize = 0; - while (j < self.m.len) : (j += 8) { - comptime var x1: usize = 0; - inline while (x1 < 8) : (x1 += 1) { - a[x1] +%= self.m[j + x1]; - } - - a[0] -%= a[4]; - a[5] ^= a[7] >> 9; - a[7] +%= a[0]; - a[1] -%= a[5]; - a[6] ^= a[0] << 9; - a[0] +%= a[1]; - a[2] -%= a[6]; - a[7] ^= a[1] >> 23; - a[1] +%= a[2]; - a[3] -%= a[7]; - a[0] ^= a[2] << 15; - a[2] +%= a[3]; - a[4] -%= a[0]; - a[1] ^= a[3] >> 14; - a[3] +%= a[4]; - a[5] -%= a[1]; - a[2] ^= a[4] << 20; - a[4] +%= a[5]; - a[6] -%= a[2]; - a[3] ^= a[5] >> 17; - a[5] +%= a[6]; - a[7] -%= a[3]; - a[4] ^= a[6] << 14; - a[6] +%= a[7]; - - comptime var x2: usize = 0; - inline while (x2 < 8) : (x2 += 1) { - self.m[j + x2] = a[x2]; - } - } - } - - mem.set(u64, self.r[0..], 0); - self.a = 0; - self.b = 0; - self.c = 0; - self.i = self.r.len; // trigger refill on first value - } - - fn fill(r: *Random, buf: []u8) void { - const self = @fieldParentPtr(Isaac64, "random", r); - - var i: usize = 0; - const aligned_len = buf.len - (buf.len & 7); - - // Fill complete 64-byte segments - while (i < aligned_len) : (i += 8) { - var n = self.next(); - comptime var j: usize = 0; - inline while (j < 8) : (j += 1) { - buf[i + j] = @truncate(u8, n); - n >>= 8; - } - } - - // Fill trailing, ignoring excess (cut the stream). - if (i != buf.len) { - var n = self.next(); - while (i < buf.len) : (i += 1) { - buf[i] = @truncate(u8, n); - n >>= 8; - } - } - } -}; - -test "isaac64 sequence" { - var r = Isaac64.init(0); - - // from reference implementation - const seq = [_]u64{ - 0xf67dfba498e4937c, - 0x84a5066a9204f380, - 0xfee34bd5f5514dbb, - 0x4d1664739b8f80d6, - 0x8607459ab52a14aa, - 0x0e78bc5a98529e49, - 0xfe5332822ad13777, - 0x556c27525e33d01a, - 0x08643ca615f3149f, - 0xd0771faf3cb04714, - 0x30e86f68a37b008d, - 0x3074ebc0488a3adf, - 0x270645ea7a2790bc, - 0x5601a0a8d3763c6a, - 0x2f83071f53f325dd, - 0xb9090f3d42d2d2ea, - }; - - for (seq) |s| { - expect(s == r.next()); - } -} - -/// Sfc64 pseudo-random number generator from Practically Random. -/// Fastest engine of pracrand and smallest footprint. -/// See http://pracrand.sourceforge.net/ -pub const Sfc64 = struct { - random: Random, - - a: u64 = undefined, - b: u64 = undefined, - c: u64 = undefined, - counter: u64 = undefined, - - const Rotation = 24; - const RightShift = 11; - const LeftShift = 3; - - pub fn init(init_s: u64) Sfc64 { - var x = Sfc64{ - .random = Random{ .fillFn = fill }, - }; - - x.seed(init_s); - return x; - } - - fn next(self: *Sfc64) u64 { - const tmp = self.a +% self.b +% self.counter; - self.counter += 1; - self.a = self.b ^ (self.b >> RightShift); - self.b = self.c +% (self.c << LeftShift); - self.c = math.rotl(u64, self.c, Rotation) +% tmp; - return tmp; - } - - fn seed(self: *Sfc64, init_s: u64) void { - self.a = init_s; - self.b = init_s; - self.c = init_s; - self.counter = 1; - var i: u32 = 0; - while (i < 12) : (i += 1) { - _ = self.next(); - } - } - - fn fill(r: *Random, buf: []u8) void { - const self = @fieldParentPtr(Sfc64, "random", r); - - var i: usize = 0; - const aligned_len = buf.len - (buf.len & 7); - - // Complete 8 byte segments. - while (i < aligned_len) : (i += 8) { - var n = self.next(); - comptime var j: usize = 0; - inline while (j < 8) : (j += 1) { - buf[i + j] = @truncate(u8, n); - n >>= 8; - } - } - - // Remaining. (cuts the stream) - if (i != buf.len) { - var n = self.next(); - while (i < buf.len) : (i += 1) { - buf[i] = @truncate(u8, n); - n >>= 8; - } - } - } -}; - -test "Sfc64 sequence" { - // Unfortunately there does not seem to be an official test sequence. - var r = Sfc64.init(0); - - const seq = [_]u64{ - 0x3acfa029e3cc6041, - 0xf5b6515bf2ee419c, - 0x1259635894a29b61, - 0xb6ae75395f8ebd6, - 0x225622285ce302e2, - 0x520d28611395cb21, - 0xdb909c818901599d, - 0x8ffd195365216f57, - 0xe8c4ad5e258ac04a, - 0x8f8ef2c89fdb63ca, - 0xf9865b01d98d8e2f, - 0x46555871a65d08ba, - 0x66868677c6298fcd, - 0x2ce15a7e6329f57d, - 0xb2f1833ca91ca79, - 0x4b0890ac9bf453ca, - }; - - for (seq) |s| { - expectEqual(s, r.next()); - } -} - // Actual Random helper function tests, pcg engine is assumed correct. test "Random float" { var prng = DefaultPrng.init(0); @@ -1147,7 +594,7 @@ fn testRangeBias(r: *Random, start: i8, end: i8, biased: bool) void { test "CSPRNG" { var secret_seed: [DefaultCsprng.secret_seed_length]u8 = undefined; - try std.crypto.randomBytes(&secret_seed); + std.crypto.random.bytes(&secret_seed); var csprng = DefaultCsprng.init(secret_seed); const a = csprng.random.int(u64); const b = csprng.random.int(u64); diff --git a/lib/std/rand/Gimli.zig b/lib/std/rand/Gimli.zig new file mode 100644 index 0000000000..32331e7153 --- /dev/null +++ b/lib/std/rand/Gimli.zig @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! CSPRNG + +const std = @import("std"); +const Random = std.rand.Random; +const mem = std.mem; +const Gimli = @This(); + +random: Random, +state: std.crypto.core.Gimli, + +pub const secret_seed_length = 32; + +/// The seed must be uniform, secret and `secret_seed_length` bytes long. +pub fn init(secret_seed: [secret_seed_length]u8) Gimli { + var initial_state: [std.crypto.core.Gimli.BLOCKBYTES]u8 = undefined; + mem.copy(u8, initial_state[0..secret_seed_length], &secret_seed); + mem.set(u8, initial_state[secret_seed_length..], 0); + var self = Gimli{ + .random = Random{ .fillFn = fill }, + .state = std.crypto.core.Gimli.init(initial_state), + }; + return self; +} + +fn fill(r: *Random, buf: []u8) void { + const self = @fieldParentPtr(Gimli, "random", r); + + if (buf.len != 0) { + self.state.squeeze(buf); + } else { + self.state.permute(); + } + mem.set(u8, self.state.toSlice()[0..std.crypto.core.Gimli.RATE], 0); +} diff --git a/lib/std/rand/Isaac64.zig b/lib/std/rand/Isaac64.zig new file mode 100644 index 0000000000..a079f19b9e --- /dev/null +++ b/lib/std/rand/Isaac64.zig @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! ISAAC64 - http://www.burtleburtle.net/bob/rand/isaacafa.html +//! +//! Follows the general idea of the implementation from here with a few shortcuts. +//! https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html + +const std = @import("std"); +const Random = std.rand.Random; +const mem = std.mem; +const Isaac64 = @This(); + +random: Random, + +r: [256]u64, +m: [256]u64, +a: u64, +b: u64, +c: u64, +i: usize, + +pub fn init(init_s: u64) Isaac64 { + var isaac = Isaac64{ + .random = Random{ .fillFn = fill }, + .r = undefined, + .m = undefined, + .a = undefined, + .b = undefined, + .c = undefined, + .i = undefined, + }; + + // seed == 0 => same result as the unseeded reference implementation + isaac.seed(init_s, 1); + return isaac; +} + +fn step(self: *Isaac64, mix: u64, base: usize, comptime m1: usize, comptime m2: usize) void { + const x = self.m[base + m1]; + self.a = mix +% self.m[base + m2]; + + const y = self.a +% self.b +% self.m[@intCast(usize, (x >> 3) % self.m.len)]; + self.m[base + m1] = y; + + self.b = x +% self.m[@intCast(usize, (y >> 11) % self.m.len)]; + self.r[self.r.len - 1 - base - m1] = self.b; +} + +fn refill(self: *Isaac64) void { + const midpoint = self.r.len / 2; + + self.c +%= 1; + self.b +%= self.c; + + { + var i: usize = 0; + while (i < midpoint) : (i += 4) { + self.step(~(self.a ^ (self.a << 21)), i + 0, 0, midpoint); + self.step(self.a ^ (self.a >> 5), i + 1, 0, midpoint); + self.step(self.a ^ (self.a << 12), i + 2, 0, midpoint); + self.step(self.a ^ (self.a >> 33), i + 3, 0, midpoint); + } + } + + { + var i: usize = 0; + while (i < midpoint) : (i += 4) { + self.step(~(self.a ^ (self.a << 21)), i + 0, midpoint, 0); + self.step(self.a ^ (self.a >> 5), i + 1, midpoint, 0); + self.step(self.a ^ (self.a << 12), i + 2, midpoint, 0); + self.step(self.a ^ (self.a >> 33), i + 3, midpoint, 0); + } + } + + self.i = 0; +} + +fn next(self: *Isaac64) u64 { + if (self.i >= self.r.len) { + self.refill(); + } + + const value = self.r[self.i]; + self.i += 1; + return value; +} + +fn seed(self: *Isaac64, init_s: u64, comptime rounds: usize) void { + // We ignore the multi-pass requirement since we don't currently expose full access to + // seeding the self.m array completely. + mem.set(u64, self.m[0..], 0); + self.m[0] = init_s; + + // prescrambled golden ratio constants + var a = [_]u64{ + 0x647c4677a2884b7c, + 0xb9f8b322c73ac862, + 0x8c0ea5053d4712a0, + 0xb29b2e824a595524, + 0x82f053db8355e0ce, + 0x48fe4a0fa5a09315, + 0xae985bf2cbfc89ed, + 0x98f5704f6c44c0ab, + }; + + comptime var i: usize = 0; + inline while (i < rounds) : (i += 1) { + var j: usize = 0; + while (j < self.m.len) : (j += 8) { + comptime var x1: usize = 0; + inline while (x1 < 8) : (x1 += 1) { + a[x1] +%= self.m[j + x1]; + } + + a[0] -%= a[4]; + a[5] ^= a[7] >> 9; + a[7] +%= a[0]; + a[1] -%= a[5]; + a[6] ^= a[0] << 9; + a[0] +%= a[1]; + a[2] -%= a[6]; + a[7] ^= a[1] >> 23; + a[1] +%= a[2]; + a[3] -%= a[7]; + a[0] ^= a[2] << 15; + a[2] +%= a[3]; + a[4] -%= a[0]; + a[1] ^= a[3] >> 14; + a[3] +%= a[4]; + a[5] -%= a[1]; + a[2] ^= a[4] << 20; + a[4] +%= a[5]; + a[6] -%= a[2]; + a[3] ^= a[5] >> 17; + a[5] +%= a[6]; + a[7] -%= a[3]; + a[4] ^= a[6] << 14; + a[6] +%= a[7]; + + comptime var x2: usize = 0; + inline while (x2 < 8) : (x2 += 1) { + self.m[j + x2] = a[x2]; + } + } + } + + mem.set(u64, self.r[0..], 0); + self.a = 0; + self.b = 0; + self.c = 0; + self.i = self.r.len; // trigger refill on first value +} + +fn fill(r: *Random, buf: []u8) void { + const self = @fieldParentPtr(Isaac64, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Fill complete 64-byte segments + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Fill trailing, ignoring excess (cut the stream). + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } +} + +test "isaac64 sequence" { + var r = Isaac64.init(0); + + // from reference implementation + const seq = [_]u64{ + 0xf67dfba498e4937c, + 0x84a5066a9204f380, + 0xfee34bd5f5514dbb, + 0x4d1664739b8f80d6, + 0x8607459ab52a14aa, + 0x0e78bc5a98529e49, + 0xfe5332822ad13777, + 0x556c27525e33d01a, + 0x08643ca615f3149f, + 0xd0771faf3cb04714, + 0x30e86f68a37b008d, + 0x3074ebc0488a3adf, + 0x270645ea7a2790bc, + 0x5601a0a8d3763c6a, + 0x2f83071f53f325dd, + 0xb9090f3d42d2d2ea, + }; + + for (seq) |s| { + std.testing.expect(s == r.next()); + } +} diff --git a/lib/std/rand/Pcg.zig b/lib/std/rand/Pcg.zig new file mode 100644 index 0000000000..1bbe8beb63 --- /dev/null +++ b/lib/std/rand/Pcg.zig @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! PCG32 - http://www.pcg-random.org/ +//! +//! PRNG + +const std = @import("std"); +const Random = std.rand.Random; +const Pcg = @This(); + +const default_multiplier = 6364136223846793005; + +random: Random, + +s: u64, +i: u64, + +pub fn init(init_s: u64) Pcg { + var pcg = Pcg{ + .random = Random{ .fillFn = fill }, + .s = undefined, + .i = undefined, + }; + + pcg.seed(init_s); + return pcg; +} + +fn next(self: *Pcg) u32 { + const l = self.s; + self.s = l *% default_multiplier +% (self.i | 1); + + const xor_s = @truncate(u32, ((l >> 18) ^ l) >> 27); + const rot = @intCast(u32, l >> 59); + + return (xor_s >> @intCast(u5, rot)) | (xor_s << @intCast(u5, (0 -% rot) & 31)); +} + +fn seed(self: *Pcg, init_s: u64) void { + // Pcg requires 128-bits of seed. + var gen = std.rand.SplitMix64.init(init_s); + self.seedTwo(gen.next(), gen.next()); +} + +fn seedTwo(self: *Pcg, init_s: u64, init_i: u64) void { + self.s = 0; + self.i = (init_s << 1) | 1; + self.s = self.s *% default_multiplier +% self.i; + self.s +%= init_i; + self.s = self.s *% default_multiplier +% self.i; +} + +fn fill(r: *Random, buf: []u8) void { + const self = @fieldParentPtr(Pcg, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 4 byte segments. + while (i < aligned_len) : (i += 4) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 4) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 4; + } + } +} + +test "pcg sequence" { + var r = Pcg.init(0); + const s0: u64 = 0x9394bf54ce5d79de; + const s1: u64 = 0x84e9c579ef59bbf7; + r.seedTwo(s0, s1); + + const seq = [_]u32{ + 2881561918, + 3063928540, + 1199791034, + 2487695858, + 1479648952, + 3247963454, + }; + + for (seq) |s| { + std.testing.expect(s == r.next()); + } +} diff --git a/lib/std/rand/Sfc64.zig b/lib/std/rand/Sfc64.zig new file mode 100644 index 0000000000..feba5d884c --- /dev/null +++ b/lib/std/rand/Sfc64.zig @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! Sfc64 pseudo-random number generator from Practically Random. +//! Fastest engine of pracrand and smallest footprint. +//! See http://pracrand.sourceforge.net/ + +const std = @import("std"); +const Random = std.rand.Random; +const math = std.math; +const Sfc64 = @This(); + +random: Random, + +a: u64 = undefined, +b: u64 = undefined, +c: u64 = undefined, +counter: u64 = undefined, + +const Rotation = 24; +const RightShift = 11; +const LeftShift = 3; + +pub fn init(init_s: u64) Sfc64 { + var x = Sfc64{ + .random = Random{ .fillFn = fill }, + }; + + x.seed(init_s); + return x; +} + +fn next(self: *Sfc64) u64 { + const tmp = self.a +% self.b +% self.counter; + self.counter += 1; + self.a = self.b ^ (self.b >> RightShift); + self.b = self.c +% (self.c << LeftShift); + self.c = math.rotl(u64, self.c, Rotation) +% tmp; + return tmp; +} + +fn seed(self: *Sfc64, init_s: u64) void { + self.a = init_s; + self.b = init_s; + self.c = init_s; + self.counter = 1; + var i: u32 = 0; + while (i < 12) : (i += 1) { + _ = self.next(); + } +} + +fn fill(r: *Random, buf: []u8) void { + const self = @fieldParentPtr(Sfc64, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 8 byte segments. + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } +} + +test "Sfc64 sequence" { + // Unfortunately there does not seem to be an official test sequence. + var r = Sfc64.init(0); + + const seq = [_]u64{ + 0x3acfa029e3cc6041, + 0xf5b6515bf2ee419c, + 0x1259635894a29b61, + 0xb6ae75395f8ebd6, + 0x225622285ce302e2, + 0x520d28611395cb21, + 0xdb909c818901599d, + 0x8ffd195365216f57, + 0xe8c4ad5e258ac04a, + 0x8f8ef2c89fdb63ca, + 0xf9865b01d98d8e2f, + 0x46555871a65d08ba, + 0x66868677c6298fcd, + 0x2ce15a7e6329f57d, + 0xb2f1833ca91ca79, + 0x4b0890ac9bf453ca, + }; + + for (seq) |s| { + std.testing.expectEqual(s, r.next()); + } +} diff --git a/lib/std/rand/Xoroshiro128.zig b/lib/std/rand/Xoroshiro128.zig new file mode 100644 index 0000000000..2cc9bf9070 --- /dev/null +++ b/lib/std/rand/Xoroshiro128.zig @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! Xoroshiro128+ - http://xoroshiro.di.unimi.it/ +//! +//! PRNG + +const std = @import("std"); +const Random = std.rand.Random; +const math = std.math; +const Xoroshiro128 = @This(); + +random: Random, + +s: [2]u64, + +pub fn init(init_s: u64) Xoroshiro128 { + var x = Xoroshiro128{ + .random = Random{ .fillFn = fill }, + .s = undefined, + }; + + x.seed(init_s); + return x; +} + +fn next(self: *Xoroshiro128) u64 { + const s0 = self.s[0]; + var s1 = self.s[1]; + const r = s0 +% s1; + + s1 ^= s0; + self.s[0] = math.rotl(u64, s0, @as(u8, 55)) ^ s1 ^ (s1 << 14); + self.s[1] = math.rotl(u64, s1, @as(u8, 36)); + + return r; +} + +// Skip 2^64 places ahead in the sequence +fn jump(self: *Xoroshiro128) void { + var s0: u64 = 0; + var s1: u64 = 0; + + const table = [_]u64{ + 0xbeac0467eba5facb, + 0xd86b048b86aa9922, + }; + + inline for (table) |entry| { + var b: usize = 0; + while (b < 64) : (b += 1) { + if ((entry & (@as(u64, 1) << @intCast(u6, b))) != 0) { + s0 ^= self.s[0]; + s1 ^= self.s[1]; + } + _ = self.next(); + } + } + + self.s[0] = s0; + self.s[1] = s1; +} + +pub fn seed(self: *Xoroshiro128, init_s: u64) void { + // Xoroshiro requires 128-bits of seed. + var gen = std.rand.SplitMix64.init(init_s); + + self.s[0] = gen.next(); + self.s[1] = gen.next(); +} + +fn fill(r: *Random, buf: []u8) void { + const self = @fieldParentPtr(Xoroshiro128, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 8 byte segments. + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } +} + +test "xoroshiro sequence" { + var r = Xoroshiro128.init(0); + r.s[0] = 0xaeecf86f7878dd75; + r.s[1] = 0x01cd153642e72622; + + const seq1 = [_]u64{ + 0xb0ba0da5bb600397, + 0x18a08afde614dccc, + 0xa2635b956a31b929, + 0xabe633c971efa045, + 0x9ac19f9706ca3cac, + 0xf62b426578c1e3fb, + }; + + for (seq1) |s| { + std.testing.expect(s == r.next()); + } + + r.jump(); + + const seq2 = [_]u64{ + 0x95344a13556d3e22, + 0xb4fb32dafa4d00df, + 0xb2011d9ccdcfe2dd, + 0x05679a9b2119b908, + 0xa860a1da7c9cd8a0, + 0x658a96efe3f86550, + }; + + for (seq2) |s| { + std.testing.expect(s == r.next()); + } +} diff --git a/lib/std/special/c.zig b/lib/std/special/c.zig index 449b70d6b0..84aeb6aebb 100644 --- a/lib/std/special/c.zig +++ b/lib/std/special/c.zig @@ -634,6 +634,16 @@ export fn cosf(a: f32) f32 { return math.cos(a); } +export fn sincos(a: f64, r_sin: *f64, r_cos: *f64) void { + r_sin.* = math.sin(a); + r_cos.* = math.cos(a); +} + +export fn sincosf(a: f32, r_sin: *f32, r_cos: *f32) void { + r_sin.* = math.sin(a); + r_cos.* = math.cos(a); +} + export fn exp(a: f64) f64 { return math.exp(a); } diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 2b2fe78262..4bb9202858 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -36,7 +36,7 @@ pub fn main() anyerror!void { } std.testing.log_level = .warn; - var test_node = root_node.start(test_fn.name, null); + var test_node = root_node.start(test_fn.name, 0); test_node.activate(); progress.refresh(); if (progress.terminal == null) { diff --git a/lib/std/start.zig b/lib/std/start.zig index 7d3a9c45c7..01fe43ca35 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -10,6 +10,7 @@ const std = @import("std.zig"); const builtin = std.builtin; const assert = std.debug.assert; const uefi = std.os.uefi; +const tlcsprng = @import("crypto/tlcsprng.zig"); var argc_argv_ptr: [*]usize = undefined; diff --git a/lib/std/std.zig b/lib/std/std.zig index f6da7afc55..5fbf2662b9 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -29,7 +29,7 @@ pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayE pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice; pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian; pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue; -pub const Progress = @import("progress.zig").Progress; +pub const Progress = @import("Progress.zig"); pub const ResetEvent = @import("reset_event.zig").ResetEvent; pub const SemanticVersion = @import("SemanticVersion.zig"); pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList; diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 69df01190d..63054892db 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -303,8 +303,7 @@ fn getCwdOrWasiPreopen() std.fs.Dir { pub fn tmpDir(opts: std.fs.Dir.OpenDirOptions) TmpDir { var random_bytes: [TmpDir.random_bytes_count]u8 = undefined; - std.crypto.randomBytes(&random_bytes) catch - @panic("unable to make tmp dir for testing: unable to get random bytes"); + std.crypto.random.bytes(&random_bytes); var sub_path: [TmpDir.sub_path_len]u8 = undefined; std.fs.base64_encoder.encode(&sub_path, &random_bytes); |