From 0d22a00f6fde79f851a7d19c2096c07f541ed0be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 7 Mar 2018 03:55:52 -0500 Subject: *WIP* async/await TCP server --- std/mem.zig | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'std/mem.zig') diff --git a/std/mem.zig b/std/mem.zig index 97cb35ae65..8a59d6251b 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -3,6 +3,7 @@ const debug = std.debug; const assert = debug.assert; const math = std.math; const builtin = @import("builtin"); +const mem = this; pub const Allocator = struct { const Error = error {OutOfMemory}; @@ -550,3 +551,28 @@ test "std.mem.rotate" { assert(eql(i32, arr, []i32{ 1, 2, 4, 5, 3 })); } + +// TODO: When https://github.com/zig-lang/zig/issues/649 is solved these can be done by +// endian-casting the pointer and then dereferencing + +pub fn endianSwapIfLe(comptime T: type, x: T) T { + return endianSwapIf(builtin.Endian.Little, T, x); +} + +pub fn endianSwapIfBe(comptime T: type, x: T) T { + return endianSwapIf(builtin.Endian.Big, T, x); +} + +pub fn endianSwapIf(endian: builtin.Endian, comptime T: type, x: T) T { + return if (builtin.endian == endian) endianSwap(T, x) else x; +} + +pub fn endianSwap(comptime T: type, x: T) T { + var buf: [@sizeOf(T)]u8 = undefined; + mem.writeInt(buf[0..], x, builtin.Endian.Little); + return mem.readInt(buf, T, builtin.Endian.Big); +} + +test "std.mem.endianSwap" { + assert(endianSwap(u32, 0xDEADBEEF) == 0xEFBEADDE); +} -- cgit v1.2.3 From 7af6ed3f20bbf0459ce6aed833c7e170ee6c927b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 22 Apr 2018 12:52:28 -0400 Subject: add alignment docs --- std/mem.zig | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'std/mem.zig') diff --git a/std/mem.zig b/std/mem.zig index 8a59d6251b..eb67ce83ef 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -11,6 +11,8 @@ pub const Allocator = struct { /// Allocate byte_count bytes and return them in a slice, with the /// slice's pointer aligned at least to alignment bytes. /// The returned newly allocated memory is undefined. + /// `alignment` is guaranteed to be >= 1 + /// `alignment` is guaranteed to be a power of 2 allocFn: fn (self: &Allocator, byte_count: usize, alignment: u29) Error![]u8, /// If `new_byte_count > old_mem.len`: @@ -22,6 +24,8 @@ pub const Allocator = struct { /// * alignment <= alignment of old_mem.ptr /// /// The returned newly allocated memory is undefined. + /// `alignment` is guaranteed to be >= 1 + /// `alignment` is guaranteed to be a power of 2 reallocFn: fn (self: &Allocator, old_mem: []u8, new_byte_count: usize, alignment: u29) Error![]u8, /// Guaranteed: `old_mem.len` is the same as what was returned from `allocFn` or `reallocFn` -- cgit v1.2.3 From 13076d5f225cf8ca2cddf1b67478baebabe8b13f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 24 Apr 2018 20:50:21 -0400 Subject: std.mem: add more slice manipulation functions * add std.mem.trimLeft * add std.mem.trimRight * add std.mem.trimRight * add std.mem.lastIndexOfScalar * add std.mem.lastIndexOfAny * add std.mem.lastIndexOf * add std.mem.endsWith closes #944 Thanks Braedon Wooding for the original PR --- std/mem.zig | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-) (limited to 'std/mem.zig') diff --git a/std/mem.zig b/std/mem.zig index eb67ce83ef..cc3161cddd 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -20,7 +20,7 @@ pub const Allocator = struct { /// * alignment >= alignment of old_mem.ptr /// /// If `new_byte_count <= old_mem.len`: - /// * this function must return successfully. + /// * this function must return successfully. /// * alignment <= alignment of old_mem.ptr /// /// The returned newly allocated memory is undefined. @@ -174,6 +174,20 @@ pub fn dupe(allocator: &Allocator, comptime T: type, m: []const T) ![]T { return new_buf; } +/// Remove values from the beginning of a slice. +pub fn trimLeft(comptime T: type, slice: []const T, values_to_strip: []const T) []const T { + var begin: usize = 0; + while (begin < slice.len and indexOfScalar(T, values_to_strip, slice[begin]) != null) : (begin += 1) {} + return slice[begin..]; +} + +/// Remove values from the end of a slice. +pub fn trimRight(comptime T: type, slice: []const T, values_to_strip: []const T) []const T { + var end: usize = slice.len; + while (end > 0 and indexOfScalar(T, values_to_strip, slice[end - 1]) != null) : (end -= 1) {} + return slice[0..end]; +} + /// Remove values from the beginning and end of a slice. pub fn trim(comptime T: type, slice: []const T, values_to_strip: []const T) []const T { var begin: usize = 0; @@ -184,6 +198,8 @@ pub fn trim(comptime T: type, slice: []const T, values_to_strip: []const T) []co } test "mem.trim" { + assert(eql(u8, trimLeft(u8, " foo\n ", " \n"), "foo\n ")); + assert(eql(u8, trimRight(u8, " foo\n ", " \n"), " foo")); assert(eql(u8, trim(u8, " foo\n ", " \n"), "foo")); assert(eql(u8, trim(u8, "foo", " \n"), "foo")); } @@ -193,6 +209,17 @@ pub fn indexOfScalar(comptime T: type, slice: []const T, value: T) ?usize { return indexOfScalarPos(T, slice, 0, value); } +/// Linear search for the last index of a scalar value inside a slice. +pub fn lastIndexOfScalar(comptime T: type, slice: []const T, value: T) ?usize { + var i: usize = slice.len; + while (i != 0) { + i -= 1; + if (slice[i] == value) + return i; + } + return null; +} + pub fn indexOfScalarPos(comptime T: type, slice: []const T, start_index: usize, value: T) ?usize { var i: usize = start_index; while (i < slice.len) : (i += 1) { @@ -206,6 +233,18 @@ pub fn indexOfAny(comptime T: type, slice: []const T, values: []const T) ?usize return indexOfAnyPos(T, slice, 0, values); } +pub fn lastIndexOfAny(comptime T: type, slice: []const T, values: []const T) ?usize { + var i: usize = slice.len; + while (i != 0) { + i -= 1; + for (values) |value| { + if (slice[i] == value) + return i; + } + } + return null; +} + pub fn indexOfAnyPos(comptime T: type, slice: []const T, start_index: usize, values: []const T) ?usize { var i: usize = start_index; while (i < slice.len) : (i += 1) { @@ -221,6 +260,22 @@ pub fn indexOf(comptime T: type, haystack: []const T, needle: []const T) ?usize return indexOfPos(T, haystack, 0, needle); } +/// Find the index in a slice of a sub-slice, searching from the end backwards. +/// To start looking at a different index, slice the haystack first. +/// TODO is there even a better algorithm for this? +pub fn lastIndexOf(comptime T: type, haystack: []const T, needle: []const T) ?usize { + if (needle.len > haystack.len) + return null; + + var i: usize = haystack.len - needle.len; + while (true) : (i -= 1) { + if (mem.eql(T, haystack[i..i+needle.len], needle)) + return i; + if (i == 0) + return null; + } +} + // TODO boyer-moore algorithm pub fn indexOfPos(comptime T: type, haystack: []const T, start_index: usize, needle: []const T) ?usize { if (needle.len > haystack.len) @@ -237,9 +292,19 @@ pub fn indexOfPos(comptime T: type, haystack: []const T, start_index: usize, nee test "mem.indexOf" { assert(??indexOf(u8, "one two three four", "four") == 14); + assert(??lastIndexOf(u8, "one two three two four", "two") == 14); assert(indexOf(u8, "one two three four", "gour") == null); + assert(lastIndexOf(u8, "one two three four", "gour") == null); assert(??indexOf(u8, "foo", "foo") == 0); + assert(??lastIndexOf(u8, "foo", "foo") == 0); assert(indexOf(u8, "foo", "fool") == null); + assert(lastIndexOf(u8, "foo", "lfoo") == null); + assert(lastIndexOf(u8, "foo", "fool") == null); + + assert(??indexOf(u8, "foo foo", "foo") == 0); + assert(??lastIndexOf(u8, "foo foo", "foo") == 4); + assert(??lastIndexOfAny(u8, "boo, cat", "abo") == 6); + assert(??lastIndexOfScalar(u8, "boo", 'o') == 2); } /// Reads an integer from memory with size equal to bytes.len. @@ -359,9 +424,24 @@ pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool return if (needle.len > haystack.len) false else eql(T, haystack[0 .. needle.len], needle); } +test "mem.startsWith" { + assert(startsWith(u8, "Bob", "Bo")); + assert(!startsWith(u8, "Needle in haystack", "haystack")); +} + +pub fn endsWith(comptime T: type, haystack: []const T, needle: []const T) bool { + return if (needle.len > haystack.len) false else eql(T, haystack[haystack.len - needle.len ..], needle); +} + + +test "mem.endsWith" { + assert(endsWith(u8, "Needle in haystack", "haystack")); + assert(!endsWith(u8, "Bob", "Bo")); +} + pub const SplitIterator = struct { buffer: []const u8, - split_bytes: []const u8, + split_bytes: []const u8, index: usize, pub fn next(self: &SplitIterator) ?[]const u8 { -- cgit v1.2.3 From 6376d96824c5205ecc02b2c621bcef5dc78f1a81 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 29 Apr 2018 02:40:22 -0400 Subject: support kernel threads for windows * remove std.os.spawnThreadAllocator - windows does not support an explicit stack, so using an allocator for a thread stack space does not work. * std.os.spawnThread - instead of accepting a stack argument, the implementation will directly allocate using OS-specific APIs. --- std/atomic/queue.zig | 8 +-- std/atomic/stack.zig | 8 +-- std/mem.zig | 1 + std/os/index.zig | 165 ++++++++++++++++++++++++++++++----------------- std/os/test.zig | 20 ++---- std/os/windows/index.zig | 6 ++ 6 files changed, 120 insertions(+), 88 deletions(-) (limited to 'std/mem.zig') diff --git a/std/atomic/queue.zig b/std/atomic/queue.zig index dd9b869f02..1acecbab2c 100644 --- a/std/atomic/queue.zig +++ b/std/atomic/queue.zig @@ -53,10 +53,6 @@ const puts_per_thread = 10000; const put_thread_count = 3; test "std.atomic.queue" { - if (builtin.os == builtin.Os.windows) { - // TODO implement kernel threads for windows - return; - } var direct_allocator = std.heap.DirectAllocator.init(); defer direct_allocator.deinit(); @@ -79,11 +75,11 @@ test "std.atomic.queue" { var putters: [put_thread_count]&std.os.Thread = undefined; for (putters) |*t| { - *t = try std.os.spawnThreadAllocator(a, &context, startPuts); + *t = try std.os.spawnThread(&context, startPuts); } var getters: [put_thread_count]&std.os.Thread = undefined; for (getters) |*t| { - *t = try std.os.spawnThreadAllocator(a, &context, startGets); + *t = try std.os.spawnThread(&context, startGets); } for (putters) |t| t.wait(); diff --git a/std/atomic/stack.zig b/std/atomic/stack.zig index 9f2ceacfa3..accbcc942a 100644 --- a/std/atomic/stack.zig +++ b/std/atomic/stack.zig @@ -60,10 +60,6 @@ const puts_per_thread = 1000; const put_thread_count = 3; test "std.atomic.stack" { - if (builtin.os == builtin.Os.windows) { - // TODO implement kernel threads for windows - return; - } var direct_allocator = std.heap.DirectAllocator.init(); defer direct_allocator.deinit(); @@ -85,11 +81,11 @@ test "std.atomic.stack" { var putters: [put_thread_count]&std.os.Thread = undefined; for (putters) |*t| { - *t = try std.os.spawnThreadAllocator(a, &context, startPuts); + *t = try std.os.spawnThread(&context, startPuts); } var getters: [put_thread_count]&std.os.Thread = undefined; for (getters) |*t| { - *t = try std.os.spawnThreadAllocator(a, &context, startGets); + *t = try std.os.spawnThread(&context, startGets); } for (putters) |t| t.wait(); diff --git a/std/mem.zig b/std/mem.zig index cc3161cddd..0f66f549cc 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -32,6 +32,7 @@ pub const Allocator = struct { freeFn: fn (self: &Allocator, old_mem: []u8) void, fn create(self: &Allocator, comptime T: type) !&T { + if (@sizeOf(T) == 0) return &{}; const slice = try self.alloc(T, 1); return &slice[0]; } diff --git a/std/os/index.zig b/std/os/index.zig index 85e46a1bf9..6842fd0fb3 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -2347,18 +2347,30 @@ pub fn posixGetSockOptConnectError(sockfd: i32) PosixConnectError!void { } pub const Thread = struct { - pid: pid_t, - allocator: ?&mem.Allocator, - stack: []u8, - pthread_handle: pthread_t, + data: Data, pub const use_pthreads = is_posix and builtin.link_libc; - const pthread_t = if (use_pthreads) c.pthread_t else void; - const pid_t = if (!use_pthreads) i32 else void; + const Data = if (use_pthreads) struct { + handle: c.pthread_t, + stack_addr: usize, + stack_len: usize, + } else switch (builtin.os) { + builtin.Os.linux => struct { + pid: i32, + stack_addr: usize, + stack_len: usize, + }, + builtin.Os.windows => struct { + handle: windows.HANDLE, + alloc_start: &c_void, + heap_handle: windows.HANDLE, + }, + else => @compileError("Unsupported OS"), + }; pub fn wait(self: &const Thread) void { if (use_pthreads) { - const err = c.pthread_join(self.pthread_handle, null); + const err = c.pthread_join(self.data.handle, null); switch (err) { 0 => {}, posix.EINVAL => unreachable, @@ -2366,23 +2378,27 @@ pub const Thread = struct { posix.EDEADLK => unreachable, else => unreachable, } - } else if (builtin.os == builtin.Os.linux) { - while (true) { - const pid_value = @atomicLoad(i32, &self.pid, builtin.AtomicOrder.SeqCst); - if (pid_value == 0) break; - const rc = linux.futex_wait(@ptrToInt(&self.pid), linux.FUTEX_WAIT, pid_value, null); - switch (linux.getErrno(rc)) { - 0 => continue, - posix.EINTR => continue, - posix.EAGAIN => continue, - else => unreachable, + assert(posix.munmap(self.data.stack_addr, self.data.stack_len) == 0); + } else switch (builtin.os) { + builtin.Os.linux => { + while (true) { + const pid_value = @atomicLoad(i32, &self.data.pid, builtin.AtomicOrder.SeqCst); + if (pid_value == 0) break; + const rc = linux.futex_wait(@ptrToInt(&self.data.pid), linux.FUTEX_WAIT, pid_value, null); + switch (linux.getErrno(rc)) { + 0 => continue, + posix.EINTR => continue, + posix.EAGAIN => continue, + else => unreachable, + } } - } - } else { - @compileError("Unsupported OS"); - } - if (self.allocator) |a| { - a.free(self.stack); + assert(posix.munmap(self.data.stack_addr, self.data.stack_len) == 0); + }, + builtin.Os.windows => { + assert(windows.WaitForSingleObject(self.data.handle, windows.INFINITE) == windows.WAIT_OBJECT_0); + assert(windows.HeapFree(self.data.heap_handle, 0, self.data.alloc_start) != 0); + }, + else => @compileError("Unsupported OS"), } } }; @@ -2407,52 +2423,60 @@ pub const SpawnThreadError = error { /// be copied. SystemResources, - /// pthreads requires at least 16384 bytes of stack space - StackTooSmall, + /// Not enough userland memory to spawn the thread. + OutOfMemory, Unexpected, }; -pub const SpawnThreadAllocatorError = SpawnThreadError || error{OutOfMemory}; - /// caller must call wait on the returned thread /// fn startFn(@typeOf(context)) T /// where T is u8, noreturn, void, or !void -pub fn spawnThreadAllocator(allocator: &mem.Allocator, context: var, comptime startFn: var) SpawnThreadAllocatorError!&Thread { +/// caller must call wait on the returned thread +pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!&Thread { // TODO compile-time call graph analysis to determine stack upper bound // https://github.com/zig-lang/zig/issues/157 const default_stack_size = 8 * 1024 * 1024; - const stack_bytes = try allocator.alignedAlloc(u8, os.page_size, default_stack_size); - const thread = try spawnThread(stack_bytes, context, startFn); - thread.allocator = allocator; - return thread; -} -/// stack must be big enough to store one Thread and one @typeOf(context), each with default alignment, at the end -/// fn startFn(@typeOf(context)) T -/// where T is u8, noreturn, void, or !void -/// caller must call wait on the returned thread -pub fn spawnThread(stack: []align(os.page_size) u8, context: var, comptime startFn: var) SpawnThreadError!&Thread { const Context = @typeOf(context); comptime assert(@ArgType(@typeOf(startFn), 0) == Context); - var stack_end: usize = @ptrToInt(stack.ptr) + stack.len; - var arg: usize = undefined; - if (@sizeOf(Context) != 0) { - stack_end -= @sizeOf(Context); - stack_end -= stack_end % @alignOf(Context); - assert(stack_end >= @ptrToInt(stack.ptr)); - const context_ptr = @alignCast(@alignOf(Context), @intToPtr(&Context, stack_end)); - *context_ptr = context; - arg = stack_end; - } + if (builtin.os == builtin.Os.windows) { + const WinThread = struct { + const OuterContext = struct { + thread: Thread, + inner: Context, + }; + extern fn threadMain(arg: windows.LPVOID) windows.DWORD { + if (@sizeOf(Context) == 0) { + return startFn({}); + } else { + return startFn(*@ptrCast(&Context, @alignCast(@alignOf(Context), arg))); + } + } + }; - stack_end -= @sizeOf(Thread); - stack_end -= stack_end % @alignOf(Thread); - assert(stack_end >= @ptrToInt(stack.ptr)); - const thread_ptr = @alignCast(@alignOf(Thread), @intToPtr(&Thread, stack_end)); - thread_ptr.stack = stack; - thread_ptr.allocator = null; + const heap_handle = windows.GetProcessHeap() ?? return SpawnThreadError.OutOfMemory; + const byte_count = @alignOf(WinThread.OuterContext) + @sizeOf(WinThread.OuterContext); + const bytes_ptr = windows.HeapAlloc(heap_handle, 0, byte_count) ?? return SpawnThreadError.OutOfMemory; + errdefer assert(windows.HeapFree(heap_handle, 0, bytes_ptr) != 0); + const bytes = @ptrCast(&u8, bytes_ptr)[0..byte_count]; + const outer_context = std.heap.FixedBufferAllocator.init(bytes).allocator.create(WinThread.OuterContext) catch unreachable; + outer_context.inner = context; + outer_context.thread.data.heap_handle = heap_handle; + outer_context.thread.data.alloc_start = bytes_ptr; + + const parameter = if (@sizeOf(Context) == 0) null else @ptrCast(&c_void, &outer_context.inner); + outer_context.thread.data.handle = windows.CreateThread(null, default_stack_size, WinThread.threadMain, + parameter, 0, null) ?? + { + const err = windows.GetLastError(); + return switch (err) { + else => os.unexpectedErrorWindows(err), + }; + }; + return &outer_context.thread; + } const MainFuncs = struct { extern fn linuxThreadMain(ctx_addr: usize) u8 { @@ -2473,6 +2497,29 @@ pub fn spawnThread(stack: []align(os.page_size) u8, context: var, comptime start } }; + const stack_len = default_stack_size; + const stack_addr = posix.mmap(null, stack_len, posix.PROT_READ|posix.PROT_WRITE, + posix.MAP_PRIVATE|posix.MAP_ANONYMOUS|posix.MAP_GROWSDOWN, -1, 0); + if (stack_addr == posix.MAP_FAILED) return error.OutOfMemory; + errdefer _ = posix.munmap(stack_addr, stack_len); + + var stack_end: usize = stack_addr + stack_len; + var arg: usize = undefined; + if (@sizeOf(Context) != 0) { + stack_end -= @sizeOf(Context); + stack_end -= stack_end % @alignOf(Context); + assert(stack_end >= stack_addr); + const context_ptr = @alignCast(@alignOf(Context), @intToPtr(&Context, stack_end)); + *context_ptr = context; + arg = stack_end; + } + + stack_end -= @sizeOf(Thread); + stack_end -= stack_end % @alignOf(Thread); + assert(stack_end >= stack_addr); + const thread_ptr = @alignCast(@alignOf(Thread), @intToPtr(&Thread, stack_end)); + + if (builtin.os == builtin.Os.windows) { // use windows API directly @compileError("TODO support spawnThread for Windows"); @@ -2484,14 +2531,12 @@ pub fn spawnThread(stack: []align(os.page_size) u8, context: var, comptime start // align to page stack_end -= stack_end % os.page_size; + assert(c.pthread_attr_setstack(&attr, @intToPtr(&c_void, stack_addr), stack_len) == 0); - const stack_size = stack_end - @ptrToInt(stack.ptr); - const setstack_err = c.pthread_attr_setstack(&attr, @ptrCast(&c_void, stack.ptr), stack_size); - if (setstack_err != 0) { - return SpawnThreadError.StackTooSmall; // pthreads requires at least 16384 bytes - } + thread_ptr.data.stack_addr = stack_addr; + thread_ptr.data.stack_len = stack_len; - const err = c.pthread_create(&thread_ptr.pthread_handle, &attr, MainFuncs.posixThreadMain, @intToPtr(&c_void, arg)); + const err = c.pthread_create(&thread_ptr.data.handle, &attr, MainFuncs.posixThreadMain, @intToPtr(&c_void, arg)); switch (err) { 0 => return thread_ptr, posix.EAGAIN => return SpawnThreadError.SystemResources, diff --git a/std/os/test.zig b/std/os/test.zig index 37e5bf4bb8..56d6e8b309 100644 --- a/std/os/test.zig +++ b/std/os/test.zig @@ -44,24 +44,12 @@ test "access file" { } test "spawn threads" { - if (builtin.os == builtin.Os.windows) { - // TODO implement threads on windows - return; - } - - var direct_allocator = std.heap.DirectAllocator.init(); - defer direct_allocator.deinit(); - var shared_ctx: i32 = 1; - const thread1 = try std.os.spawnThreadAllocator(&direct_allocator.allocator, {}, start1); - const thread4 = try std.os.spawnThreadAllocator(&direct_allocator.allocator, &shared_ctx, start2); - - var stack1: [20 * 1024]u8 align(os.page_size) = undefined; - var stack2: [20 * 1024]u8 align(os.page_size) = undefined; - - const thread2 = try std.os.spawnThread(stack1[0..], &shared_ctx, start2); - const thread3 = try std.os.spawnThread(stack2[0..], &shared_ctx, start2); + const thread1 = try std.os.spawnThread({}, start1); + const thread2 = try std.os.spawnThread(&shared_ctx, start2); + const thread3 = try std.os.spawnThread(&shared_ctx, start2); + const thread4 = try std.os.spawnThread(&shared_ctx, start2); thread1.wait(); thread2.wait(); diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index d6ef7205e8..e13ed0f131 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -28,6 +28,9 @@ pub extern "kernel32" stdcallcc fn CreateProcessA(lpApplicationName: ?LPCSTR, lp pub extern "kernel32" stdcallcc fn CreateSymbolicLinkA(lpSymlinkFileName: LPCSTR, lpTargetFileName: LPCSTR, dwFlags: DWORD) BOOLEAN; + +pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ATTRIBUTES, dwStackSize: SIZE_T, lpStartAddress: LPTHREAD_START_ROUTINE, lpParameter: ?LPVOID, dwCreationFlags: DWORD, lpThreadId: ?LPDWORD) ?HANDLE; + pub extern "kernel32" stdcallcc fn DeleteFileA(lpFileName: LPCSTR) BOOL; pub extern "kernel32" stdcallcc fn ExitProcess(exit_code: UINT) noreturn; @@ -318,6 +321,9 @@ pub const HEAP_CREATE_ENABLE_EXECUTE = 0x00040000; pub const HEAP_GENERATE_EXCEPTIONS = 0x00000004; pub const HEAP_NO_SERIALIZE = 0x00000001; +pub const PTHREAD_START_ROUTINE = extern fn(LPVOID) DWORD; +pub const LPTHREAD_START_ROUTINE = PTHREAD_START_ROUTINE; + test "import" { _ = @import("util.zig"); } -- cgit v1.2.3 From b7095912c77900eab6ad667a6eeb1add18ac8071 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 29 Apr 2018 15:48:53 -0400 Subject: zig fmt: respect comments before statements --- std/mem.zig | 18 +++- std/zig/ast.zig | 4 +- std/zig/parser.zig | 291 ++++++++++++++++++++++++++++++----------------------- 3 files changed, 180 insertions(+), 133 deletions(-) (limited to 'std/mem.zig') diff --git a/std/mem.zig b/std/mem.zig index 0f66f549cc..d874f8a6c9 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -37,6 +37,20 @@ pub const Allocator = struct { return &slice[0]; } + // TODO once #733 is solved, this will replace create + fn construct(self: &Allocator, init: var) t: { + // TODO this is a workaround for type getting parsed as Error!&const T + const T = @typeOf(init).Child; + break :t Error!&T; + } { + const T = @typeOf(init).Child; + if (@sizeOf(T) == 0) return &{}; + const slice = try self.alloc(T, 1); + const ptr = &slice[0]; + *ptr = *init; + return ptr; + } + fn destroy(self: &Allocator, ptr: var) void { self.free(ptr[0..1]); } @@ -54,7 +68,7 @@ pub const Allocator = struct { const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory; const byte_slice = try self.allocFn(self, byte_count, alignment); assert(byte_slice.len == byte_count); - // This loop should get optimized out in ReleaseFast mode + // This loop gets optimized out in ReleaseFast mode for (byte_slice) |*byte| { *byte = undefined; } @@ -81,7 +95,7 @@ pub const Allocator = struct { const byte_slice = try self.reallocFn(self, old_byte_slice, byte_count, alignment); assert(byte_slice.len == byte_count); if (n > old_mem.len) { - // This loop should get optimized out in ReleaseFast mode + // This loop gets optimized out in ReleaseFast mode for (byte_slice[old_byte_slice.len..]) |*byte| { *byte = undefined; } diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 76977a979a..66c0455e8c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -6,6 +6,7 @@ const mem = std.mem; pub const Node = struct { id: Id, + comments: ?&LineComment, pub const Id = enum { // Top level @@ -139,7 +140,6 @@ pub const Node = struct { pub const VarDecl = struct { base: Node, - comments: ?&LineComment, visib_token: ?Token, name_token: Token, eq_token: Token, @@ -421,7 +421,6 @@ pub const Node = struct { pub const FnProto = struct { base: Node, - comments: ?&LineComment, visib_token: ?Token, fn_token: Token, name_token: ?Token, @@ -1732,7 +1731,6 @@ pub const Node = struct { pub const TestDecl = struct { base: Node, - comments: ?&LineComment, test_token: Token, name: &Node, body_node: &Node, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7f45cce28b..b5fba6a9b9 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -182,6 +182,11 @@ pub const Parser = struct { } }; + const AddCommentsCtx = struct { + node_ptr: &&ast.Node, + comments: ?&ast.Node.LineComment, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -221,6 +226,7 @@ pub const Parser = struct { Statement: &ast.Node.Block, ComptimeStatement: ComptimeStatementCtx, Semicolon: &&ast.Node, + AddComments: AddCommentsCtx, AsmOutputItems: &ArrayList(&ast.Node.AsmOutput), AsmOutputReturnOrType: &ast.Node.AsmOutput, @@ -345,24 +351,26 @@ pub const Parser = struct { Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; - const block = try self.createNode(arena, ast.Node.Block, - ast.Node.Block { - .base = undefined, - .label = null, - .lbrace = undefined, - .statements = ArrayList(&ast.Node).init(arena), - .rbrace = undefined, - } - ); - const test_node = try self.createAttachNode(arena, &root_node.decls, ast.Node.TestDecl, - ast.Node.TestDecl { - .base = undefined, + const block = try arena.construct(ast.Node.Block { + .base = ast.Node { + .id = ast.Node.Id.Block, + .comments = null, + }, + .label = null, + .lbrace = undefined, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + }); + const test_node = try arena.construct(ast.Node.TestDecl { + .base = ast.Node { + .id = ast.Node.Id.TestDecl, .comments = comments, - .test_token = token, - .name = undefined, - .body_node = &block.base, - } - ); + }, + .test_token = token, + .name = undefined, + .body_node = &block.base, + }); + try root_node.decls.append(&test_node.base); stack.append(State { .Block = block }) catch unreachable; try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -530,24 +538,25 @@ pub const Parser = struct { }, Token.Id.Keyword_fn, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc, Token.Id.Keyword_async => { - const fn_proto = try self.createAttachNode(arena, ctx.decls, ast.Node.FnProto, - ast.Node.FnProto { - .base = undefined, + const fn_proto = try arena.construct(ast.Node.FnProto { + .base = ast.Node { + .id = ast.Node.Id.FnProto, .comments = comments, - .visib_token = ctx.visib_token, - .name_token = null, - .fn_token = undefined, - .params = ArrayList(&ast.Node).init(arena), - .return_type = undefined, - .var_args_token = null, - .extern_export_inline_token = ctx.extern_export_inline_token, - .cc_token = null, - .async_attr = null, - .body_node = null, - .lib_name = ctx.lib_name, - .align_expr = null, - } - ); + }, + .visib_token = ctx.visib_token, + .name_token = null, + .fn_token = undefined, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_export_inline_token = ctx.extern_export_inline_token, + .cc_token = null, + .async_attr = null, + .body_node = null, + .lib_name = ctx.lib_name, + .align_expr = null, + }); + try ctx.decls.append(&fn_proto.base); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); @@ -789,24 +798,25 @@ pub const Parser = struct { State.VarDecl => |ctx| { - const var_decl = try self.createAttachNode(arena, ctx.list, ast.Node.VarDecl, - ast.Node.VarDecl { - .base = undefined, + const var_decl = try arena.construct(ast.Node.VarDecl { + .base = ast.Node { + .id = ast.Node.Id.VarDecl, .comments = ctx.comments, - .visib_token = ctx.visib_token, - .mut_token = ctx.mut_token, - .comptime_token = ctx.comptime_token, - .extern_export_token = ctx.extern_export_token, - .type_node = null, - .align_node = null, - .init_node = null, - .lib_name = ctx.lib_name, - // initialized later - .name_token = undefined, - .eq_token = undefined, - .semicolon_token = undefined, - } - ); + }, + .visib_token = ctx.visib_token, + .mut_token = ctx.mut_token, + .comptime_token = ctx.comptime_token, + .extern_export_token = ctx.extern_export_token, + .type_node = null, + .align_node = null, + .init_node = null, + .lib_name = ctx.lib_name, + // initialized later + .name_token = undefined, + .eq_token = undefined, + .semicolon_token = undefined, + }); + try ctx.list.append(&var_decl.base); stack.append(State { .VarDeclAlign = var_decl }) catch unreachable; try stack.append(State { .TypeExprBegin = OptionalCtx { .RequiredNull = &var_decl.type_node} }); @@ -1218,19 +1228,23 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_defer, Token.Id.Keyword_errdefer => { - const node = try self.createAttachNode(arena, &block.statements, ast.Node.Defer, - ast.Node.Defer { - .base = undefined, - .defer_token = token, - .kind = switch (token.id) { - Token.Id.Keyword_defer => ast.Node.Defer.Kind.Unconditional, - Token.Id.Keyword_errdefer => ast.Node.Defer.Kind.Error, - else => unreachable, - }, - .expr = undefined, - } - ); - stack.append(State { .Semicolon = &&node.base }) catch unreachable; + const node = try arena.construct(ast.Node.Defer { + .base = ast.Node { + .id = ast.Node.Id.Defer, + .comments = comments, + }, + .defer_token = token, + .kind = switch (token.id) { + Token.Id.Keyword_defer => ast.Node.Defer.Kind.Unconditional, + Token.Id.Keyword_errdefer => ast.Node.Defer.Kind.Error, + else => unreachable, + }, + .expr = undefined, + }); + const node_ptr = try block.statements.addOne(); + *node_ptr = &node.base; + + stack.append(State { .Semicolon = node_ptr }) catch unreachable; try stack.append(State { .AssignmentExpressionBegin = OptionalCtx{ .Required = &node.expr } }); continue; }, @@ -1249,9 +1263,13 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - const statememt = try block.statements.addOne(); - stack.append(State { .Semicolon = statememt }) catch unreachable; - try stack.append(State { .AssignmentExpressionBegin = OptionalCtx{ .Required = statememt } }); + const statement = try block.statements.addOne(); + stack.append(State { .Semicolon = statement }) catch unreachable; + try stack.append(State { .AddComments = AddCommentsCtx { + .node_ptr = statement, + .comments = comments, + }}); + try stack.append(State { .AssignmentExpressionBegin = OptionalCtx{ .Required = statement } }); continue; } } @@ -1293,6 +1311,12 @@ pub const Parser = struct { continue; }, + State.AddComments => |add_comments_ctx| { + const node = *add_comments_ctx.node_ptr; + node.comments = add_comments_ctx.comments; + continue; + }, + State.AsmOutputItems => |items| { const lbracket = self.getNextToken(); @@ -1576,24 +1600,25 @@ pub const Parser = struct { State.ExternType => |ctx| { if (self.eatToken(Token.Id.Keyword_fn)) |fn_token| { - const fn_proto = try self.createToCtxNode(arena, ctx.opt_ctx, ast.Node.FnProto, - ast.Node.FnProto { - .base = undefined, + const fn_proto = try arena.construct(ast.Node.FnProto { + .base = ast.Node { + .id = ast.Node.Id.FnProto, .comments = ctx.comments, - .visib_token = null, - .name_token = null, - .fn_token = fn_token, - .params = ArrayList(&ast.Node).init(arena), - .return_type = undefined, - .var_args_token = null, - .extern_export_inline_token = ctx.extern_token, - .cc_token = null, - .async_attr = null, - .body_node = null, - .lib_name = null, - .align_expr = null, - } - ); + }, + .visib_token = null, + .name_token = null, + .fn_token = fn_token, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_export_inline_token = ctx.extern_token, + .cc_token = null, + .async_attr = null, + .body_node = null, + .lib_name = null, + .align_expr = null, + }); + ctx.opt_ctx.store(&fn_proto.base); stack.append(State { .FnProto = fn_proto }) catch unreachable; continue; } @@ -2546,46 +2571,48 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - const fn_proto = try self.createToCtxNode(arena, opt_ctx, ast.Node.FnProto, - ast.Node.FnProto { - .base = undefined, + const fn_proto = try arena.construct(ast.Node.FnProto { + .base = ast.Node { + .id = ast.Node.Id.FnProto, .comments = null, - .visib_token = null, - .name_token = null, - .fn_token = token, - .params = ArrayList(&ast.Node).init(arena), - .return_type = undefined, - .var_args_token = null, - .extern_export_inline_token = null, - .cc_token = null, - .async_attr = null, - .body_node = null, - .lib_name = null, - .align_expr = null, - } - ); + }, + .visib_token = null, + .name_token = null, + .fn_token = token, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_export_inline_token = null, + .cc_token = null, + .async_attr = null, + .body_node = null, + .lib_name = null, + .align_expr = null, + }); + opt_ctx.store(&fn_proto.base); stack.append(State { .FnProto = fn_proto }) catch unreachable; continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - const fn_proto = try self.createToCtxNode(arena, opt_ctx, ast.Node.FnProto, - ast.Node.FnProto { - .base = undefined, + const fn_proto = try arena.construct(ast.Node.FnProto { + .base = ast.Node { + .id = ast.Node.Id.FnProto, .comments = null, - .visib_token = null, - .name_token = null, - .fn_token = undefined, - .params = ArrayList(&ast.Node).init(arena), - .return_type = undefined, - .var_args_token = null, - .extern_export_inline_token = null, - .cc_token = token, - .async_attr = null, - .body_node = null, - .lib_name = null, - .align_expr = null, - } - ); + }, + .visib_token = null, + .name_token = null, + .fn_token = undefined, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_export_inline_token = null, + .cc_token = token, + .async_attr = null, + .body_node = null, + .lib_name = null, + .align_expr = null, + }); + opt_ctx.store(&fn_proto.base); stack.append(State { .FnProto = fn_proto }) catch unreachable; try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -2747,13 +2774,13 @@ pub const Parser = struct { if (result) |comment_node| { break :blk comment_node; } else { - const comment_node = try arena.create(ast.Node.LineComment); - *comment_node = ast.Node.LineComment { + const comment_node = try arena.construct(ast.Node.LineComment { .base = ast.Node { .id = ast.Node.Id.LineComment, + .comments = null, }, .lines = ArrayList(Token).init(arena), - }; + }); result = comment_node; break :blk comment_node; } @@ -3096,7 +3123,7 @@ pub const Parser = struct { *node = *init_to; node.base = blk: { const id = ast.Node.typeToId(T); - break :blk ast.Node {.id = id}; + break :blk ast.Node {.id = id, .comments = null}; }; return node; @@ -3269,7 +3296,7 @@ pub const Parser = struct { switch (decl.id) { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl); - try self.renderComments(stream, fn_proto, indent); + try self.renderComments(stream, &fn_proto.base, indent); if (fn_proto.body_node) |body_node| { stack.append(RenderState { .Expression = body_node}) catch unreachable; @@ -3295,7 +3322,7 @@ pub const Parser = struct { }, ast.Node.Id.TestDecl => { const test_decl = @fieldParentPtr(ast.Node.TestDecl, "base", decl); - try self.renderComments(stream, test_decl, indent); + try self.renderComments(stream, &test_decl.base, indent); try stream.print("test "); try stack.append(RenderState { .Expression = test_decl.body_node }); try stack.append(RenderState { .Text = " " }); @@ -3338,7 +3365,6 @@ pub const Parser = struct { }, RenderState.FieldInitializer => |field_init| { - //TODO try self.renderComments(stream, field_init, indent); try stream.print(".{}", self.tokenizer.getTokenSlice(field_init.name_token)); try stream.print(" = "); try stack.append(RenderState { .Expression = field_init.expr }); @@ -3385,7 +3411,6 @@ pub const Parser = struct { RenderState.ParamDecl => |base| { const param_decl = @fieldParentPtr(ast.Node.ParamDecl, "base", base); - // TODO try self.renderComments(stream, param_decl, indent); if (param_decl.comptime_token) |comptime_token| { try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token)); } @@ -4328,10 +4353,10 @@ pub const Parser = struct { ast.Node.Id.ParamDecl => unreachable, }, RenderState.Statement => |base| { + try self.renderComments(stream, base, indent); switch (base.id) { ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", base); - try self.renderComments(stream, var_decl, indent); try stack.append(RenderState { .VarDecl = var_decl}); }, else => { @@ -4348,7 +4373,7 @@ pub const Parser = struct { } } - fn renderComments(self: &Parser, stream: var, node: var, indent: usize) !void { + fn renderComments(self: &Parser, stream: var, node: &ast.Node, indent: usize) !void { const comment = node.comments ?? return; for (comment.lines.toSliceConst()) |line_token| { try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token)); @@ -4430,6 +4455,16 @@ fn testCanonical(source: []const u8) !void { } } +test "zig fmt: preserve comments before statements" { + try testCanonical( + \\test "std" { + \\ // statement comment + \\ _ = @import("foo/bar.zig"); + \\} + \\ + ); +} + test "zig fmt: preserve top level comments" { try testCanonical( \\// top level comment -- cgit v1.2.3