Merge pull request #9258 from ziglang/shared-cache-locking

Shared Cache Locking

4 files changed, 356 insertions, 81 deletions

diff --git a/lib/std/fs.zig b/lib/std/fs.zig
index 9a39bc0425..4ecdc3803b 100644
--- a/lib/std/fs.zig
+++ b/lib/std/fs.zig
@@ -883,24 +883,39 @@ pub const Dir = struct {
     /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
     pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
         const w = os.windows;
-        return @as(File, .{
-            .handle = try os.windows.OpenFile(sub_path_w, .{
+        const file: File = .{
+            .handle = try w.OpenFile(sub_path_w, .{
                 .dir = self.fd,
                 .access_mask = w.SYNCHRONIZE |
                     (if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
                     (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
-                .share_access = switch (flags.lock) {
-                    .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
-                    .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
-                    .Exclusive => w.FILE_SHARE_DELETE,
-                },
-                .share_access_nonblocking = flags.lock_nonblocking,
                 .creation = w.FILE_OPEN,
                 .io_mode = flags.intended_io_mode,
             }),
             .capable_io_mode = std.io.default_mode,
             .intended_io_mode = flags.intended_io_mode,
-        });
+        };
+        var io: w.IO_STATUS_BLOCK = undefined;
+        const range_off: w.LARGE_INTEGER = 0;
+        const range_len: w.LARGE_INTEGER = 1;
+        const exclusive = switch (flags.lock) {
+            .None => return file,
+            .Shared => false,
+            .Exclusive => true,
+        };
+        try w.LockFile(
+            file.handle,
+            null,
+            null,
+            null,
+            &io,
+            &range_off,
+            &range_len,
+            null,
+            @boolToInt(flags.lock_nonblocking),
+            @boolToInt(exclusive),
+        );
+        return file;
     }
 
     /// Creates, opens, or overwrites a file with write access.
@@ -1019,16 +1034,10 @@ pub const Dir = struct {
     pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
         const w = os.windows;
         const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
-        return @as(File, .{
+        const file: File = .{
             .handle = try os.windows.OpenFile(sub_path_w, .{
                 .dir = self.fd,
                 .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
-                .share_access = switch (flags.lock) {
-                    .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
-                    .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
-                    .Exclusive => w.FILE_SHARE_DELETE,
-                },
-                .share_access_nonblocking = flags.lock_nonblocking,
                 .creation = if (flags.exclusive)
                     @as(u32, w.FILE_CREATE)
                 else if (flags.truncate)
@@ -1039,7 +1048,28 @@ pub const Dir = struct {
             }),
             .capable_io_mode = std.io.default_mode,
             .intended_io_mode = flags.intended_io_mode,
-        });
+        };
+        var io: w.IO_STATUS_BLOCK = undefined;
+        const range_off: w.LARGE_INTEGER = 0;
+        const range_len: w.LARGE_INTEGER = 1;
+        const exclusive = switch (flags.lock) {
+            .None => return file,
+            .Shared => false,
+            .Exclusive => true,
+        };
+        try w.LockFile(
+            file.handle,
+            null,
+            null,
+            null,
+            &io,
+            &range_off,
+            &range_len,
+            null,
+            @boolToInt(flags.lock_nonblocking),
+            @boolToInt(exclusive),
+        );
+        return file;
     }
 
     pub const openRead = @compileError("deprecated in favor of openFile");
diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig
index fecf2310dd..a4300e7376 100644
--- a/lib/std/fs/file.zig
+++ b/lib/std/fs/file.zig
@@ -74,17 +74,28 @@ pub const File = struct {
         read: bool = true,
         write: bool = false,
 
-        /// Open the file with a lock to prevent other processes from accessing it at the
-        /// same time. An exclusive lock will prevent other processes from acquiring a lock.
-        /// A shared lock will prevent other processes from acquiring a exclusive lock, but
-        /// doesn't prevent other process from getting their own shared locks.
+        /// Open the file with an advisory lock to coordinate with other processes
+        /// accessing it at the same time. An exclusive lock will prevent other
+        /// processes from acquiring a lock. A shared lock will prevent other
+        /// processes from acquiring a exclusive lock, but does not prevent
+        /// other process from getting their own shared locks.
         ///
-        /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
+        /// The lock is advisory, except on Linux in very specific cirsumstances[1].
         /// This means that a process that does not respect the locking API can still get access
         /// to the file, despite the lock.
         ///
-        /// Windows' file locks are mandatory, and any process attempting to access the file will
-        /// receive an error.
+        /// On these operating systems, the lock is acquired atomically with
+        /// opening the file:
+        /// * Darwin
+        /// * DragonFlyBSD
+        /// * FreeBSD
+        /// * Haiku
+        /// * NetBSD
+        /// * OpenBSD
+        /// On these operating systems, the lock is acquired via a separate syscall
+        /// after opening the file:
+        /// * Linux
+        /// * Windows
         ///
         /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
         lock: Lock = .None,
@@ -120,17 +131,28 @@ pub const File = struct {
         /// `error.PathAlreadyExists` to be returned.
         exclusive: bool = false,
 
-        /// Open the file with a lock to prevent other processes from accessing it at the
-        /// same time. An exclusive lock will prevent other processes from acquiring a lock.
-        /// A shared lock will prevent other processes from acquiring a exclusive lock, but
-        /// doesn't prevent other process from getting their own shared locks.
+        /// Open the file with an advisory lock to coordinate with other processes
+        /// accessing it at the same time. An exclusive lock will prevent other
+        /// processes from acquiring a lock. A shared lock will prevent other
+        /// processes from acquiring a exclusive lock, but does not prevent
+        /// other process from getting their own shared locks.
         ///
-        /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
+        /// The lock is advisory, except on Linux in very specific cirsumstances[1].
         /// This means that a process that does not respect the locking API can still get access
         /// to the file, despite the lock.
         ///
-        /// Windows's file locks are mandatory, and any process attempting to access the file will
-        /// receive an error.
+        /// On these operating systems, the lock is acquired atomically with
+        /// opening the file:
+        /// * Darwin
+        /// * DragonFlyBSD
+        /// * FreeBSD
+        /// * Haiku
+        /// * NetBSD
+        /// * OpenBSD
+        /// On these operating systems, the lock is acquired via a separate syscall
+        /// after opening the file:
+        /// * Linux
+        /// * Windows
         ///
         /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
         lock: Lock = .None,
@@ -829,4 +851,165 @@ pub const File = struct {
     pub fn seekableStream(file: File) SeekableStream {
         return .{ .context = file };
     }
+
+    const range_off: windows.LARGE_INTEGER = 0;
+    const range_len: windows.LARGE_INTEGER = 1;
+
+    pub const LockError = error{
+        SystemResources,
+    } || os.UnexpectedError;
+
+    /// Blocks when an incompatible lock is held by another process.
+    /// A process may hold only one type of lock (shared or exclusive) on
+    /// a file. When a process terminates in any way, the lock is released.
+    ///
+    /// Assumes the file is unlocked.
+    ///
+    /// TODO: integrate with async I/O
+    pub fn lock(file: File, l: Lock) LockError!void {
+        if (is_windows) {
+            var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+            const exclusive = switch (l) {
+                .None => return,
+                .Shared => false,
+                .Exclusive => true,
+            };
+            return windows.LockFile(
+                file.handle,
+                null,
+                null,
+                null,
+                &io_status_block,
+                &range_off,
+                &range_len,
+                null,
+                windows.FALSE, // non-blocking=false
+                @boolToInt(exclusive),
+            ) catch |err| switch (err) {
+                error.WouldBlock => unreachable, // non-blocking=false
+                else => |e| return e,
+            };
+        } else {
+            return os.flock(file.handle, switch (l) {
+                .None => os.LOCK_UN,
+                .Shared => os.LOCK_SH,
+                .Exclusive => os.LOCK_EX,
+            }) catch |err| switch (err) {
+                error.WouldBlock => unreachable, // non-blocking=false
+                else => |e| return e,
+            };
+        }
+    }
+
+    /// Assumes the file is locked.
+    pub fn unlock(file: File) void {
+        if (is_windows) {
+            var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+            return windows.UnlockFile(
+                file.handle,
+                &io_status_block,
+                &range_off,
+                &range_len,
+                null,
+            ) catch |err| switch (err) {
+                error.RangeNotLocked => unreachable, // Function assumes unlocked.
+                error.Unexpected => unreachable, // Resource deallocation must succeed.
+            };
+        } else {
+            return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
+                error.WouldBlock => unreachable, // unlocking can't block
+                error.SystemResources => unreachable, // We are deallocating resources.
+                error.Unexpected => unreachable, // Resource deallocation must succeed.
+            };
+        }
+    }
+
+    /// Attempts to obtain a lock, returning `true` if the lock is
+    /// obtained, and `false` if there was an existing incompatible lock held.
+    /// A process may hold only one type of lock (shared or exclusive) on
+    /// a file. When a process terminates in any way, the lock is released.
+    ///
+    /// Assumes the file is unlocked.
+    ///
+    /// TODO: integrate with async I/O
+    pub fn tryLock(file: File, l: Lock) LockError!bool {
+        if (is_windows) {
+            var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+            const exclusive = switch (l) {
+                .None => return,
+                .Shared => false,
+                .Exclusive => true,
+            };
+            windows.LockFile(
+                file.handle,
+                null,
+                null,
+                null,
+                &io_status_block,
+                &range_off,
+                &range_len,
+                null,
+                windows.TRUE, // non-blocking=true
+                @boolToInt(exclusive),
+            ) catch |err| switch (err) {
+                error.WouldBlock => return false,
+                else => |e| return e,
+            };
+        } else {
+            os.flock(file.handle, switch (l) {
+                .None => os.LOCK_UN,
+                .Shared => os.LOCK_SH | os.LOCK_NB,
+                .Exclusive => os.LOCK_EX | os.LOCK_NB,
+            }) catch |err| switch (err) {
+                error.WouldBlock => return false,
+                else => |e| return e,
+            };
+        }
+        return true;
+    }
+
+    /// Assumes the file is already locked in exclusive mode.
+    /// Atomically modifies the lock to be in shared mode, without releasing it.
+    ///
+    /// TODO: integrate with async I/O
+    pub fn downgradeLock(file: File) LockError!void {
+        if (is_windows) {
+            // On Windows it works like a semaphore + exclusivity flag. To implement this
+            // function, we first obtain another lock in shared mode. This changes the
+            // exclusivity flag, but increments the semaphore to 2. So we follow up with
+            // an NtUnlockFile which decrements the semaphore but does not modify the
+            // exclusivity flag.
+            var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+            windows.LockFile(
+                file.handle,
+                null,
+                null,
+                null,
+                &io_status_block,
+                &range_off,
+                &range_len,
+                null,
+                windows.TRUE, // non-blocking=true
+                windows.FALSE, // exclusive=false
+            ) catch |err| switch (err) {
+                error.WouldBlock => unreachable, // File was not locked in exclusive mode.
+                else => |e| return e,
+            };
+            return windows.UnlockFile(
+                file.handle,
+                &io_status_block,
+                &range_off,
+                &range_len,
+                null,
+            ) catch |err| switch (err) {
+                error.RangeNotLocked => unreachable, // File was not locked.
+                error.Unexpected => unreachable, // Resource deallocation must succeed.
+            };
+        } else {
+            return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
+                error.WouldBlock => unreachable, // File was not locked in exclusive mode.
+                else => |e| return e,
+            };
+        }
+    }
 };
diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 1fade2a462..cbd4ce065b 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -49,7 +49,6 @@ pub const OpenFileOptions = struct {
     dir: ?HANDLE = null,
     sa: ?*SECURITY_ATTRIBUTES = null,
     share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
-    share_access_nonblocking: bool = false,
     creation: ULONG,
     io_mode: std.io.ModeOverride,
     /// If true, tries to open path as a directory.
@@ -60,8 +59,6 @@ pub const OpenFileOptions = struct {
     follow_symlinks: bool = true,
 };
 
-/// TODO when share_access_nonblocking is false, this implementation uses
-/// untinterruptible sleep() to block. This is not the final iteration of the API.
 pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
     if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
         return error.IsDir;
@@ -94,53 +91,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
     // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
     const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
 
-    var delay: usize = 1;
-    while (true) {
-        const rc = ntdll.NtCreateFile(
-            &result,
-            options.access_mask,
-            &attr,
-            &io,
-            null,
-            FILE_ATTRIBUTE_NORMAL,
-            options.share_access,
-            options.creation,
-            flags,
-            null,
-            0,
-        );
-        switch (rc) {
-            .SUCCESS => {
-                if (std.io.is_async and options.io_mode == .evented) {
-                    _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
-                }
-                return result;
-            },
-            .OBJECT_NAME_INVALID => unreachable,
-            .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
-            .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
-            .NO_MEDIA_IN_DEVICE => return error.NoDevice,
-            .INVALID_PARAMETER => unreachable,
-            .SHARING_VIOLATION => {
-                if (options.share_access_nonblocking) {
-                    return error.WouldBlock;
-                }
-                // TODO sleep in a way that is interruptable
-                // TODO integrate with async I/O
-                std.time.sleep(delay);
-                if (delay < 1 * std.time.ns_per_s) {
-                    delay *= 2;
-                }
-                continue;
-            },
-            .ACCESS_DENIED => return error.AccessDenied,
-            .PIPE_BUSY => return error.PipeBusy,
-            .OBJECT_PATH_SYNTAX_BAD => unreachable,
-            .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
-            .FILE_IS_A_DIRECTORY => return error.IsDir,
-            .NOT_A_DIRECTORY => return error.NotDir,
-            else => return unexpectedStatus(rc),
-        }
+    const rc = ntdll.NtCreateFile(
+        &result,
+        options.access_mask,
+        &attr,
+        &io,
+        null,
+        FILE_ATTRIBUTE_NORMAL,
+        options.share_access,
+        options.creation,
+        flags,
+        null,
+        0,
+    );
+    switch (rc) {
+        .SUCCESS => {
+            if (std.io.is_async and options.io_mode == .evented) {
+                _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
+            }
+            return result;
+        },
+        .OBJECT_NAME_INVALID => unreachable,
+        .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
+        .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
+        .NO_MEDIA_IN_DEVICE => return error.NoDevice,
+        .INVALID_PARAMETER => unreachable,
+        .SHARING_VIOLATION => return error.AccessDenied,
+        .ACCESS_DENIED => return error.AccessDenied,
+        .PIPE_BUSY => return error.PipeBusy,
+        .OBJECT_PATH_SYNTAX_BAD => unreachable,
+        .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
+        .FILE_IS_A_DIRECTORY => return error.IsDir,
+        .NOT_A_DIRECTORY => return error.NotDir,
+        else => return unexpectedStatus(rc),
     }
 }
 
@@ -1679,6 +1662,64 @@ pub fn SetFileTime(
     }
 }
 
+pub const LockFileError = error{
+    SystemResources,
+    WouldBlock,
+} || std.os.UnexpectedError;
+
+pub fn LockFile(
+    FileHandle: HANDLE,
+    Event: ?HANDLE,
+    ApcRoutine: ?*IO_APC_ROUTINE,
+    ApcContext: ?*c_void,
+    IoStatusBlock: *IO_STATUS_BLOCK,
+    ByteOffset: *const LARGE_INTEGER,
+    Length: *const LARGE_INTEGER,
+    Key: ?*ULONG,
+    FailImmediately: BOOLEAN,
+    ExclusiveLock: BOOLEAN,
+) !void {
+    const rc = ntdll.NtLockFile(
+        FileHandle,
+        Event,
+        ApcRoutine,
+        ApcContext,
+        IoStatusBlock,
+        ByteOffset,
+        Length,
+        Key,
+        FailImmediately,
+        ExclusiveLock,
+    );
+    switch (rc) {
+        .SUCCESS => return,
+        .INSUFFICIENT_RESOURCES => return error.SystemResources,
+        .LOCK_NOT_GRANTED => return error.WouldBlock,
+        .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
+        else => return unexpectedStatus(rc),
+    }
+}
+
+pub const UnlockFileError = error{
+    RangeNotLocked,
+} || std.os.UnexpectedError;
+
+pub fn UnlockFile(
+    FileHandle: HANDLE,
+    IoStatusBlock: *IO_STATUS_BLOCK,
+    ByteOffset: *const LARGE_INTEGER,
+    Length: *const LARGE_INTEGER,
+    Key: ?*ULONG,
+) !void {
+    const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key);
+    switch (rc) {
+        .SUCCESS => return,
+        .RANGE_NOT_LOCKED => return error.RangeNotLocked,
+        .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
+        else => return unexpectedStatus(rc),
+    }
+}
+
 pub fn teb() *TEB {
     return switch (builtin.target.cpu.arch) {
         .i386 => asm volatile (
diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig
index aa7e382ee0..ddc08e4bb2 100644
--- a/lib/std/os/windows/ntdll.zig
+++ b/lib/std/os/windows/ntdll.zig
@@ -139,3 +139,24 @@ pub extern "NtDll" fn RtlWaitOnAddress(
     AddressSize: SIZE_T,
     Timeout: ?*const LARGE_INTEGER,
 ) callconv(WINAPI) NTSTATUS;
+
+pub extern "NtDll" fn NtLockFile(
+    FileHandle: HANDLE,
+    Event: ?HANDLE,
+    ApcRoutine: ?*IO_APC_ROUTINE,
+    ApcContext: ?*c_void,
+    IoStatusBlock: *IO_STATUS_BLOCK,
+    ByteOffset: *const LARGE_INTEGER,
+    Length: *const LARGE_INTEGER,
+    Key: ?*ULONG,
+    FailImmediately: BOOLEAN,
+    ExclusiveLock: BOOLEAN,
+) callconv(WINAPI) NTSTATUS;
+
+pub extern "NtDll" fn NtUnlockFile(
+    FileHandle: HANDLE,
+    IoStatusBlock: *IO_STATUS_BLOCK,
+    ByteOffset: *const LARGE_INTEGER,
+    Length: *const LARGE_INTEGER,
+    Key: ?*ULONG,
+) callconv(WINAPI) NTSTATUS;