std: rework TTY detection and printing

This commit sketches an idea for how to deal with detection of file
streams as being terminals.

When a File stream is a terminal, writes through the stream should have
their escapes stripped unless the programmer explicitly enables terminal
escapes. Furthermore, the programmer needs a convenient API for
intentionally outputting escapes into the stream. In particular it
should be possible to set colors that are silently discarded when the
stream is not a terminal.

This commit makes `Io.File.Writer` track the terminal mode in the
already-existing `mode` field, making it the appropriate place to
implement escape stripping.

`Io.lockStderrWriter` returns a `*Io.File.Writer` with terminal
detection already done by default. This is a higher-level application
layer stream for writing to stderr.

Meanwhile, `std.debug.lockStderrWriter` also returns a `*Io.File.Writer`
but a lower-level one that is hard-coded to use a static single-threaded
`std.Io.Threaded` instance. This is the same instance that is used for
collecting debug information and iterating the unwind info.

3 files changed, 268 insertions, 157 deletions

diff --git a/lib/std/Io/File/Writer.zig b/lib/std/Io/File/Writer.zig
index ec58824c06..0e995908c4 100644
--- a/lib/std/Io/File/Writer.zig
+++ b/lib/std/Io/File/Writer.zig
@@ -1,4 +1,6 @@
 const Writer = @This();
+const builtin = @import("builtin");
+const is_windows = builtin.os.tag == .windows;
 
 const std = @import("../../std.zig");
 const Io = std.Io;
@@ -16,7 +18,144 @@ write_file_err: ?WriteFileError = null,
 seek_err: ?SeekError = null,
 interface: Io.Writer,
 
-pub const Mode = File.Reader.Mode;
+pub const Mode = union(enum) {
+    /// Uses `Io.VTable.fileWriteFileStreaming` if possible. Not a terminal.
+    /// `setColor` does nothing.
+    streaming,
+    /// Uses `Io.VTable.fileWriteFilePositional` if possible. Not a terminal.
+    /// `setColor` does nothing.
+    positional,
+    /// Avoids `Io.VTable.fileWriteFileStreaming`. Not a terminal. `setColor`
+    /// does nothing.
+    streaming_simple,
+    /// Avoids `Io.VTable.fileWriteFilePositional`. Not a terminal. `setColor`
+    /// does nothing.
+    positional_simple,
+    /// It's a terminal. Writes are escaped so as to strip escape sequences.
+    /// Color is enabled.
+    terminal_escaped,
+    /// It's a terminal. Colors are enabled via calling
+    /// SetConsoleTextAttribute. Writes are not escaped.
+    terminal_winapi: TerminalWinapi,
+    /// Indicates writing cannot continue because of a seek failure.
+    failure,
+
+    pub fn toStreaming(m: @This()) @This() {
+        return switch (m) {
+            .positional, .streaming => .streaming,
+            .positional_simple, .streaming_simple => .streaming_simple,
+            inline else => |_, x| x,
+        };
+    }
+
+    pub fn toSimple(m: @This()) @This() {
+        return switch (m) {
+            .positional, .positional_simple => .positional_simple,
+            .streaming, .streaming_simple => .streaming_simple,
+            inline else => |x| x,
+        };
+    }
+
+    pub fn toUnescaped(m: @This()) @This() {
+        return switch (m) {
+            .terminal_escaped => .streaming_simple,
+            inline else => |x| x,
+        };
+    }
+
+    pub const TerminalWinapi = if (!is_windows) noreturn else struct {
+        handle: File.Handle,
+        reset_attributes: u16,
+    };
+
+    /// Detect suitable TTY configuration options for the given file (commonly
+    /// stdout/stderr).
+    ///
+    /// Will attempt to enable ANSI escape code support if necessary/possible.
+    pub fn detect(io: Io, file: File, want_color: bool, fallback: Mode) Io.Cancelable!Mode {
+        if (!want_color) return if (try file.isTty(io)) .terminal_escaped else fallback;
+
+        if (file.enableAnsiEscapeCodes(io)) |_| {
+            return .terminal_escaped;
+        } else |err| switch (err) {
+            error.Canceled => return error.Canceled,
+            error.NotTerminalDevice, error.Unexpected => {},
+        }
+
+        if (is_windows and file.isTty(io)) {
+            const windows = std.os.windows;
+            var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
+            if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.FALSE) {
+                return .{ .terminal_winapi = .{
+                    .handle = file.handle,
+                    .reset_attributes = info.wAttributes,
+                } };
+            }
+            return .terminal_escaped;
+        }
+
+        return fallback;
+    }
+
+    pub const SetColorError = std.os.windows.SetConsoleTextAttributeError || Io.Writer.Error;
+
+    pub fn setColor(mode: Mode, io_w: *Io.Writer, color: Color) Mode.SetColorError!void {
+        switch (mode) {
+            .streaming, .positional, .streaming_simple, .positional_simple, .failure => return,
+            .terminal_escaped => {
+                const color_string = switch (color) {
+                    .black => "\x1b[30m",
+                    .red => "\x1b[31m",
+                    .green => "\x1b[32m",
+                    .yellow => "\x1b[33m",
+                    .blue => "\x1b[34m",
+                    .magenta => "\x1b[35m",
+                    .cyan => "\x1b[36m",
+                    .white => "\x1b[37m",
+                    .bright_black => "\x1b[90m",
+                    .bright_red => "\x1b[91m",
+                    .bright_green => "\x1b[92m",
+                    .bright_yellow => "\x1b[93m",
+                    .bright_blue => "\x1b[94m",
+                    .bright_magenta => "\x1b[95m",
+                    .bright_cyan => "\x1b[96m",
+                    .bright_white => "\x1b[97m",
+                    .bold => "\x1b[1m",
+                    .dim => "\x1b[2m",
+                    .reset => "\x1b[0m",
+                };
+                try io_w.writeAll(color_string);
+            },
+            .terminal_winapi => |ctx| {
+                const windows = std.os.windows;
+                const attributes: windows.WORD = switch (color) {
+                    .black => 0,
+                    .red => windows.FOREGROUND_RED,
+                    .green => windows.FOREGROUND_GREEN,
+                    .yellow => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN,
+                    .blue => windows.FOREGROUND_BLUE,
+                    .magenta => windows.FOREGROUND_RED | windows.FOREGROUND_BLUE,
+                    .cyan => windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE,
+                    .white => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE,
+                    .bright_black => windows.FOREGROUND_INTENSITY,
+                    .bright_red => windows.FOREGROUND_RED | windows.FOREGROUND_INTENSITY,
+                    .bright_green => windows.FOREGROUND_GREEN | windows.FOREGROUND_INTENSITY,
+                    .bright_yellow => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_INTENSITY,
+                    .bright_blue => windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
+                    .bright_magenta => windows.FOREGROUND_RED | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
+                    .bright_cyan => windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
+                    .bright_white, .bold => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
+                    // "dim" is not supported using basic character attributes, but let's still make it do *something*.
+                    // This matches the old behavior of TTY.Color before the bright variants were added.
+                    .dim => windows.FOREGROUND_INTENSITY,
+                    .reset => ctx.reset_attributes,
+                };
+                try io_w.flush();
+                try windows.SetConsoleTextAttribute(ctx.handle, attributes);
+            },
+        }
+    }
+};
 
 pub const Error = error{
     DiskQuota,
@@ -74,6 +213,16 @@ pub fn initStreaming(file: File, io: Io, buffer: []u8) Writer {
     };
 }
 
+/// Detects if `file` is terminal and sets the mode accordingly.
+pub fn initDetect(file: File, io: Io, buffer: []u8) Io.Cancelable!Writer {
+    return .{
+        .io = io,
+        .file = file,
+        .interface = initInterface(buffer),
+        .mode = try .detect(io, file, true, .positional),
+    };
+}
+
 pub fn initInterface(buffer: []u8) Io.Writer {
     return .{
         .vtable = &.{
@@ -99,8 +248,9 @@ pub fn moveToReader(w: *Writer) File.Reader {
 pub fn drain(io_w: *Io.Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize {
     const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w));
     switch (w.mode) {
-        .positional, .positional_reading => return drainPositional(w, data, splat),
-        .streaming, .streaming_reading => return drainStreaming(w, data, splat),
+        .positional, .positional_simple => return drainPositional(w, data, splat),
+        .streaming, .streaming_simple, .terminal_winapi => return drainStreaming(w, data, splat),
+        .terminal_escaped => return drainEscaping(w, data, splat),
         .failure => return error.WriteFailed,
     }
 }
@@ -141,13 +291,38 @@ fn drainStreaming(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.
     return w.interface.consume(n);
 }
 
+fn findTerminalEscape(buffer: []const u8) ?usize {
+    return std.mem.findScalar(u8, buffer, 0x1b);
+}
+
+fn drainEscaping(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize {
+    const io = w.io;
+    const header = w.interface.buffered();
+    if (findTerminalEscape(header)) |i| {
+        _ = i;
+        @panic("TODO strip terminal escape sequence");
+    }
+    for (data) |d| {
+        if (findTerminalEscape(d)) |i| {
+            _ = i;
+            @panic("TODO strip terminal escape sequence");
+        }
+    }
+    const n = io.vtable.fileWriteStreaming(io.userdata, w.file, header, data, splat) catch |err| {
+        w.err = err;
+        return error.WriteFailed;
+    };
+    w.pos += n;
+    return w.interface.consume(n);
+}
+
 pub fn sendFile(io_w: *Io.Writer, file_reader: *Io.File.Reader, limit: Io.Limit) Io.Writer.FileError!usize {
     const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w));
     switch (w.mode) {
         .positional => return sendFilePositional(w, file_reader, limit),
-        .positional_reading => return error.Unimplemented,
+        .positional_simple => return error.Unimplemented,
         .streaming => return sendFileStreaming(w, file_reader, limit),
-        .streaming_reading => return error.Unimplemented,
+        .streaming_simple, .terminal_escaped, .terminal_winapi => return error.Unimplemented,
         .failure => return error.WriteFailed,
     }
 }
@@ -214,10 +389,10 @@ pub fn seekToUnbuffered(w: *Writer, offset: u64) SeekError!void {
     assert(w.interface.buffered().len == 0);
     const io = w.io;
     switch (w.mode) {
-        .positional, .positional_reading => {
+        .positional, .positional_simple => {
             w.pos = offset;
         },
-        .streaming, .streaming_reading => {
+        .streaming, .streaming_simple, .terminal_escaped, .terminal_winapi => {
             if (w.seek_err) |err| return err;
             io.vtable.fileSeekTo(io.userdata, w.file, offset) catch |err| {
                 w.seek_err = err;
@@ -243,15 +418,65 @@ pub fn end(w: *Writer) EndError!void {
     try w.interface.flush();
     switch (w.mode) {
         .positional,
-        .positional_reading,
+        .positional_simple,
         => w.file.setLength(io, w.pos) catch |err| switch (err) {
             error.NonResizable => return,
             else => |e| return e,
         },
 
         .streaming,
-        .streaming_reading,
+        .streaming_simple,
         .failure,
         => {},
     }
 }
+
+pub const Color = enum {
+    black,
+    red,
+    green,
+    yellow,
+    blue,
+    magenta,
+    cyan,
+    white,
+    bright_black,
+    bright_red,
+    bright_green,
+    bright_yellow,
+    bright_blue,
+    bright_magenta,
+    bright_cyan,
+    bright_white,
+    dim,
+    bold,
+    reset,
+};
+
+pub const SetColorError = Mode.SetColorError;
+
+pub fn setColor(w: *Writer, color: Color) SetColorError!void {
+    return w.mode.setColor(&w.interface, color);
+}
+
+pub fn disableEscape(w: *Writer) Mode {
+    const prev = w.mode;
+    w.mode = w.mode.toUnescaped();
+    return prev;
+}
+
+pub fn restoreEscape(w: *Writer, mode: Mode) void {
+    w.mode = mode;
+}
+
+pub fn writeAllUnescaped(w: *Writer, bytes: []const u8) Io.Error!void {
+    const prev_mode = w.disableEscape();
+    defer w.restoreEscape(prev_mode);
+    return w.interface.writeAll(bytes);
+}
+
+pub fn printUnescaped(w: *Writer, comptime fmt: []const u8, args: anytype) Io.Error!void {
+    const prev_mode = w.disableEscape();
+    defer w.restoreEscape(prev_mode);
+    return w.interface.print(fmt, args);
+}
diff --git a/lib/std/Io/Threaded.zig b/lib/std/Io/Threaded.zig
index 9d6d9f979e..c2a3e15f15 100644
--- a/lib/std/Io/Threaded.zig
+++ b/lib/std/Io/Threaded.zig
@@ -77,7 +77,13 @@ use_sendfile: UseSendfile = .default,
 use_copy_file_range: UseCopyFileRange = .default,
 use_fcopyfile: UseFcopyfile = .default,
 
-stderr_writer: Io.Writer,
+stderr_writer: File.Writer = .{
+    .io = undefined,
+    .interface = Io.File.Writer.initInterface(&.{}),
+    .file = if (is_windows) undefined else .stderr(),
+    .mode = undefined,
+},
+stderr_writer_initialized: bool = false,
 
 pub const RobustCancel = if (std.Thread.use_pthreads or native_os == .linux) enum {
     enabled,
@@ -737,6 +743,9 @@ pub fn io(t: *Threaded) Io {
 
             .processExecutableOpen = processExecutableOpen,
             .processExecutablePath = processExecutablePath,
+            .lockStderrWriter = lockStderrWriter,
+            .tryLockStderrWriter = tryLockStderrWriter,
+            .unlockStderrWriter = unlockStderrWriter,
 
             .now = now,
             .sleep = sleep,
@@ -864,6 +873,9 @@ pub fn ioBasic(t: *Threaded) Io {
 
             .processExecutableOpen = processExecutableOpen,
             .processExecutablePath = processExecutablePath,
+            .lockStderrWriter = lockStderrWriter,
+            .tryLockStderrWriter = tryLockStderrWriter,
+            .unlockStderrWriter = unlockStderrWriter,
 
             .now = now,
             .sleep = sleep,
@@ -9516,33 +9528,42 @@ fn netLookupFallible(
     return error.OptionUnsupported;
 }
 
-fn lockStderrWriter(userdata: ?*anyopaque, buffer: []u8) Io.Cancelable!*Io.Writer {
+fn lockStderrWriter(userdata: ?*anyopaque, buffer: []u8) Io.Cancelable!*File.Writer {
     const t: *Threaded = @ptrCast(@alignCast(userdata));
     // Only global mutex since this is Threaded.
     Io.stderr_thread_mutex.lock();
-    if (is_windows) t.stderr_writer.file = .stderr();
+    if (!t.stderr_writer_initialized) {
+        if (is_windows) t.stderr_writer.file = .stderr();
+        t.stderr_writer.mode = try .detect(ioBasic(t), t.stderr_writer.file, true, .streaming_simple);
+        t.stderr_writer_initialized = true;
+    }
     std.Progress.clearWrittenWithEscapeCodes(&t.stderr_writer) catch {};
-    t.stderr_writer.flush() catch {};
-    t.stderr_writer.buffer = buffer;
+    t.stderr_writer.interface.flush() catch {};
+    t.stderr_writer.interface.buffer = buffer;
     return &t.stderr_writer;
 }
 
-fn tryLockStderrWriter(userdata: ?*anyopaque, buffer: []u8) ?*Io.Writer {
+fn tryLockStderrWriter(userdata: ?*anyopaque, buffer: []u8) ?*File.Writer {
     const t: *Threaded = @ptrCast(@alignCast(userdata));
     // Only global mutex since this is Threaded.
     if (!Io.stderr_thread_mutex.tryLock()) return null;
-    std.Progress.clearWrittenWithEscapeCodes(t.io()) catch {};
-    if (is_windows) t.stderr_writer.file = .stderr();
-    t.stderr_writer.flush() catch {};
-    t.stderr_writer.buffer = buffer;
+    if (!t.stderr_writer_initialized) {
+        if (is_windows) t.stderr_writer.file = .stderr();
+        t.stderr_writer.mode = File.Writer.Mode.detect(ioBasic(t), t.stderr_writer.file, true, .streaming_simple) catch
+            return null;
+        t.stderr_writer_initialized = true;
+    }
+    std.Progress.clearWrittenWithEscapeCodes(&t.stderr_writer) catch {};
+    t.stderr_writer.interface.flush() catch {};
+    t.stderr_writer.interface.buffer = buffer;
     return &t.stderr_writer;
 }
 
 fn unlockStderrWriter(userdata: ?*anyopaque) void {
     const t: *Threaded = @ptrCast(@alignCast(userdata));
-    t.stderr_writer.flush() catch {};
-    t.stderr_writer.end = 0;
-    t.stderr_writer.buffer = &.{};
+    t.stderr_writer.interface.flush() catch {};
+    t.stderr_writer.interface.end = 0;
+    t.stderr_writer.interface.buffer = &.{};
     Io.stderr_thread_mutex.unlock();
 }
 
diff --git a/lib/std/Io/tty.zig b/lib/std/Io/tty.zig
deleted file mode 100644
index 65f1b7dad5..0000000000
--- a/lib/std/Io/tty.zig
+++ /dev/null
@@ -1,135 +0,0 @@
-const builtin = @import("builtin");
-const native_os = builtin.os.tag;
-
-const std = @import("std");
-const Io = std.Io;
-const File = std.Io.File;
-const process = std.process;
-const windows = std.os.windows;
-
-pub const Color = enum {
-    black,
-    red,
-    green,
-    yellow,
-    blue,
-    magenta,
-    cyan,
-    white,
-    bright_black,
-    bright_red,
-    bright_green,
-    bright_yellow,
-    bright_blue,
-    bright_magenta,
-    bright_cyan,
-    bright_white,
-    dim,
-    bold,
-    reset,
-};
-
-/// Provides simple functionality for manipulating the terminal in some way,
-/// such as coloring text, etc.
-pub const Config = union(enum) {
-    no_color,
-    escape_codes,
-    windows_api: if (native_os == .windows) WindowsContext else noreturn,
-
-    /// Detect suitable TTY configuration options for the given file (commonly stdout/stderr).
-    /// This includes feature checks for ANSI escape codes and the Windows console API, as well as
-    /// respecting the `NO_COLOR` and `CLICOLOR_FORCE` environment variables to override the default.
-    /// Will attempt to enable ANSI escape code support if necessary/possible.
-    pub fn detect(io: Io, file: File) Config {
-        const force_color: ?bool = if (builtin.os.tag == .wasi)
-            null // wasi does not support environment variables
-        else if (process.hasNonEmptyEnvVarConstant("NO_COLOR"))
-            false
-        else if (process.hasNonEmptyEnvVarConstant("CLICOLOR_FORCE"))
-            true
-        else
-            null;
-
-        if (force_color == false) return .no_color;
-
-        if (file.enableAnsiEscapeCodes(io)) |_| {
-            return .escape_codes;
-        } else |_| {}
-
-        if (native_os == .windows and file.isTty()) {
-            var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
-            if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) == windows.FALSE) {
-                return if (force_color == true) .escape_codes else .no_color;
-            }
-            return .{ .windows_api = .{
-                .handle = file.handle,
-                .reset_attributes = info.wAttributes,
-            } };
-        }
-
-        return if (force_color == true) .escape_codes else .no_color;
-    }
-
-    pub const WindowsContext = struct {
-        handle: File.Handle,
-        reset_attributes: u16,
-    };
-
-    pub const SetColorError = std.os.windows.SetConsoleTextAttributeError || Io.Writer.Error;
-
-    pub fn setColor(conf: Config, w: *Io.Writer, color: Color) SetColorError!void {
-        nosuspend switch (conf) {
-            .no_color => return,
-            .escape_codes => {
-                const color_string = switch (color) {
-                    .black => "\x1b[30m",
-                    .red => "\x1b[31m",
-                    .green => "\x1b[32m",
-                    .yellow => "\x1b[33m",
-                    .blue => "\x1b[34m",
-                    .magenta => "\x1b[35m",
-                    .cyan => "\x1b[36m",
-                    .white => "\x1b[37m",
-                    .bright_black => "\x1b[90m",
-                    .bright_red => "\x1b[91m",
-                    .bright_green => "\x1b[92m",
-                    .bright_yellow => "\x1b[93m",
-                    .bright_blue => "\x1b[94m",
-                    .bright_magenta => "\x1b[95m",
-                    .bright_cyan => "\x1b[96m",
-                    .bright_white => "\x1b[97m",
-                    .bold => "\x1b[1m",
-                    .dim => "\x1b[2m",
-                    .reset => "\x1b[0m",
-                };
-                try w.writeAll(color_string);
-            },
-            .windows_api => |ctx| {
-                const attributes = switch (color) {
-                    .black => 0,
-                    .red => windows.FOREGROUND_RED,
-                    .green => windows.FOREGROUND_GREEN,
-                    .yellow => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN,
-                    .blue => windows.FOREGROUND_BLUE,
-                    .magenta => windows.FOREGROUND_RED | windows.FOREGROUND_BLUE,
-                    .cyan => windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE,
-                    .white => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE,
-                    .bright_black => windows.FOREGROUND_INTENSITY,
-                    .bright_red => windows.FOREGROUND_RED | windows.FOREGROUND_INTENSITY,
-                    .bright_green => windows.FOREGROUND_GREEN | windows.FOREGROUND_INTENSITY,
-                    .bright_yellow => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_INTENSITY,
-                    .bright_blue => windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
-                    .bright_magenta => windows.FOREGROUND_RED | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
-                    .bright_cyan => windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
-                    .bright_white, .bold => windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY,
-                    // "dim" is not supported using basic character attributes, but let's still make it do *something*.
-                    // This matches the old behavior of TTY.Color before the bright variants were added.
-                    .dim => windows.FOREGROUND_INTENSITY,
-                    .reset => ctx.reset_attributes,
-                };
-                try w.flush();
-                try windows.SetConsoleTextAttribute(ctx.handle, attributes);
-            },
-        };
-    }
-};