From 29f41896ed9d99e82a88f4b63efa182ca0d2f93c Mon Sep 17 00:00:00 2001 From: Travis Staloch Date: Thu, 2 Sep 2021 13:50:24 -0700 Subject: sat-arithmetic: add operator support - adds initial support for the operators +|, -|, *|, <<|, +|=, -|=, *|=, <<|= - uses operators in addition to builtins in behavior test - adds binOpExt() and assignBinOpExt() to AstGen.zig. these need to be audited --- src/stage1/tokenizer.cpp | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) (limited to 'src/stage1/tokenizer.cpp') diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp index f10579c966..3560193927 100644 --- a/src/stage1/tokenizer.cpp +++ b/src/stage1/tokenizer.cpp @@ -226,8 +226,10 @@ enum TokenizeState { TokenizeState_pipe, TokenizeState_minus, TokenizeState_minus_percent, + TokenizeState_minus_pipe, TokenizeState_asterisk, TokenizeState_asterisk_percent, + TokenizeState_asterisk_pipe, TokenizeState_slash, TokenizeState_line_comment_start, TokenizeState_line_comment, @@ -257,8 +259,10 @@ enum TokenizeState { TokenizeState_percent, TokenizeState_plus, TokenizeState_plus_percent, + TokenizeState_plus_pipe, TokenizeState_angle_bracket_left, TokenizeState_angle_bracket_angle_bracket_left, + TokenizeState_angle_bracket_angle_bracket_left_pipe, TokenizeState_angle_bracket_right, TokenizeState_angle_bracket_angle_bracket_right, TokenizeState_period, @@ -548,6 +552,9 @@ void tokenize(const char *source, Tokenization *out) { case '%': t.state = TokenizeState_asterisk_percent; break; + case '|': + t.state = TokenizeState_asterisk_pipe; + break; default: t.state = TokenizeState_start; continue; @@ -568,6 +575,21 @@ void tokenize(const char *source, Tokenization *out) { continue; } break; + case TokenizeState_asterisk_pipe: + switch (c) { + case 0: + t.out->ids.last() = TokenIdTimesPipe; + goto eof; + case '=': + t.out->ids.last() = TokenIdTimesPipeEq; + t.state = TokenizeState_start; + break; + default: + t.out->ids.last() = TokenIdTimesPipe; + t.state = TokenizeState_start; + continue; + } + break; case TokenizeState_percent: switch (c) { case 0: @@ -596,6 +618,9 @@ void tokenize(const char *source, Tokenization *out) { case '%': t.state = TokenizeState_plus_percent; break; + case '|': + t.state = TokenizeState_plus_pipe; + break; default: t.state = TokenizeState_start; continue; @@ -616,6 +641,21 @@ void tokenize(const char *source, Tokenization *out) { continue; } break; + case TokenizeState_plus_pipe: + switch (c) { + case 0: + t.out->ids.last() = TokenIdPlusPipe; + goto eof; + case '=': + t.out->ids.last() = TokenIdPlusPipeEq; + t.state = TokenizeState_start; + break; + default: + t.out->ids.last() = TokenIdPlusPipe; + t.state = TokenizeState_start; + continue; + } + break; case TokenizeState_caret: switch (c) { case 0: @@ -891,6 +931,9 @@ void tokenize(const char *source, Tokenization *out) { case '%': t.state = TokenizeState_minus_percent; break; + case '|': + t.state = TokenizeState_minus_pipe; + break; default: t.state = TokenizeState_start; continue; @@ -911,6 +954,21 @@ void tokenize(const char *source, Tokenization *out) { continue; } break; + case TokenizeState_minus_pipe: + switch (c) { + case 0: + t.out->ids.last() = TokenIdMinusPipe; + goto eof; + case '=': + t.out->ids.last() = TokenIdMinusPipeEq; + t.state = TokenizeState_start; + break; + default: + t.out->ids.last() = TokenIdMinusPipe; + t.state = TokenizeState_start; + continue; + } + break; case TokenizeState_angle_bracket_left: switch (c) { case 0: @@ -936,12 +994,31 @@ void tokenize(const char *source, Tokenization *out) { t.out->ids.last() = TokenIdBitShiftLeftEq; t.state = TokenizeState_start; break; + case '|': + // t.out->ids.last() = TokenIdBitShiftLeftPipe; + t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe; + break; default: t.out->ids.last() = TokenIdBitShiftLeft; t.state = TokenizeState_start; continue; } break; + case TokenizeState_angle_bracket_angle_bracket_left_pipe: + switch (c) { + case 0: + t.out->ids.last() = TokenIdBitShiftLeftPipe; + goto eof; + case '=': + t.out->ids.last() = TokenIdBitShiftLeftPipeEq; + t.state = TokenizeState_start; + break; + default: + t.out->ids.last() = TokenIdBitShiftLeftPipe; + t.state = TokenizeState_start; + continue; + } + break; case TokenizeState_angle_bracket_right: switch (c) { case 0: @@ -1437,6 +1514,8 @@ const char * token_name(TokenId id) { case TokenIdBitOrEq: return "|="; case TokenIdBitShiftLeft: return "<<"; case TokenIdBitShiftLeftEq: return "<<="; + case TokenIdBitShiftLeftPipe: return "<<|"; + case TokenIdBitShiftLeftPipeEq: return "<<|="; case TokenIdBitShiftRight: return ">>"; case TokenIdBitShiftRightEq: return ">>="; case TokenIdBitXorEq: return "^="; @@ -1521,12 +1600,16 @@ const char * token_name(TokenId id) { case TokenIdMinusEq: return "-="; case TokenIdMinusPercent: return "-%"; case TokenIdMinusPercentEq: return "-%="; + case TokenIdMinusPipe: return "-|"; + case TokenIdMinusPipeEq: return "-|="; case TokenIdModEq: return "%="; case TokenIdPercent: return "%"; case TokenIdPlus: return "+"; case TokenIdPlusEq: return "+="; case TokenIdPlusPercent: return "+%"; case TokenIdPlusPercentEq: return "+%="; + case TokenIdPlusPipe: return "+|"; + case TokenIdPlusPipeEq: return "+|="; case TokenIdPlusPlus: return "++"; case TokenIdRBrace: return "}"; case TokenIdRBracket: return "]"; @@ -1542,6 +1625,8 @@ const char * token_name(TokenId id) { case TokenIdTimesEq: return "*="; case TokenIdTimesPercent: return "*%"; case TokenIdTimesPercentEq: return "*%="; + case TokenIdTimesPipe: return "*|"; + case TokenIdTimesPipeEq: return "*|="; case TokenIdBuiltin: return "Builtin"; case TokenIdCount: zig_unreachable(); -- cgit v1.2.3 From dcbc52ec85d00fbd3603c314aaaab98fb3866892 Mon Sep 17 00:00:00 2001 From: Travis Staloch Date: Wed, 8 Sep 2021 15:58:37 -0700 Subject: sat-arithmetic: correctly tokenize <<|, <<|= - set state rather than result.tag in tokenizer.zig - add test to tokenizer.zig for <<, <<|, <<|= --- lib/std/zig/tokenizer.zig | 8 +++++++- src/Air.zig | 6 +++--- src/stage1/tokenizer.cpp | 1 - 3 files changed, 10 insertions(+), 5 deletions(-) (limited to 'src/stage1/tokenizer.cpp') diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 6afe7750d3..a7442b8b25 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1007,7 +1007,7 @@ pub const Tokenizer = struct { break; }, '|' => { - result.tag = .angle_bracket_angle_bracket_left_pipe; + state = .angle_bracket_angle_bracket_left_pipe; }, else => { result.tag = .angle_bracket_angle_bracket_left; @@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" { try testTokenize("'\\u", &.{.invalid}); } +test "tokenizer - saturating" { + try testTokenize("<<", &.{.angle_bracket_angle_bracket_left}); + try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe}); + try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal}); +} + fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { diff --git a/src/Air.zig b/src/Air.zig index b7d3938352..00f223ad21 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -44,7 +44,7 @@ pub const Inst = struct { /// is the same as both operands. /// Uses the `bin_op` field. addwrap, - /// Saturating integer addition. + /// Saturating integer addition. /// Both operands are guaranteed to be the same type, and the result type /// is the same as both operands. /// Uses the `bin_op` field. @@ -59,7 +59,7 @@ pub const Inst = struct { /// is the same as both operands. /// Uses the `bin_op` field. subwrap, - /// Saturating integer subtraction. + /// Saturating integer subtraction. /// Both operands are guaranteed to be the same type, and the result type /// is the same as both operands. /// Uses the `bin_op` field. @@ -74,7 +74,7 @@ pub const Inst = struct { /// is the same as both operands. /// Uses the `bin_op` field. mulwrap, - /// Saturating integer multiplication. + /// Saturating integer multiplication. /// Both operands are guaranteed to be the same type, and the result type /// is the same as both operands. /// Uses the `bin_op` field. diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp index 3560193927..47e324c933 100644 --- a/src/stage1/tokenizer.cpp +++ b/src/stage1/tokenizer.cpp @@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) { t.state = TokenizeState_start; break; case '|': - // t.out->ids.last() = TokenIdBitShiftLeftPipe; t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe; break; default: -- cgit v1.2.3