1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
/*
* Copyright (c) 2016 Andrew Kelley
*
* This file is part of zig, which is MIT licensed.
* See http://opensource.org/licenses/MIT
*/
#ifndef ZIG_C_TOKENIZER_HPP
#define ZIG_C_TOKENIZER_HPP
#include "buffer.hpp"
enum CTokId {
CTokIdCharLit,
CTokIdStrLit,
CTokIdNumLitInt,
CTokIdNumLitFloat,
CTokIdSymbol,
CTokIdMinus,
CTokIdLParen,
CTokIdRParen,
CTokIdEOF,
CTokIdDot,
CTokIdAsterisk,
CTokIdBang,
CTokIdTilde,
};
enum CNumLitSuffix {
CNumLitSuffixNone,
CNumLitSuffixL,
CNumLitSuffixU,
CNumLitSuffixLU,
CNumLitSuffixLL,
CNumLitSuffixLLU,
};
struct CNumLitInt {
uint64_t x;
CNumLitSuffix suffix;
};
struct CTok {
enum CTokId id;
union {
uint8_t char_lit;
Buf str_lit;
CNumLitInt num_lit_int;
double num_lit_float;
Buf symbol;
} data;
};
enum CTokState {
CTokStateStart,
CTokStateExpectChar,
CTokStateCharEscape,
CTokStateExpectEndQuot,
CTokStateOpenComment,
CTokStateLineComment,
CTokStateComment,
CTokStateCommentStar,
CTokStateBackslash,
CTokStateString,
CTokStateIdentifier,
CTokStateDecimal,
CTokStateOctal,
CTokStateGotZero,
CTokStateHex,
CTokStateFloat,
CTokStateExpSign,
CTokStateFloatExp,
CTokStateFloatExpFirst,
CTokStateStrHex,
CTokStateStrOctal,
CTokStateNumLitIntSuffixU,
CTokStateNumLitIntSuffixL,
CTokStateNumLitIntSuffixLL,
CTokStateNumLitIntSuffixUL,
};
struct CTokenize {
ZigList<CTok> tokens;
CTokState state;
bool error;
CTok *cur_tok;
Buf buf;
uint8_t cur_char;
int octal_index;
};
void tokenize_c_macro(CTokenize *ctok, const uint8_t *c);
#endif
|
