Line data Source code
1 : // wok_token -- the token rosters and the scanner.
2 : //
3 : // The division of labour with the parser is deliberate: THE SCANNER RESOLVES
4 : // EVERYTHING THAT DEPENDS ON THE SOURCE TEXT; the parser only ever looks at
5 : // fields of a token. No production reads a token's bytes to make a decision --
6 : // text is carried into the AST or quoted in a diagnostic, never compared.
7 :
8 : #pragma once
9 :
10 : // The prelude comes FIRST: it carries the POSIX feature-test macros, which
11 : // have no effect once a system header has been read. wok_base.h hard-errors
12 : // if it is reached too late.
13 : #include "wok_base.h"
14 :
15 : #include <stdbool.h>
16 : #include <stddef.h>
17 : #include <stdint.h>
18 :
19 : #include "wok_arena.h"
20 : #include "wok_diag.h"
21 :
22 : // ---------------------------------------------------------------- kinds
23 : //
24 : // One roster. Every table keyed by a kind carries a static_assert against
25 : // WOK_KIND_COUNT, and no switch over a kind may carry a `default:` label --
26 : // that is what turns a missing enumerator into -Werror=switch.
27 :
28 : #define WOK_KINDS(X) \
29 : X(WT_EOF) \
30 : /* layout tokens: produced by stage 2, never by the scanner */ \
31 : X(WT_NEWLINE) X(WT_INDENT) X(WT_DEDENT) \
32 : /* open lexical classes */ \
33 : X(WT_VARID) X(WT_CONID) X(WT_VARSYM) X(WT_KEYWORD) \
34 : X(WT_INT) X(WT_STRING) X(WT_CHAR) \
35 : /* brackets */ \
36 : X(WT_LPAREN) X(WT_RPAREN) X(WT_LBRACKET) X(WT_RBRACKET) \
37 : X(WT_LBRACE) X(WT_RBRACE) \
38 : /* fixed punctuation */ \
39 : X(WT_COMMA) X(WT_DOT) X(WT_DOTDOT) X(WT_BACKTICK) \
40 : X(WT_UNDERSCORE) X(WT_LAMBDA) \
41 : /* reserved operator runs, carved out of WT_VARSYM by table lookup */ \
42 : X(WT_ARROW) X(WT_FATARROW) X(WT_EQUALS) X(WT_COLON) \
43 : X(WT_COLONCOLON) X(WT_ASSIGN) X(WT_BAR) \
44 : /* v2 has no statement separator; scanned only to diagnose it (D22) */ \
45 : X(WT_SEMI) \
46 : X(WT_BAD)
47 :
48 : typedef enum wok_kind : unsigned char {
49 : #define WOK_X(name) name,
50 : WOK_KINDS(WOK_X)
51 : #undef WOK_X
52 : WOK_KIND_COUNT
53 : } WokKind;
54 :
55 : static_assert(WOK_KIND_COUNT <= 255, "kind must fit in one byte");
56 :
57 : WOK_READONLY const char *wok_kind_name(WokKind);
58 :
59 : // ---------------------------------------------------------------- words
60 : //
61 : // The interned identity of a NAMED SPELLING. Three registers stay apart
62 : // without a second roster because each records the kind it is emitted as:
63 : //
64 : // WT_KEYWORD -- reserved; never a variable.
65 : // WT_VARID -- CONTEXTUAL; an ordinary identifier that carries its tag, so
66 : // the one production that wants it matches an integer and
67 : // every other production sees a plain name. `own`/`lend`/
68 : // `copy` must be contextual: spec.md section 2 writes
69 : // `Bytes.copy` itself.
70 : // WT_VARSYM -- a user operator that one production happens to name; `+`
71 : // stays freely redefinable.
72 : //
73 : // Matching a word is an integer compare, and `wok_word_is(t, WW_HANLDE)`
74 : // fails to COMPILE where a string comparison would compile, run, and simply
75 : // never be true.
76 :
77 : #define WOK_WORDS(X) \
78 : /* --- ML skeleton: the ordinary word on familiar ground (D10) --- */ \
79 : X(WW_MODULE, "module", WT_KEYWORD) \
80 : X(WW_IMPORT, "import", WT_KEYWORD) \
81 : X(WW_AS, "as", WT_KEYWORD) \
82 : X(WW_TYPE, "type", WT_KEYWORD) \
83 : X(WW_ALIAS, "alias", WT_KEYWORD) \
84 : X(WW_CLASS, "class", WT_KEYWORD) \
85 : X(WW_INSTANCE, "instance", WT_KEYWORD) \
86 : X(WW_LET, "let", WT_KEYWORD) \
87 : X(WW_IN, "in", WT_KEYWORD) \
88 : X(WW_CASE, "case", WT_KEYWORD) \
89 : X(WW_OF, "of", WT_KEYWORD) \
90 : X(WW_IF, "if", WT_KEYWORD) \
91 : X(WW_THEN, "then", WT_KEYWORD) \
92 : X(WW_ELSE, "else", WT_KEYWORD) \
93 : X(WW_WHERE, "where", WT_KEYWORD) \
94 : /* --- law words: the keyword states the promise (D10) --- */ \
95 : X(WW_EFFECT, "effect", WT_KEYWORD) \
96 : X(WW_HANDLER, "handler", WT_KEYWORD) \
97 : X(WW_HANDLE, "handle", WT_KEYWORD) \
98 : X(WW_USE, "use", WT_KEYWORD) \
99 : X(WW_ABORT, "abort", WT_KEYWORD) \
100 : X(WW_RETURN, "return", WT_KEYWORD) \
101 : X(WW_VAR, "var", WT_KEYWORD) \
102 : X(WW_WITH, "with", WT_KEYWORD) \
103 : X(WW_FOREIGN, "foreign", WT_KEYWORD) \
104 : X(WW_EXTERN, "extern", WT_KEYWORD) \
105 : X(WW_FIXITY, "fixity", WT_KEYWORD) \
106 : /* --- contextual: tagged, but still ordinary identifiers --- */ \
107 : /* `once` is CUT (D25): reserved at CLAUSE-HEAD position only, where */ \
108 : /* parse_clause reads the tag to emit the v1-migration diagnostic. */ \
109 : X(WW_ONCE, "once", WT_VARID) \
110 : X(WW_OWN, "own", WT_VARID) \
111 : X(WW_LEND, "lend", WT_VARID) \
112 : X(WW_COPY, "copy", WT_VARID) \
113 : /* the five words of a `fixity` line. Contextual for the same reason */ \
114 : /* `own` is: `left`, `right` and `than` are ordinary names anywhere */ \
115 : /* else, and reserving them would cost every program that has one. */ \
116 : X(WW_LEFT, "left", WT_VARID) \
117 : X(WW_RIGHT, "right", WT_VARID) \
118 : X(WW_TIGHTER, "tighter", WT_VARID) \
119 : X(WW_LOOSER, "looser", WT_VARID) \
120 : X(WW_THAN, "than", WT_VARID) \
121 : X(WW_ROW, "row", WT_VARID) \
122 : X(WW_EFF, "eff", WT_VARID) \
123 : /* --- an operator run one production reads by name --- */ \
124 : X(WW_PLUS, "+", WT_VARSYM)
125 :
126 : typedef enum wok_word : unsigned char {
127 : WW_NONE = 0,
128 : #define WOK_X(name, text, kind) name,
129 : WOK_WORDS(WOK_X)
130 : #undef WOK_X
131 : WOK_WORD_COUNT
132 : } WokWord;
133 :
134 : static_assert(WOK_WORD_COUNT <= 255, "word must fit in one byte");
135 :
136 : WOK_READONLY const char *wok_word_text(WokWord);
137 : WOK_READONLY WokKind wok_word_kind(WokWord);
138 :
139 : // ---------------------------------------------------------------- token
140 :
141 : enum {
142 : WOK_TF_FIRST_ON_LINE = 1u << 0, // this token opens its physical line
143 : WOK_TF_HAS_ESCAPE = 1u << 1, // literal text differs from source bytes
144 : };
145 :
146 : typedef struct {
147 : u32 off; // byte offset into the source; text is never copied
148 : u32 len;
149 : u32 col; // 1-based; tabs are rejected in indentation, so this is exact
150 : u8 kind; // WokKind
151 : u8 word; // WokWord, or WW_NONE
152 : u16 flags;
153 : } WokToken;
154 :
155 : static_assert(sizeof(WokToken) == 16, "token is the hot array; keep it 16 bytes");
156 :
157 : typedef struct {
158 : WokToken *tok;
159 : usize n; // always >= 1; the last token is WT_EOF and is never consumed past
160 : } WokTokens;
161 :
162 : // Comments are collected into a side list rather than discarded, so attaching
163 : // them as trivia does not have to re-open the scanner. wok_trivia_attach
164 : // consumes this list; the tokens themselves never mention a comment.
165 : typedef struct {
166 : u32 off, len;
167 : bool block; // {- -} rather than --
168 : } WokComment;
169 :
170 : typedef struct {
171 : WokTokens tokens;
172 : WokComment *comments;
173 : usize ncomments;
174 : } WokScanResult;
175 :
176 : WokScanResult wok_scan(const char *src, usize src_len, WokArena *,
177 : WokDiagSink *);
178 :
179 : // ------------------------------------------------- lexical classification
180 :
181 : // Static inline, not out-of-line: these are asked once per token, and
182 : // profiling showed the calls themselves on the clock (~5% of a parse).
183 6789841 : WOK_PURE static inline bool wok_kind_is_open_bracket(WokKind k) {
184 6789841 : return k == WT_LPAREN || k == WT_LBRACKET || k == WT_LBRACE;
185 : }
186 6430039 : WOK_PURE static inline bool wok_kind_is_close_bracket(WokKind k) {
187 6430039 : return k == WT_RPAREN || k == WT_RBRACKET || k == WT_RBRACE;
188 : }
189 :
190 : // THE predicate stage 2 is built on: a token that can never BEGIN a block
191 : // item, and therefore marks its line as a CONTINUATION of the previous one.
192 : // It lives here, not in wok_layout.c, because it is a lexical fact -- the
193 : // layout filter must not be able to name a keyword.
194 : //
195 : // Two things hold this down, and a third does not.
196 : //
197 : // - -Wswitch. The switch below carries no `default:`, over the kinds OR the
198 : // words, so a token added to either roster does not COMPILE until someone
199 : // classifies it here. That is the load-bearing half, and it is the reason
200 : // no roster test is needed to say every token has an answer.
201 : // - test_fill pins the PRINTER's own copy of this question against this one
202 : // for every lead a fill break can produce, so the two cannot drift into a
203 : // second opinion about the lexis.
204 : //
205 : // What is NOT checked is that each answer is RIGHT -- that a token called a
206 : // lead cannot in fact begin an item. A wrong answer there compiles, and under
207 : // an offside rule it is silent: the line joins the previous one instead of
208 : // starting an item, and only the tree shows it. Checking it needs the FIRST
209 : // set of the grammar, and the only honest source for that is a generator that
210 : // builds items rather than a second list typed out beside this one -- so if it
211 : // is ever written, it belongs in test_generative, which already builds items
212 : // across 80 of the schema's 82 tags.
213 765787 : WOK_READONLY static inline bool wok_token_is_continuation_lead(
214 : const WokToken *t) {
215 765787 : switch ((WokKind)t->kind) {
216 : // Every operator lead. No prefix operator exists except `-`, and a line
217 : // beginning with `-` is subtraction continuing the previous line -- the
218 : // same call v2 already made when it decided `1-2` is subtraction.
219 : case WT_VARSYM:
220 : case WT_ARROW:
221 : case WT_FATARROW:
222 : case WT_EQUALS:
223 : case WT_COLON:
224 : case WT_COLONCOLON:
225 : case WT_ASSIGN:
226 : case WT_BAR:
227 : case WT_COMMA:
228 : case WT_DOT:
229 : case WT_DOTDOT:
230 : case WT_BACKTICK:
231 : case WT_RPAREN:
232 : case WT_RBRACKET:
233 : case WT_RBRACE:
234 : return true;
235 : // Words that can never begin a block item.
236 186462 : case WT_KEYWORD:
237 186462 : switch ((WokWord)t->word) {
238 : case WW_WHERE:
239 : case WW_IN:
240 : case WW_THEN:
241 : case WW_ELSE:
242 : case WW_OF:
243 : case WW_AS:
244 : case WW_WITH:
245 : return true;
246 175072 : case WW_NONE:
247 : case WW_MODULE:
248 : case WW_IMPORT:
249 : case WW_TYPE:
250 : case WW_ALIAS:
251 : case WW_CLASS:
252 : case WW_INSTANCE:
253 : case WW_LET:
254 : case WW_CASE:
255 : case WW_IF:
256 : case WW_EFFECT:
257 : case WW_HANDLER:
258 : case WW_HANDLE:
259 : case WW_USE:
260 : case WW_ABORT:
261 : case WW_ONCE:
262 : case WW_RETURN:
263 : case WW_VAR:
264 : case WW_FOREIGN:
265 : case WW_EXTERN:
266 : case WW_FIXITY:
267 : case WW_OWN:
268 : case WW_LEND:
269 : case WW_COPY:
270 : case WW_ROW:
271 : case WW_EFF:
272 : case WW_LEFT:
273 : case WW_RIGHT:
274 : case WW_TIGHTER:
275 : case WW_LOOSER:
276 : case WW_THAN:
277 : case WW_PLUS:
278 : case WOK_WORD_COUNT:
279 175072 : return false;
280 : }
281 0 : WOK_UNREACHABLE();
282 474967 : case WT_EOF:
283 : case WT_NEWLINE:
284 : case WT_INDENT:
285 : case WT_DEDENT:
286 : case WT_VARID:
287 : case WT_CONID:
288 : case WT_INT:
289 : case WT_STRING:
290 : case WT_CHAR:
291 : case WT_LPAREN:
292 : case WT_LBRACKET:
293 : case WT_LBRACE:
294 : case WT_UNDERSCORE:
295 : case WT_LAMBDA:
296 : case WT_SEMI:
297 : case WT_BAD:
298 : case WOK_KIND_COUNT:
299 474967 : return false;
300 : }
301 0 : WOK_UNREACHABLE();
302 : }
303 :
304 : // Decodes an integer literal, reporting E-LEX-INT-RANGE on overflow past U64.
305 : // Uses ckd_mul/ckd_add where available: literal conversion is where parsers
306 : // get CVEs, and this is a defect-class elimination rather than a style choice.
307 : bool wok_token_int_value(const char *src, const WokToken *, WokDiagSink *,
308 : u64 *out);
309 :
310 : static inline const char *wok_token_text(const char *src, const WokToken *t) {
311 : return src + t->off;
312 : }
313 :
314 : // Test hooks. The symbol charset is a bitmap for speed; test_roster.c checks
315 : // it against the readable definition for all 256 byte values, which is what
316 : // makes a hand-written bitmap safe. wok_print.c also reads two of them, so the
317 : // filler's lead test cannot grow a second opinion about the charset.
318 : bool wok_test_is_sym(unsigned char);
319 : bool wok_test_is_ident_start(unsigned char);
320 : bool wok_test_is_ident_cont(unsigned char);
321 :
|
