blob: 40a7cc9b58bfdbc9ad12b7b0904820a4c4e59b28 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
#ifndef LEXER_H
#define LEXER_H
#include "token.h"
/**
* \page Lexer
*
* The lexer cuts some input text into blocks called tokens.
* This process is done **on demand**: the lexer doesn't read the
* input more than it needs, only creates tokens when lexer_peek
* or lexer_pop is called, and no token is available.
*
* "2 + 3" will produce 3 tokens:
* - TOKEN_NUMBER { .value = 2 }
* - TOKEN_PLUS
* - TOKEN_NUMBER { .value = 3 }
*/
struct lexer
{
const char *input; // The input data
size_t pos; // The current offset inside the input data
struct token current_tok; // The next token, if processed
};
/**
* \brief Creates a new lexer given an input string.
*/
struct lexer *lexer_new(const char *input);
/**
** \brief Frees the given lexer, but not its input.
*/
void lexer_free(struct lexer *lexer);
/**
* \brief Returns a token from the input string.
* This function goes through the input string character by character and
* builds a token. lexer_peek and lexer_pop should call it. If the input is
* invalid, you must print something on stderr and return the appropriate token.
*/
struct token lexer_next_token(struct lexer *lexer);
/**
* \brief Returns the next token, but doesn't move forward: calling lexer_peek
* multiple times in a row always returns the same result.
* This function is meant to help the parser check if the next token matches
* some rule.
*/
struct token lexer_peek(struct lexer *lexer);
/**
* \brief Returns the next token, and removes it from the stream:
* calling lexer_pop in a loop will iterate over all tokens until EOF.
*/
struct token lexer_pop(struct lexer *lexer);
#endif /* !LEXER_H */
|
