1 files changed, 40 insertions, 0 deletions
diff --git a/src/lexer.h b/src/lexer.h
new file mode 100644
index 0000000..2fd7fa7
--- /dev/null
+++ b/src/lexer.h
@@ -0,0 +1,40 @@
+#ifndef LEXER_H
+#define LEXER_H
+
+#include "prelexer.h"
+
+/* Structure to hold single token. */
+typedef struct
+{
+ gchar* string; /* Poniter to local copy of token string. */
+ guint start_index; /* Start index in original stream. */
+ guint end_index; /* End index in original stream. */
+ LexerTokenType token_type; /* Type of token. */
+} LexerToken;
+
+/* Structure to hold lexer state and all the tokens. */
+typedef struct
+{
+ PreLexerState *prelexer; /* Pre-lexer state. Pre-lexer is part of lexer. */
+ LexerToken *tokens; /* Pointer to the dynamic array of LexerTokens. */
+ guint token_count; /* Count of tokens in array. */
+ guint next_token; /* Index of next, to be sent, token. */
+ struct parser_state *parent; /* Pointer to the parent parser. */
+} LexerState;
+
+/* Create a new LexerState object and fill the dynamic array with tokens. */
+LexerState* l_create_lexer(const gchar*, struct parser_state*);
+
+/* Destroy LexerState object and free up space. */
+void l_destroy_lexer(LexerState*);
+
+/* Tokanize complete string. */
+void l_insert_all_tokens(LexerState*);
+
+/* Return next, to be sent, token. */
+LexerToken* l_get_next_token(LexerState*);
+
+/* Roll back one token. */
+void l_roll_back(LexerState*);
+
+#endif /* LEXER_H */