type flag[link]

type flag = enum uint {
	NONE = 0,
	COMMENTS = 1 << 0, // Enables lexing comments
};

Flags which apply to this lexer

type location[link]

type location = struct {
	path: str,
	line: uint,
	col: uint,
};

A location within a source file. The path is borrowed from the file name given to the lexer.

type ltok[link]

type ltok = enum uint {
	ATTR_FINI,
	ATTR_INIT,
	ATTR_OFFSET,
	ATTR_PACKED,
	ATTR_SYMBOL,
	ATTR_TEST,
	ATTR_THREADLOCAL,
	UNDERSCORE,
	ABORT,
	ALIGN,
	ALLOC,
	APPEND,
	AS,
	ASSERT,
	BOOL,
	BREAK,
	CASE,
	CONST,
	CONTINUE,
	DEF,
	DEFER,
	DELETE,
	DONE,
	ELSE,
	ENUM,
	EXPORT,
	F32,
	F64,
	FALSE,
	FN,
	FOR,
	FREE,
	I16,
	I32,
	I64,
	I8,
	IF,
	INSERT,
	INT,
	IS,
	LEN,
	LET,
	MATCH,
	NEVER,
	NULL,
	NULLABLE,
	OFFSET,
	OPAQUE,
	RETURN,
	RUNE,
	SIZE,
	STATIC,
	STR,
	STRUCT,
	SWITCH,
	TRUE,
	TYPE,
	U16,
	U32,
	U64,
	U8,
	UINT,
	UINTPTR,
	UNION,
	USE,
	VAARG,
	VAEND,
	VALIST,
	VASTART,
	VOID,
	YIELD,
	LAST_KEYWORD = YIELD,
	ARROW,
	BAND,
	BANDEQ,
	BNOT,
	BOR,
	BOREQ,
	BXOR,
	BXOREQ,
	COLON,
	COMMA,
	DIV,
	DIVEQ,
	DOT,
	DOUBLE_COLON,
	DOUBLE_DOT,
	ELLIPSIS,
	EQUAL,
	GT,
	GTEQ,
	LAND,
	LANDEQ,
	LBRACE,
	LBRACKET,
	LEQUAL,
	LESS,
	LESSEQ,
	LNOT,
	LOR,
	LOREQ,
	LPAREN,
	LSHIFT,
	LSHIFTEQ,
	LXOR,
	LXOREQ,
	MINUS,
	MINUSEQ,
	MODEQ,
	MODULO,
	NEQUAL,
	PLUS,
	PLUSEQ,
	QUESTION,
	RBRACE,
	RBRACKET,
	RPAREN,
	RSHIFT,
	RSHIFTEQ,
	SEMICOLON,
	TIMES,
	TIMESEQ,
	LAST_BTOK = TIMESEQ,
	LIT_U8,
	LIT_U16,
	LIT_U32,
	LIT_U64,
	LIT_UINT,
	LIT_SIZE,
	LIT_I8,
	LIT_I16,
	LIT_I32,
	LIT_I64,
	LIT_INT,
	LIT_ICONST,
	LIT_F32,
	LIT_F64,
	LIT_FCONST,
	LIT_RCONST,
	LIT_STR,
	LAST_LITERAL = LIT_STR,
	NAME,
	EOF,
};

A lexical token class.

type token[link]

type token = (ltok, value, location);

A single lexical token.

type value[link]

type value = (str | rune | u64 | f64 | void);

A token value, used for tokens such as '1337' (an integer).

type lexer[link]

type lexer = struct {
	in: *bufio::scanner,
	path: str,
	loc: (uint, uint),
	prevrloc: (uint, uint),
	// ltok::EOF when no token was unlexed
	un: token,
	prevunlocs: [2]((uint, uint), (uint, uint)),
	flags: flag,
	comment: str,
	require_int: bool,
};

type error[link]

type error = !(io::error | syntax);

All possible lexer errors

type syntax[link]

type syntax = !(location, str);

A syntax error

fn comment[link]

fn comment(lex: *lexer) str;

Returns the current value of the comment buffer, or empty string if unset (or if flag::COMMENTS was not enabled for this lexer).

fn init[link]

fn init(in: *bufio::scanner, path: str, flags: flag = flag::NONE) lexer;

Initializes a new lexer for the given bufio::scanner. The path is borrowed.

fn lex[link]

fn lex(lex: *lexer) (token | error);

Returns the next token from the lexer.

fn strerror[link]

fn strerror(err: error) const str;

Returns a human-friendly string for a given error. The result may be statically allocated.

fn tokstr[link]

fn tokstr(tok: token) const str;

Converts a token to its string representation.

fn unlex[link]

fn unlex(lex: *lexer, tok: token) void;

Unlex a single token. The next call to lex will return this token. Only one unlex is supported at a time; you must call lex before calling unlex again.

fn mkloc[link]

fn mkloc(lex: *lexer) location;

fn prevloc[link]

fn prevloc(lex: *lexer) location;

fn syntaxerr[link]

fn syntaxerr(loc: location, why: str) error;