type annotatefn[permalink] [source]

type annotatefn = fn(id: []str, lex: *lexer, user: nullable *opaque) (void | error);

A function which implements an annotation. The state of the lexer at the time of the callback will be immediately following the identifier, and the callback should lex all tokens up to and including the terminating ']'.

#[example::annotation(...)]
                     ^------- start of callback
                           ^- end of callback

The lifetime of the id parameter is only that of the callback's execution duration.

type flag[permalink] [source]

type flag = enum uint {
	NONE = 0,
	COMMENTS = 1 << 0, // Enables lexing comments
};

Flags which apply to this lexer

type location[permalink] [source]

type location = struct {
	path: str,
	line: uint,
	col: uint,
	off: io::off,
};

A location within a source file. The path is borrowed from the file name given to the lexer.

type ltok[permalink] [source]

type ltok = enum uint {
	ATTR_FINI,
	ATTR_INIT,
	ATTR_OFFSET,
	ATTR_PACKED,
	ATTR_SYMBOL,
	ATTR_TEST,
	ATTR_THREADLOCAL,
	UNDERSCORE,
	ABORT,
	ALIGN,
	ALLOC,
	APPEND,
	AS,
	ASSERT,
	BOOL,
	BREAK,
	CASE,
	CONST,
	CONTINUE,
	DEF,
	DEFER,
	DELETE,
	DONE,
	ELSE,
	ENUM,
	EXPORT,
	F32,
	F64,
	FALSE,
	FN,
	FOR,
	FREE,
	I16,
	I32,
	I64,
	I8,
	IF,
	INSERT,
	INT,
	IS,
	LEN,
	LET,
	MATCH,
	NEVER,
	NOMEM,
	NULL,
	NULLABLE,
	OFFSET,
	OPAQUE,
	RETURN,
	RUNE,
	SIZE,
	STATIC,
	STR,
	STRUCT,
	SWITCH,
	TRUE,
	TYPE,
	U16,
	U32,
	U64,
	U8,
	UINT,
	UINTPTR,
	UNION,
	USE,
	VAARG,
	VAEND,
	VALIST,
	VASTART,
	VOID,
	YIELD,
	LAST_KEYWORD = YIELD,
	ARROW,
	BAND,
	BANDEQ,
	BNOT,
	BOR,
	BOREQ,
	BXOR,
	BXOREQ,
	COLON,
	COMMA,
	DIV,
	DIVEQ,
	DOT,
	DOUBLE_COLON,
	DOUBLE_DOT,
	ELLIPSIS,
	EQUAL,
	GT,
	GTEQ,
	LAND,
	LANDEQ,
	LBRACE,
	LBRACKET,
	LEQUAL,
	LESS,
	LESSEQ,
	LNOT,
	LOR,
	LOREQ,
	LPAREN,
	LSHIFT,
	LSHIFTEQ,
	LXOR,
	LXOREQ,
	MINUS,
	MINUSEQ,
	MODEQ,
	MODULO,
	NEQUAL,
	PLUS,
	PLUSEQ,
	QUESTION,
	RBRACE,
	RBRACKET,
	RPAREN,
	RSHIFT,
	RSHIFTEQ,
	SEMICOLON,
	TIMES,
	TIMESEQ,
	LAST_BTOK = TIMESEQ,
	LIT_U8,
	LIT_U16,
	LIT_U32,
	LIT_U64,
	LIT_UINT,
	LIT_SIZE,
	LIT_I8,
	LIT_I16,
	LIT_I32,
	LIT_I64,
	LIT_INT,
	LIT_ICONST,
	LIT_F32,
	LIT_F64,
	LIT_FCONST,
	LIT_RCONST,
	LIT_STR,
	LAST_LITERAL = LIT_STR,
	NAME,
	EOF,
};

A lexical token class.

type token[permalink] [source]

type token = (ltok, value, location);

A single lexical token.

type value[permalink] [source]

type value = (str | rune | u64 | f64 | void);

A token value, used for tokens such as '1337' (an integer).

type lexer[permalink] [source]

type lexer = struct {
	in: *bufio::scanner,
	path: str,
	loc: (uint, uint, io::off),
	prevrloc: (uint, uint, io::off),
	// ltok::EOF when no token was unlexed
	un: token,
	prevunlocs: [2]((uint, uint, io::off), (uint, uint, io::off)),
	flags: flag,
	comment: str,
	require_int: bool,
	annotations: []([]str, *annotatefn, nullable *opaque),
};

type restore_point[permalink] [source]

type restore_point = struct {
	off: io::off,
	state: lexer,
};

type error[permalink] [source]

type error = !(io::error | syntax);

All possible lexer errors

type syntax[permalink] [source]

type syntax = !(location, str);

A syntax error

fn comment[permalink] [source]

fn comment(lex: *lexer) str;

Returns the current value of the comment buffer, or empty string if unset (or if flag::COMMENTS was not enabled for this lexer).

fn init[permalink] [source]

fn init(in: *bufio::scanner, path: str, flags: flag = flag::NONE) lexer;

Initializes a new lexer for the given bufio::scanner. The path is borrowed.

fn lex[permalink] [source]

fn lex(lex: *lexer) (token | error);

Returns the next token from the lexer.

fn register_annotation[permalink] [source]

fn register_annotation(lex: *lexer, id: []str, cb: *annotatefn, user: nullable *opaque) void;

Registers an annotation callback with this lexer for the given identifier. The id is borrowed for the lifetime of the lexer. Unregistered annotations are silently discarded by the lexer.

fn restore[permalink] [source]

fn restore(lex: *lexer, rp: *restore_point) (void | io::error);

Restores a lexer to a state previously recorded with save.

fn save[permalink] [source]

fn save(lex: *lexer) (restore_point | io::error);

Saves the state of a lexer, to be restored later with restore. The underlying I/O source must be seekable.

fn strerror[permalink] [source]

fn strerror(err: error) const str;

Returns a human-friendly string for a given error. The result may be statically allocated.

fn syntaxerr[permalink] [source]

fn syntaxerr(loc: location, why: str) error;

Builds a lexer syntax error from a location and a reason string.

fn tokstr[permalink] [source]

fn tokstr(tok: token) const str;

Converts a token to its string representation.

fn unlex[permalink] [source]

fn unlex(lex: *lexer, tok: token) void;

Unlex a single token. The next call to lex will return this token. Only one unlex is supported at a time; you must call lex before calling unlex again.

fn mkloc[permalink] [source]

fn mkloc(lex: *lexer) location;

fn prevloc[permalink] [source]

fn prevloc(lex: *lexer) location;