type annotatefn[permalink] [source]

type annotatefn = fn(id: []str, lex: *lexer, user: nullable *opaque) (void | error);

A function which implements an annotation. The state of the lexer at the time of the callback will be immediately following the identifier, and the callback should lex all tokens up to and including the terminating ']'.

#[example::annotation(...)]
                     ^------- start of callback
                           ^- end of callback

The lifetime of the id parameter is only that of the callback's execution duration.

type flag[permalink] [source]

type flag = enum uint {
	NONE = 0,
	COMMENTS = 1 << 0, // Enables lexing comments.
};

Flags which apply to this lexer.

type location[permalink] [source]

type location = struct {
	path: str,
	line: uint,
	col: uint,
	off: io::off,
};

A location within a source file. The path is borrowed from the file name given to the lexer.

type ltok[permalink] [source]

type ltok = enum uint {
	ATTR_FINI,
	ATTR_INIT,
	ATTR_PACKED,
	ATTR_SYMBOL,
	ATTR_TEST,
	ATTR_THREADLOCAL,
	UNDERSCORE,
	ABORT,
	ALIGN,
	ALLOC,
	APPEND,
	AS,
	ASSERT,
	BOOL,
	BREAK,
	CASE,
	CONST,
	CONTINUE,
	DEF,
	DEFER,
	DELETE,
	DONE,
	ELSE,
	ENUM,
	EXPORT,
	F32,
	F64,
	FALSE,
	FN,
	FOR,
	FREE,
	I16,
	I32,
	I64,
	I8,
	IF,
	INSERT,
	INT,
	IS,
	LEN,
	LET,
	MATCH,
	NEVER,
	NOMEM,
	NULL,
	NULLABLE,
	OFFSET,
	OPAQUE,
	RETURN,
	RUNE,
	SIZE,
	STATIC,
	STR,
	STRUCT,
	SWITCH,
	TRUE,
	TYPE,
	U16,
	U32,
	U64,
	U8,
	UINT,
	UINTPTR,
	UNION,
	USE,
	VAARG,
	VAEND,
	VALIST,
	VASTART,
	VOID,
	YIELD,
	LAST_KEYWORD = YIELD,
	ARROW,
	BAND,
	BANDEQ,
	BNOT,
	BOR,
	BOREQ,
	BXOR,
	BXOREQ,
	COLON,
	COMMA,
	DIV,
	DIVEQ,
	DOT,
	DOUBLE_COLON,
	DOUBLE_DOT,
	ELLIPSIS,
	EQUAL,
	GT,
	GTEQ,
	LAND,
	LANDEQ,
	LBRACE,
	LBRACKET,
	LEQUAL,
	LESS,
	LESSEQ,
	LNOT,
	LOR,
	LOREQ,
	LPAREN,
	LSHIFT,
	LSHIFTEQ,
	LXOR,
	LXOREQ,
	MINUS,
	MINUSEQ,
	MODEQ,
	MODULO,
	NEQUAL,
	PLUS,
	PLUSEQ,
	QUESTION,
	RBRACE,
	RBRACKET,
	RPAREN,
	RSHIFT,
	RSHIFTEQ,
	SEMICOLON,
	TIMES,
	TIMESEQ,
	LAST_BTOK = TIMESEQ,
	LIT_U8,
	LIT_U16,
	LIT_U32,
	LIT_U64,
	LIT_UINT,
	LIT_SIZE,
	LIT_I8,
	LIT_I16,
	LIT_I32,
	LIT_I64,
	LIT_INT,
	LIT_ICONST,
	LIT_F32,
	LIT_F64,
	LIT_FCONST,
	LIT_RCONST,
	LIT_STR,
	LAST_LITERAL = LIT_STR,
	NAME,
	EOF,
};

A lexical token class.

type restore_point[permalink] [source]

type restore_point = struct {
	off: io::off,
	state: lexer,
};

A restore point for a lexer.

type token[permalink] [source]

type token = (ltok, value, location);

A single lexical token.

type value[permalink] [source]

type value = (str | rune | u64 | f64 | void);

A token value, used for tokens such as '1337' (an integer).

type lexer[permalink] [source]

type lexer = struct {
	in: *bufio::scanner,
	path: str,
	loc: (uint, uint, io::off),
	prevrloc: (uint, uint, io::off),
	// ltok::EOF when no token was unlexed
	un: token,
	prevunlocs: [2]((uint, uint, io::off), (uint, uint, io::off)),
	flags: flag,
	comment: str,
	require_int: bool,
	annotations: []([]str, *annotatefn, nullable *opaque),
};

type error[permalink] [source]

type error = !(io::error | syntax);

All possible lexer errors.

type syntax[permalink] [source]

type syntax = !(location, str);

A syntax error.

fn comment[permalink] [source]

fn comment(lex: *lexer) str;

Returns the current value of the comment buffer, or empty string if unset (or if flag::COMMENTS was not enabled for this lexer).

fn init[permalink] [source]

fn init(in: *bufio::scanner, path: str, flags: flag = flag::NONE) lexer;

Initializes a new lexer for the given bufio::scanner. The path is borrowed and must remain valid for the lifetime of the lexer.

fn lex[permalink] [source]

fn lex(lex: *lexer) (token | error);

Returns the next token from the lexer.

fn mkloc[permalink] [source]

fn mkloc(lex: *lexer) location;

The lexer's current location.

fn prevloc[permalink] [source]

fn prevloc(lex: *lexer) location;

The location of the previous rune.

fn register_annotation[permalink] [source]

fn register_annotation(lex: *lexer, id: []str, cb: *annotatefn, user: nullable *opaque) void;

Registers an annotation callback with this lexer for the given identifier. The id is borrowed for the lifetime of the lexer. Unregistered annotations are silently discarded by the lexer.

fn restore[permalink] [source]

fn restore(lex: *lexer, rp: *restore_point) (void | io::error);

Restores a lexer to a state previously recorded with save.

fn save[permalink] [source]

fn save(lex: *lexer) (restore_point | io::error);

Saves the state of a lexer, to be restored later with restore. The underlying I/O source must be seekable.

fn strerror[permalink] [source]

fn strerror(err: error) const str;

Returns a human-friendly string for a given error. The result is statically allocated.

fn syntaxerr[permalink] [source]

fn syntaxerr(loc: location, why: str) error;

Builds a lexer syntax error from a location and a reason string.

fn tokstr[permalink] [source]

fn tokstr(tok: token) const str;

Converts a token to its string representation.

fn unlex[permalink] [source]

fn unlex(lex: *lexer, tok: token) void;

Unlex a single token. The next call to lex will return this token. Only one unlex is supported at a time; you must call lex before calling unlex again.