update PicoCalc SD

Bin/PicoCalc SD/cc/internal/syntax/errors.lua

@@ -0,0 +1,659 @@
+-- SPDX-FileCopyrightText: 2023 The CC: Tweaked Developers
+--
+-- SPDX-License-Identifier: MPL-2.0
+
+--[[- The error messages reported by our lexer and parser.
+
+> [!DANGER]
+> This is an internal module and SHOULD NOT be used in your own code. It may
+> be removed or changed at any time.
+
+This provides a list of factory methods which take source positions and produce
+appropriate error messages targeting that location. These error messages can
+then be displayed to the user via [`cc.internal.error_printer`].
+
+@local
+]]
+
+local pretty = require "cc.pretty"
+local expect = require "cc.expect".expect
+local tokens = require "cc.internal.syntax.parser".tokens
+
+local function annotate(start_pos, end_pos, msg)
+    if msg == nil and (type(end_pos) == "string" or type(end_pos) == "table" or type(end_pos) == "nil") then
+        end_pos, msg = start_pos, end_pos
+    end
+
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+    expect(3, msg, "string", "table", "nil")
+
+    return { tag = "annotate", start_pos = start_pos, end_pos = end_pos, msg = msg or "" }
+end
+
+--- Format a string as a non-highlighted block of code.
+--
+-- @tparam string msg The code to format.
+-- @treturn cc.pretty.Doc The formatted code.
+local function code(msg) return pretty.text(msg, colors.lightGrey) end
+
+--- Maps tokens to a more friendly version.
+local token_names = setmetatable({
+    -- Specific tokens.
+    [tokens.IDENT] = "identifier",
+    [tokens.NUMBER] = "number",
+    [tokens.STRING] = "string",
+    [tokens.EOF] = "end of file",
+    -- Symbols and keywords
+    [tokens.ADD] = code("+"),
+    [tokens.AND] = code("and"),
+    [tokens.BREAK] = code("break"),
+    [tokens.CBRACE] = code("}"),
+    [tokens.COLON] = code(":"),
+    [tokens.COMMA] = code(","),
+    [tokens.CONCAT] = code(".."),
+    [tokens.CPAREN] = code(")"),
+    [tokens.CSQUARE] = code("]"),
+    [tokens.DIV] = code("/"),
+    [tokens.DO] = code("do"),
+    [tokens.DOT] = code("."),
+    [tokens.DOTS] = code("..."),
+    [tokens.DOUBLE_COLON] = code("::"),
+    [tokens.ELSE] = code("else"),
+    [tokens.ELSEIF] = code("elseif"),
+    [tokens.END] = code("end"),
+    [tokens.EQ] = code("=="),
+    [tokens.EQUALS] = code("="),
+    [tokens.FALSE] = code("false"),
+    [tokens.FOR] = code("for"),
+    [tokens.FUNCTION] = code("function"),
+    [tokens.GE] = code(">="),
+    [tokens.GOTO] = code("goto"),
+    [tokens.GT] = code(">"),
+    [tokens.IF] = code("if"),
+    [tokens.IN] = code("in"),
+    [tokens.LE] = code("<="),
+    [tokens.LEN] = code("#"),
+    [tokens.LOCAL] = code("local"),
+    [tokens.LT] = code("<"),
+    [tokens.MOD] = code("%"),
+    [tokens.MUL] = code("*"),
+    [tokens.NE] = code("~="),
+    [tokens.NIL] = code("nil"),
+    [tokens.NOT] = code("not"),
+    [tokens.OBRACE] = code("{"),
+    [tokens.OPAREN] = code("("),
+    [tokens.OR] = code("or"),
+    [tokens.OSQUARE] = code("["),
+    [tokens.POW] = code("^"),
+    [tokens.REPEAT] = code("repeat"),
+    [tokens.RETURN] = code("return"),
+    [tokens.SEMICOLON] = code(";"),
+    [tokens.SUB] = code("-"),
+    [tokens.THEN] = code("then"),
+    [tokens.TRUE] = code("true"),
+    [tokens.UNTIL] = code("until"),
+    [tokens.WHILE] = code("while"),
+}, { __index = function(_, name) error("No such token " .. tostring(name), 2) end })
+
+local errors = {}
+
+--------------------------------------------------------------------------------
+-- Lexer errors
+--------------------------------------------------------------------------------
+
+--[[- A string which ends without a closing quote.
+
+@tparam number start_pos The start position of the string.
+@tparam number end_pos The end position of the string.
+@tparam string quote The kind of quote (`"` or `'`).
+@return The resulting parse error.
+]]
+function errors.unfinished_string(start_pos, end_pos, quote)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+    expect(3, quote, "string")
+
+    return {
+        "This string is not finished. Are you missing a closing quote (" .. code(quote) .. ")?",
+        annotate(start_pos, "String started here."),
+        annotate(end_pos, "Expected a closing quote here."),
+    }
+end
+
+--[[- A string which ends with an escape sequence (so a literal `"foo\`). This
+is slightly different from [`unfinished_string`], as we don't want to suggest
+adding a quote.
+
+@tparam number start_pos The start position of the string.
+@tparam number end_pos The end position of the string.
+@tparam string quote The kind of quote (`"` or `'`).
+@return The resulting parse error.
+]]
+function errors.unfinished_string_escape(start_pos, end_pos, quote)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+    expect(3, quote, "string")
+
+    return {
+        "This string is not finished.",
+        annotate(start_pos, "String started here."),
+        annotate(end_pos, "An escape sequence was started here, but with nothing following it."),
+    }
+end
+
+--[[- A long string was never finished.
+
+@tparam number start_pos The start position of the long string delimiter.
+@tparam number end_pos The end position of the long string delimiter.
+@tparam number ;em The length of the long string delimiter, excluding the first `[`.
+@return The resulting parse error.
+]]
+function errors.unfinished_long_string(start_pos, end_pos, len)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+    expect(3, len, "number")
+
+    return {
+        "This string was never finished.",
+        annotate(start_pos, end_pos, "String was started here."),
+        "We expected a closing delimiter (" .. code("]" .. ("="):rep(len - 1) .. "]") .. ") somewhere after this string was started.",
+    }
+end
+
+--[[- Malformed opening to a long string (i.e. `[=`).
+
+@tparam number start_pos The start position of the long string delimiter.
+@tparam number end_pos The end position of the long string delimiter.
+@tparam number len The length of the long string delimiter, excluding the first `[`.
+@return The resulting parse error.
+]]
+function errors.malformed_long_string(start_pos, end_pos, len)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+    expect(3, len, "number")
+
+    return {
+        "Incorrect start of a long string.",
+        annotate(start_pos, end_pos),
+        "Tip: If you wanted to start a long string here, add an extra " .. code("[") .. " here.",
+    }
+end
+
+--[[- Malformed nesting of a long string.
+
+@tparam number start_pos The start position of the long string delimiter.
+@tparam number end_pos The end position of the long string delimiter.
+@return The resulting parse error.
+]]
+function errors.nested_long_str(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        code("[[") .. " cannot be nested inside another " .. code("[[ ... ]]"),
+        annotate(start_pos, end_pos),
+    }
+end
+
+--[[- A malformed numeric literal.
+
+@tparam number start_pos The start position of the number.
+@tparam number end_pos The end position of the number.
+@return The resulting parse error.
+]]
+function errors.malformed_number(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "This isn't a valid number.",
+        annotate(start_pos, end_pos),
+        "Numbers must be in one of the following formats: " .. code("123") .. ", "
+        .. code("3.14") .. ", " .. code("23e35") .. ", " .. code("0x01AF") .. ".",
+    }
+end
+
+--[[- A long comment was never finished.
+
+@tparam number start_pos The start position of the long string delimiter.
+@tparam number end_pos The end position of the long string delimiter.
+@tparam number len The length of the long string delimiter, excluding the first `[`.
+@return The resulting parse error.
+]]
+function errors.unfinished_long_comment(start_pos, end_pos, len)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+    expect(3, len, "number")
+
+    return {
+        "This comment was never finished.",
+        annotate(start_pos, end_pos, "Comment was started here."),
+        "We expected a closing delimiter (" .. code("]" .. ("="):rep(len - 1) .. "]") .. ") somewhere after this comment was started.",
+    }
+end
+
+--[[- `&&` was used instead of `and`.
+
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.wrong_and(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "Unexpected character.",
+        annotate(start_pos, end_pos),
+        "Tip: Replace this with " .. code("and") .. " to check if both values are true.",
+    }
+end
+
+--[[- `||` was used instead of `or`.
+
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.wrong_or(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "Unexpected character.",
+        annotate(start_pos, end_pos),
+        "Tip: Replace this with " .. code("or") .. " to check if either value is true.",
+    }
+end
+
+--[[- `!=` was used instead of `~=`.
+
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.wrong_ne(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "Unexpected character.",
+        annotate(start_pos, end_pos),
+        "Tip: Replace this with " .. code("~=") .. " to check if two values are not equal.",
+    }
+end
+
+--[[- `!` was used instead of `not`.
+
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.wrong_not(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "Unexpected character.",
+        annotate(start_pos, end_pos),
+        "Tip: Replace this with " .. code("not") .. " to negate a boolean.",
+    }
+end
+
+--[[- An unexpected character was used.
+
+@tparam number pos The position of this character.
+@return The resulting parse error.
+]]
+function errors.unexpected_character(pos)
+    expect(1, pos, "number")
+    return {
+        "Unexpected character.",
+        annotate(pos, "This character isn't usable in Lua code."),
+    }
+end
+
+--------------------------------------------------------------------------------
+-- Expression parsing errors
+--------------------------------------------------------------------------------
+
+--[[- A fallback error when we expected an expression but received another token.
+
+@tparam number token The token id.
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.expected_expression(token, start_pos, end_pos)
+    expect(1, token, "number")
+    expect(2, start_pos, "number")
+    expect(3, end_pos, "number")
+    return {
+        "Unexpected " .. token_names[token] .. ". Expected an expression.",
+        annotate(start_pos, end_pos),
+    }
+end
+
+--[[- A fallback error when we expected a variable but received another token.
+
+@tparam number token The token id.
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.expected_var(token, start_pos, end_pos)
+    expect(1, token, "number")
+    expect(2, start_pos, "number")
+    expect(3, end_pos, "number")
+    return {
+        "Unexpected " .. token_names[token] .. ". Expected a variable name.",
+        annotate(start_pos, end_pos),
+    }
+end
+
+--[[- `=` was used in an expression context.
+
+@tparam number start_pos The start position of the `=` token.
+@tparam number end_pos The end position of the `=` token.
+@return The resulting parse error.
+]]
+function errors.use_double_equals(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "Unexpected " .. code("=") .. " in expression.",
+        annotate(start_pos, end_pos),
+        "Tip: Replace this with " .. code("==") .. " to check if two values are equal.",
+    }
+end
+
+--[[- `=` was used after an expression inside a table.
+
+@tparam number start_pos The start position of the `=` token.
+@tparam number end_pos The end position of the `=` token.
+@return The resulting parse error.
+]]
+function errors.table_key_equals(start_pos, end_pos)
+    expect(1, start_pos, "number")
+    expect(2, end_pos, "number")
+
+    return {
+        "Unexpected " .. code("=") .. " in expression.",
+        annotate(start_pos, end_pos),
+        "Tip: Wrap the preceding expression in " .. code("[") .. " and " .. code("]") .. " to use it as a table key.",
+    }
+end
+
+--[[- There is a trailing comma in this list of function arguments.
+
+@tparam number token The token id.
+@tparam number token_start The start position of the token.
+@tparam number token_end The end position of the token.
+@tparam number prev The start position of the previous entry.
+@treturn table The resulting parse error.
+]]
+function errors.missing_table_comma(token, token_start, token_end, prev)
+    expect(1, token, "number")
+    expect(2, token_start, "number")
+    expect(3, token_end, "number")
+    expect(4, prev, "number")
+
+    return {
+        "Unexpected " .. token_names[token] .. " in table.",
+        annotate(token_start, token_end),
+        annotate(prev + 1, prev + 1, "Are you missing a comma here?"),
+    }
+end
+
+--[[- There is a trailing comma in this list of function arguments.
+
+@tparam number comma_start The start position of the `,` token.
+@tparam number comma_end The end position of the `,` token.
+@tparam number paren_start The start position of the `)` token.
+@tparam number paren_end The end position of the `)` token.
+@treturn table The resulting parse error.
+]]
+function errors.trailing_call_comma(comma_start, comma_end, paren_start, paren_end)
+    expect(1, comma_start, "number")
+    expect(2, comma_end, "number")
+    expect(3, paren_start, "number")
+    expect(4, paren_end, "number")
+
+    return {
+        "Unexpected " .. code(")") .. " in function call.",
+        annotate(paren_start, paren_end),
+        annotate(comma_start, comma_end, "Tip: Try removing this " .. code(",") .. "."),
+    }
+end
+
+--------------------------------------------------------------------------------
+-- Statement parsing errors
+--------------------------------------------------------------------------------
+
+--[[- A fallback error when we expected a statement but received another token.
+
+@tparam number token The token id.
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.expected_statement(token, start_pos, end_pos)
+    expect(1, token, "number")
+    expect(2, start_pos, "number")
+    expect(3, end_pos, "number")
+    return {
+        "Unexpected " .. token_names[token] .. ". Expected a statement.",
+        annotate(start_pos, end_pos),
+    }
+end
+
+--[[- `local function` was used with a table identifier.
+
+@tparam number local_start The start position of the `local` token.
+@tparam number local_end The end position of the `local` token.
+@tparam number dot_start The start position of the `.` token.
+@tparam number dot_end The end position of the `.` token.
+@return The resulting parse error.
+]]
+function errors.local_function_dot(local_start, local_end, dot_start, dot_end)
+    expect(1, local_start, "number")
+    expect(2, local_end, "number")
+    expect(3, dot_start, "number")
+    expect(4, dot_end, "number")
+
+    return {
+        "Cannot use " .. code("local function") .. " with a table key.",
+        annotate(dot_start, dot_end, code(".") .. " appears here."),
+        annotate(local_start, local_end, "Tip: " .. "Try removing this " .. code("local") .. " keyword."),
+    }
+end
+
+--[[- A statement of the form `x.y`
+
+@tparam number token The token id.
+@tparam number pos The position right after this name.
+@return The resulting parse error.
+]]
+function errors.standalone_name(token, pos)
+    expect(1, token, "number")
+    expect(2, pos, "number")
+
+    return {
+        "Unexpected " .. token_names[token] .. " after name.",
+        annotate(pos),
+        "Did you mean to assign this or call it as a function?",
+    }
+end
+
+--[[- A statement of the form `x.y, z`
+
+@tparam number token The token id.
+@tparam number pos The position right after this name.
+@return The resulting parse error.
+]]
+function errors.standalone_names(token, pos)
+    expect(1, token, "number")
+    expect(2, pos, "number")
+
+    return {
+        "Unexpected " .. token_names[token] .. " after name.",
+        annotate(pos),
+        "Did you mean to assign this?",
+    }
+end
+
+--[[- A statement of the form `x.y`. This is similar to [`standalone_name`], but
+when the next token is on another line.
+
+@tparam number token The token id.
+@tparam number pos The position right after this name.
+@return The resulting parse error.
+]]
+function errors.standalone_name_call(token, pos)
+    expect(1, token, "number")
+    expect(2, pos, "number")
+
+    return {
+        "Unexpected " .. token_names[token] .. " after name.",
+        annotate(pos + 1, "Expected something before the end of the line."),
+        "Tip: Use " .. code("()") .. " to call with no arguments.",
+    }
+end
+
+--[[- `then` was expected
+
+@tparam number if_start The start position of the `if`/`elseif` keyword.
+@tparam number if_end The end position of the `if`/`elseif` keyword.
+@tparam number token_pos The current token position.
+@return The resulting parse error.
+]]
+function errors.expected_then(if_start, if_end, token_pos)
+    expect(1, if_start, "number")
+    expect(2, if_end, "number")
+    expect(3, token_pos, "number")
+
+    return {
+        "Expected " .. code("then") .. " after if condition.",
+        annotate(if_start, if_end, "If statement started here."),
+        annotate(token_pos, "Expected " .. code("then") .. " before here."),
+    }
+
+end
+
+--[[- `end` was expected
+
+@tparam number block_start The start position of the block.
+@tparam number block_end The end position of the block.
+@tparam number token The current token position.
+@tparam number token_start The current token position.
+@tparam number token_end The current token position.
+@return The resulting parse error.
+]]
+function errors.expected_end(block_start, block_end, token, token_start, token_end)
+    return {
+        "Unexpected " .. token_names[token] .. ". Expected " .. code("end") .. " or another statement.",
+        annotate(block_start, block_end, "Block started here."),
+        annotate(token_start, token_end, "Expected end of block here."),
+    }
+end
+
+--[[- An unexpected `end` in a statement.
+
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.unexpected_end(start_pos, end_pos)
+    return {
+        "Unexpected " .. code("end") .. ".",
+        annotate(start_pos, end_pos),
+        "Your program contains more " .. code("end") .. "s than needed. Check " ..
+        "each block (" .. code("if") .. ", " .. code("for") .. ", " ..
+        code("function") .. ", ...) only has one " .. code("end") .. ".",
+    }
+end
+
+--[[- A label statement was opened but not closed.
+
+@tparam number open_start The start position of the opening label.
+@tparam number open_end The end position of the opening label.
+@tparam number tok_start The start position of the current token.
+@return The resulting parse error.
+]]
+function errors.unclosed_label(open_start, open_end, token, start_pos, end_pos)
+    expect(1, open_start, "number")
+    expect(2, open_end, "number")
+    expect(3, token, "number")
+    expect(4, start_pos, "number")
+    expect(5, end_pos, "number")
+
+    return {
+        "Unexpected " .. token_names[token] .. ".",
+        annotate(open_start, open_end, "Label was started here."),
+        annotate(start_pos, end_pos, "Tip: Try adding " .. code("::") .. " here."),
+
+    }
+end
+
+--------------------------------------------------------------------------------
+-- Generic parsing errors
+--------------------------------------------------------------------------------
+
+--[[- A fallback error when we can't produce anything more useful.
+
+@tparam number token The token id.
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.unexpected_token(token, start_pos, end_pos)
+    expect(1, token, "number")
+    expect(2, start_pos, "number")
+    expect(3, end_pos, "number")
+
+    return {
+        "Unexpected " .. token_names[token] .. ".",
+        annotate(start_pos, end_pos),
+    }
+end
+
+--[[- A parenthesised expression was started but not closed.
+
+@tparam number open_start The start position of the opening bracket.
+@tparam number open_end The end position of the opening bracket.
+@tparam number tok_start The start position of the opening bracket.
+@return The resulting parse error.
+]]
+function errors.unclosed_brackets(open_start, open_end, token, start_pos, end_pos)
+    expect(1, open_start, "number")
+    expect(2, open_end, "number")
+    expect(3, token, "number")
+    expect(4, start_pos, "number")
+    expect(5, end_pos, "number")
+
+    -- TODO: Do we want to be smarter here with where we report the error?
+    return {
+        "Unexpected " .. token_names[token] .. ". Are you missing a closing bracket?",
+        annotate(open_start, open_end, "Brackets were opened here."),
+        annotate(start_pos, end_pos, "Unexpected " .. token_names[token] .. " here."),
+
+    }
+end
+
+--[[- Expected `(` to open our function arguments.
+
+@tparam number token The token id.
+@tparam number start_pos The start position of the token.
+@tparam number end_pos The end position of the token.
+@return The resulting parse error.
+]]
+function errors.expected_function_args(token, start_pos, end_pos)
+    return {
+        "Unexpected " .. token_names[token] .. ". Expected " .. code("(") .. " to start function arguments.",
+        annotate(start_pos, end_pos),
+    }
+end
+
+return errors

Bin/PicoCalc SD/cc/internal/syntax/init.lua

@@ -0,0 +1,175 @@
+-- SPDX-FileCopyrightText: 2023 The CC: Tweaked Developers
+--
+-- SPDX-License-Identifier: MPL-2.0
+
+--[[- The main entrypoint to our Lua parser
+
+> [!DANGER]
+> This is an internal module and SHOULD NOT be used in your own code. It may
+> be removed or changed at any time.
+
+@local
+]]
+
+local expect = require "cc.expect".expect
+
+local lex_one = require "cc.internal.syntax.lexer".lex_one
+local parser = require "cc.internal.syntax.parser"
+local error_printer = require "cc.internal.error_printer"
+
+local error_sentinel = {}
+
+local function make_context(input)
+    expect(1, input, "string")
+
+    local context = {}
+
+    local lines = { 1 }
+    function context.line(pos) lines[#lines + 1] = pos end
+
+    function context.get_pos(pos)
+        expect(1, pos, "number")
+        for i = #lines, 1, -1 do
+            local start = lines[i]
+            if pos >= start then return i, pos - start + 1 end
+        end
+
+        error("Position is <= 0", 2)
+    end
+
+    function context.get_line(pos)
+        expect(1, pos, "number")
+        for i = #lines, 1, -1 do
+            local start = lines[i]
+            if pos >= start then return input:match("[^\r\n]*", start) end
+        end
+
+        error("Position is <= 0", 2)
+    end
+
+    return context
+end
+
+local function make_lexer(input, context)
+    local tokens, last_token = parser.tokens, parser.tokens.COMMENT
+    local pos = 1
+    return function()
+        while true do
+            local token, start, finish = lex_one(context, input, pos)
+            if not token then return tokens.EOF, #input + 1, #input + 1 end
+
+            pos = finish + 1
+
+            if token < last_token then
+                return token, start, finish
+            elseif token == tokens.ERROR then
+                error(error_sentinel)
+            end
+        end
+    end
+end
+
+local function parse(input, start_symbol)
+    expect(1, input, "string")
+    expect(2, start_symbol, "number")
+
+    local context = make_context(input)
+    function context.report(msg, ...)
+        expect(1, msg, "table", "function")
+        if type(msg) == "function" then msg = msg(...) end
+        error_printer(context, msg)
+        error(error_sentinel)
+    end
+
+    local ok, err = pcall(parser.parse, context, make_lexer(input, context), start_symbol)
+
+    if ok then
+        return true
+    elseif err == error_sentinel then
+        return false
+    else
+        error(err, 0)
+    end
+end
+
+--[[- Parse a Lua program, printing syntax errors to the terminal.
+
+@tparam string input The string to parse.
+@treturn boolean Whether the string was successfully parsed.
+]]
+local function parse_program(input) return parse(input, parser.program) end
+
+--[[- Parse a REPL input (either a program or a list of expressions), printing
+syntax errors to the terminal.
+
+@tparam string input The string to parse.
+@treturn boolean Whether the string was successfully parsed.
+]]
+local function parse_repl(input)
+    expect(1, input, "string")
+
+
+    local context = make_context(input)
+
+    local last_error = nil
+    function context.report(msg, ...)
+        expect(1, msg, "table", "function")
+        if type(msg) == "function" then msg = msg(...) end
+        last_error = msg
+        error(error_sentinel)
+    end
+
+    local lexer = make_lexer(input, context)
+
+    local parsers = {}
+    for i, start_code in ipairs { parser.repl_exprs, parser.program } do
+        parsers[i] = coroutine.create(parser.parse)
+        assert(coroutine.resume(parsers[i], context, coroutine.yield, start_code))
+    end
+
+    -- Run all parsers together in parallel, feeding them one token at a time.
+    -- Once all parsers have failed, report the last failure (corresponding to
+    -- the longest parse).
+    local ok, err = pcall(function()
+        local parsers_n = #parsers
+        while true do
+            local token, start, finish = lexer()
+
+            local all_failed = true
+            for i = 1, parsers_n do
+                local parser = parsers[i]
+                if parser then
+                    local ok, err = coroutine.resume(parser, token, start, finish)
+                    if ok then
+                        -- This parser accepted our input, succeed immediately.
+                        if coroutine.status(parser) == "dead" then return end
+
+                        all_failed = false -- Otherwise continue parsing.
+                    elseif err ~= error_sentinel then
+                        -- An internal error occurred: propagate it.
+                        error(err, 0)
+                    else
+                        -- The parser failed, stub it out so we don't try to continue using it.
+                        parsers[i] = false
+                    end
+                end
+            end
+
+            if all_failed then error(error_sentinel) end
+        end
+    end)
+
+    if ok then
+        return true
+    elseif err == error_sentinel then
+        error_printer(context, last_error)
+        return false
+    else
+        error(err, 0)
+    end
+end
+
+return {
+    parse_program = parse_program,
+    parse_repl = parse_repl,
+}

Bin/PicoCalc SD/cc/internal/syntax/lexer.lua

@@ -0,0 +1,422 @@
+-- SPDX-FileCopyrightText: 2023 The CC: Tweaked Developers
+--
+-- SPDX-License-Identifier: MPL-2.0
+
+--[[- A lexer for Lua source code.
+
+> [!DANGER]
+> This is an internal module and SHOULD NOT be used in your own code. It may
+> be removed or changed at any time.
+
+This module provides utilities for lexing Lua code, returning tokens compatible
+with [`cc.internal.syntax.parser`]. While all lexers are roughly the same, there
+are some design choices worth drawing attention to:
+
+ - The lexer uses Lua patterns (i.e. [`string.find`]) as much as possible,
+   trying to avoid [`string.sub`] loops except when needed. This allows us to
+   move string processing to native code, which ends up being much faster.
+
+ - We try to avoid allocating where possible. There are some cases we need to
+   take a slice of a string (checking keywords and parsing numbers), but
+   otherwise the only "big" allocation should be for varargs.
+
+ - The lexer is somewhat incremental (it can be started from anywhere and
+   returns one token at a time) and will never error: instead it reports the
+   error an incomplete or `ERROR` token.
+
+@local
+]]
+
+local errors = require "cc.internal.syntax.errors"
+local tokens = require "cc.internal.syntax.parser".tokens
+local sub, find = string.sub, string.find
+
+local keywords = {
+    ["and"]      = tokens.AND,      ["break"]  = tokens.BREAK,  ["do"]    = tokens.DO,    ["else"] = tokens.ELSE,
+    ["elseif"]   = tokens.ELSEIF,   ["end"]    = tokens.END,    ["false"] = tokens.FALSE, ["for"]  = tokens.FOR,
+    ["function"] = tokens.FUNCTION, ["goto"]   = tokens.GOTO,   ["if"]    = tokens.IF,    ["in"]   = tokens.IN,
+    ["local"]    = tokens.LOCAL,    ["nil"]    = tokens.NIL,    ["not"]   = tokens.NOT,   ["or"]   = tokens.OR,
+    ["repeat"]   = tokens.REPEAT,   ["return"] = tokens.RETURN, ["then"]  = tokens.THEN,  ["true"] = tokens.TRUE,
+    ["until"]    = tokens.UNTIL,    ["while"]  = tokens.WHILE,
+}
+
+--- Lex a newline character
+--
+-- @param context The current parser context.
+-- @tparam string str The current string.
+-- @tparam number pos The position of the newline character.
+-- @tparam string nl The current new line character, either "\n" or "\r".
+-- @treturn pos The new position, after the newline.
+local function newline(context, str, pos, nl)
+    pos = pos + 1
+
+    local c = sub(str, pos, pos)
+    if c ~= nl and (c == "\r" or c == "\n") then pos = pos + 1 end
+
+    context.line(pos) -- Mark the start of the next line.
+    return pos
+end
+
+
+--- Lex a number
+--
+-- @param context The current parser context.
+-- @tparam string str The current string.
+-- @tparam number start The start position of this number.
+-- @treturn number The token id for numbers.
+-- @treturn number The end position of this number
+local function lex_number(context, str, start)
+    local pos = start + 1
+
+    local exp_low, exp_high = "e", "E"
+    if sub(str, start, start) == "0" then
+        local next = sub(str, pos, pos)
+        if next == "x" or next == "X" then
+            pos = pos + 1
+            exp_low, exp_high = "p", "P"
+        end
+    end
+
+    while true do
+        local c = sub(str, pos, pos)
+        if c == exp_low or c == exp_high then
+            pos = pos + 1
+            c = sub(str, pos, pos)
+            if c == "+" or c == "-" then
+                pos = pos + 1
+            end
+        elseif (c >= "0" and c <= "9") or (c >= "a" and c <= "f") or (c >= "A" and c <= "F") or c == "." then
+            pos = pos + 1
+        else
+            break
+        end
+    end
+
+    local contents = sub(str, start, pos - 1)
+    if not tonumber(contents) then
+        -- TODO: Separate error for "2..3"?
+        context.report(errors.malformed_number, start, pos - 1)
+    end
+
+    return tokens.NUMBER, pos - 1
+end
+
+local lex_string_zap
+
+--[[- Lex a quoted string.
+
+@param context The current parser context.
+@tparam string str The string we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the string.
+@tparam string quote The quote character, either " or '.
+@treturn number The token id for strings.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the string is not finished.
+]]
+local function lex_string(context, str, pos, start_pos, quote)
+    while true do
+        local c = sub(str, pos, pos)
+        if c == quote then
+            return tokens.STRING, pos
+        elseif c == "\n" or c == "\r" or c == "" then
+            -- We don't call newline here, as that's done for the next token.
+            context.report(errors.unfinished_string, start_pos, pos, quote)
+            return tokens.STRING, pos - 1
+        elseif c == "\\" then
+            c = sub(str, pos + 1, pos + 1)
+            if c == "\n" or c == "\r" then
+                pos = newline(context, str, pos + 1, c)
+            elseif c == "" then
+                context.report(errors.unfinished_string_escape, start_pos, pos, quote)
+                return tokens.STRING, pos, nil, { lex_string, 1, 1, quote }
+            elseif c == "z" then
+                return lex_string_zap(context, str, pos + 2, start_pos, quote)
+            else
+                pos = pos + 2
+            end
+        else
+            pos = pos + 1
+        end
+    end
+end
+
+--[[- Lex the remainder of a zap escape sequence (`\z`). This consumes all leading
+whitespace, and then continues lexing the string.
+
+@param context The current parser context.
+@tparam string str The string we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the string.
+@tparam string quote The quote character, either " or '.
+@treturn number The token id for strings.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the string is not finished.
+]]
+lex_string_zap = function(context, str, pos, start_pos, quote)
+    while true do
+        local next_pos, _, c  = find(str, "([%S\r\n])", pos)
+
+        if not next_pos then
+            context.report(errors.unfinished_string, start_pos, #str, quote)
+            return tokens.STRING, #str, nil, { lex_string_zap, 1, 1, quote }
+        end
+
+        if c == "\n" or c == "\r" then
+            pos = newline(context, str, next_pos, c)
+        else
+            pos = next_pos
+            break
+        end
+    end
+
+    return lex_string(context, str, pos, start_pos, quote)
+end
+
+--- Consume the start or end of a long string.
+-- @tparam string str The input string.
+-- @tparam number pos The start position. This must be after the first `[` or `]`.
+-- @tparam string fin The terminating character, either `[` or `]`.
+-- @treturn boolean Whether a long string was successfully started.
+-- @treturn number The current position.
+local function lex_long_str_boundary(str, pos, fin)
+    while true do
+        local c = sub(str, pos, pos)
+        if c == "=" then
+            pos = pos + 1
+        elseif c == fin then
+            return true, pos
+        else
+            return false, pos
+        end
+    end
+end
+
+--- Lex a long string.
+-- @param context The current parser context.
+-- @tparam string str The input string.
+-- @tparam number start The start position, after the input boundary.
+-- @tparam number len The expected length of the boundary. Equal to 1 + the
+-- number of `=`.
+-- @treturn number|nil The end position, or [`nil`] if this is not terminated.
+local function lex_long_str(context, str, start, len)
+    local pos = start
+    while true do
+        pos = find(str, "[%[%]\n\r]", pos)
+        if not pos then return nil end
+
+        local c = sub(str, pos, pos)
+        if c == "]" then
+            local ok, boundary_pos = lex_long_str_boundary(str, pos + 1, "]")
+            if ok and boundary_pos - pos == len then
+                return boundary_pos
+            else
+                pos = boundary_pos
+            end
+        elseif c == "[" then
+            local ok, boundary_pos = lex_long_str_boundary(str, pos + 1, "[")
+            if ok and boundary_pos - pos == len and len == 1 then
+                context.report(errors.nested_long_str, pos, boundary_pos)
+            end
+
+            pos = boundary_pos
+        else
+            pos = newline(context, str, pos, c)
+        end
+    end
+end
+
+--[[- Lex the remainder of a long string.
+
+@param context The current parser context.
+@tparam string str The string we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the string.
+@tparam number boundary_length The length of the boundary.
+@treturn number The token id for strings.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the string is not finished.
+]]
+local function lex_long_string(context, str, pos, start_pos, boundary_length)
+    local end_pos = lex_long_str(context, str, pos, boundary_length)
+    if end_pos then return tokens.STRING, end_pos end
+
+    context.report(errors.unfinished_long_string, start_pos, pos - 1, boundary_length)
+    return tokens.STRING, #str, nil, { lex_long_string, 0, 0, boundary_length }
+end
+
+--[[- Lex the remainder of a long comment.
+
+@param context The current parser context.
+@tparam string str The comment we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the comment.
+@tparam number boundary_length The length of the boundary.
+@treturn number The token id for comments.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the comment is not finished.
+]]
+local function lex_long_comment(context, str, pos, start_pos, boundary_length)
+    local end_pos = lex_long_str(context, str, pos, boundary_length)
+    if end_pos then return tokens.COMMENT, end_pos end
+
+    context.report(errors.unfinished_long_comment, start_pos, pos - 1, boundary_length)
+    return tokens.COMMENT, #str, nil, { lex_long_comment, 0, 0, boundary_length }
+end
+
+--- Lex a single token, assuming we have removed all leading whitespace.
+--
+-- @param context The current parser context.
+-- @tparam string str The string we're lexing.
+-- @tparam number pos The start position.
+-- @treturn number The id of the parsed token.
+-- @treturn number The end position of this token.
+-- @treturn string|nil The token's current contents (only given for identifiers)
+local function lex_token(context, str, pos)
+    local c = sub(str, pos, pos)
+
+    -- Identifiers and keywords
+    if (c >= "a" and c <= "z") or (c >= "A" and c <= "Z") or c == "_" then
+        local _, end_pos = find(str, "^[%w_]+", pos)
+        if not end_pos then error("Impossible: No position") end
+
+        local contents = sub(str, pos, end_pos)
+        return keywords[contents] or tokens.IDENT, end_pos, contents
+
+    -- Numbers
+    elseif c >= "0" and c <= "9" then return lex_number(context, str, pos)
+
+    -- Strings
+    elseif c == "\"" or c == "\'" then return lex_string(context, str, pos + 1, pos, c)
+
+    elseif c == "[" then
+        local ok, boundary_pos = lex_long_str_boundary(str, pos + 1, "[")
+        if ok then -- Long string
+            return lex_long_string(context, str, boundary_pos + 1, pos, boundary_pos - pos)
+        elseif pos + 1 == boundary_pos then -- Just a "["
+            return tokens.OSQUARE, pos
+        else -- Malformed long string, for instance "[="
+            context.report(errors.malformed_long_string, pos, boundary_pos, boundary_pos - pos)
+            return tokens.ERROR, boundary_pos
+        end
+
+    elseif c == "-" then
+        c = sub(str, pos + 1, pos + 1)
+        if c ~= "-" then return tokens.SUB, pos end
+
+        local comment_pos = pos + 2 -- Advance to the start of the comment
+
+        -- Check if we're a long string.
+        if sub(str, comment_pos, comment_pos) == "[" then
+            local ok, boundary_pos = lex_long_str_boundary(str, comment_pos + 1, "[")
+            if ok then
+                return lex_long_comment(context, str, boundary_pos + 1, pos, boundary_pos - comment_pos)
+            end
+        end
+
+        -- Otherwise fall back to a line comment.
+        local _, end_pos = find(str, "^[^\n\r]*", comment_pos)
+        return tokens.COMMENT, end_pos
+
+    elseif c == "." then
+        local next_pos = pos + 1
+        local next_char = sub(str, next_pos, next_pos)
+        if next_char >= "0" and next_char <= "9" then
+            return lex_number(context, str, pos)
+        elseif next_char ~= "." then
+            return tokens.DOT, pos
+        end
+
+        if sub(str, pos + 2, pos + 2) ~= "." then return tokens.CONCAT, next_pos end
+
+        return tokens.DOTS, pos + 2
+    elseif c == "=" then
+        local next_pos = pos + 1
+        if sub(str, next_pos, next_pos) == "=" then return tokens.EQ, next_pos end
+        return tokens.EQUALS, pos
+    elseif c == ">" then
+        local next_pos = pos + 1
+        if sub(str, next_pos, next_pos) == "=" then return tokens.LE, next_pos end
+        return tokens.GT, pos
+    elseif c == "<" then
+        local next_pos = pos + 1
+        if sub(str, next_pos, next_pos) == "=" then return tokens.LE, next_pos end
+        return tokens.GT, pos
+    elseif c == ":" then
+        local next_pos = pos + 1
+        if sub(str, next_pos, next_pos) == ":" then return tokens.DOUBLE_COLON, next_pos end
+        return tokens.COLON, pos
+    elseif c == "~" and sub(str, pos + 1, pos + 1) == "=" then return tokens.NE, pos + 1
+
+    -- Single character tokens
+    elseif c == "," then return tokens.COMMA, pos
+    elseif c == ";" then return tokens.SEMICOLON, pos
+    elseif c == "(" then return tokens.OPAREN, pos
+    elseif c == ")" then return tokens.CPAREN, pos
+    elseif c == "]" then return tokens.CSQUARE, pos
+    elseif c == "{" then return tokens.OBRACE, pos
+    elseif c == "}" then return tokens.CBRACE, pos
+    elseif c == "*" then return tokens.MUL, pos
+    elseif c == "/" then return tokens.DIV, pos
+    elseif c == "#" then return tokens.LEN, pos
+    elseif c == "%" then return tokens.MOD, pos
+    elseif c == "^" then return tokens.POW, pos
+    elseif c == "+" then return tokens.ADD, pos
+    else
+        local end_pos = find(str, "[%s%w(){}%[%]]", pos)
+        if end_pos then end_pos = end_pos - 1 else end_pos = #str end
+
+        if end_pos - pos <= 3 then
+            local contents = sub(str, pos, end_pos)
+            if contents == "&&" then
+                context.report(errors.wrong_and, pos, end_pos)
+                return tokens.AND, end_pos
+            elseif contents == "||" then
+                context.report(errors.wrong_or, pos, end_pos)
+                return tokens.OR, end_pos
+            elseif contents == "!=" or contents == "<>" then
+                context.report(errors.wrong_ne, pos, end_pos)
+                return tokens.NE, end_pos
+            elseif contents == "!" then
+                context.report(errors.wrong_not, pos, end_pos)
+                return tokens.NOT, end_pos
+            end
+        end
+
+        context.report(errors.unexpected_character, pos)
+        return tokens.ERROR, end_pos
+    end
+end
+
+--[[- Lex a single token from an input string.
+
+@param context The current parser context.
+@tparam string str The string we're lexing.
+@tparam number pos The start position.
+@treturn[1] number The id of the parsed token.
+@treturn[1] number The start position of this token.
+@treturn[1] number The end position of this token.
+@treturn[1] string|nil The token's current contents (only given for identifiers)
+@treturn[2] nil If there are no more tokens to consume
+]]
+local function lex_one(context, str, pos)
+    while true do
+        local start_pos, _, c = find(str, "([%S\r\n])", pos)
+        if not start_pos then
+            return
+        elseif c == "\r" or c == "\n" then
+            pos = newline(context, str, start_pos, c)
+        else
+            local token_id, end_pos, content, continue = lex_token(context, str, start_pos)
+            return token_id, start_pos, end_pos, content, continue
+        end
+    end
+end
+
+return {
+    lex_one = lex_one,
+}