Class: Ruby::Rego::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby/rego/lexer.rb,
lib/ruby/rego/lexer/stream.rb,
lib/ruby/rego/lexer/number_reader.rb,
lib/ruby/rego/lexer/string_reader.rb,
lib/ruby/rego/lexer/template_string_reader.rb

Overview

Template string lexer helpers.

Constant Summary collapse

KEYWORDS = 
{
  "package" => TokenType::PACKAGE,
  "import" => TokenType::IMPORT,
  "as" => TokenType::AS,
  "default" => TokenType::DEFAULT,
  "if" => TokenType::IF,
  "contains" => TokenType::CONTAINS,
  "some" => TokenType::SOME,
  "in" => TokenType::IN,
  "every" => TokenType::EVERY,
  "not" => TokenType::NOT,
  "and" => TokenType::AND,
  "or" => TokenType::OR,
  "with" => TokenType::WITH,
  "else" => TokenType::ELSE,
  "true" => TokenType::TRUE,
  "false" => TokenType::FALSE,
  "null" => TokenType::NULL,
  "data" => TokenType::DATA,
  "input" => TokenType::INPUT
}.freeze
SINGLE_CHAR_TOKENS = 
{
  "(" => TokenType::LPAREN,
  ")" => TokenType::RPAREN,
  "[" => TokenType::LBRACKET,
  "]" => TokenType::RBRACKET,
  "{" => TokenType::LBRACE,
  "}" => TokenType::RBRACE,
  "." => TokenType::DOT,
  "," => TokenType::COMMA,
  ";" => TokenType::SEMICOLON,
  "+" => TokenType::PLUS,
  "-" => TokenType::MINUS,
  "*" => TokenType::STAR,
  "/" => TokenType::SLASH,
  "%" => TokenType::PERCENT,
  "|" => TokenType::PIPE,
  "&" => TokenType::AMPERSAND
}.freeze
COMPOUND_TOKENS = 
{
  ":" => [TokenType::COLON, TokenType::ASSIGN],
  "=" => [TokenType::UNIFY, TokenType::EQ],
  "!" => [nil, TokenType::NEQ],
  "<" => [TokenType::LT, TokenType::LTE],
  ">" => [TokenType::GT, TokenType::GTE]
}.freeze
NEWLINE_CHARS = 
["\n", "\r"].freeze
WHITESPACE_CHARS = 
[" ", "\t"].freeze
EXPONENT_CHARS = 
%w[e E].freeze
SIGN_CHARS = 
%w[+ -].freeze
IDENTIFIER_START = 
/[A-Za-z_]/
IDENTIFIER_PART = 
/[A-Za-z0-9_]/
DIGIT = 
/\d/
HEX_DIGIT = 
/[0-9A-Fa-f]/
TEMPLATE_ESCAPE = 
"\u0000"

Instance Method Summary collapse

Constructor Details

#initialize(source) ⇒ Lexer

Create a lexer for the provided source.

Parameters:

  • source (String) —

    Rego source code



78
79
80
81
82
83
84
 
# File 'lib/ruby/rego/lexer.rb', line 78

def initialize(source)
  @source = source.to_s
  @position = 0
  @line = 1
  @column = 1
  @offset = 0
end

Instance Method Details

#tokenize ⇒ Array<Token>

Tokenize the source into a list of tokens, including EOF.

Returns:

  • (Array<Token>) —

    token stream



89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/ruby/rego/lexer.rb', line 89

def tokenize
  # @type var tokens: Array[Token]
  tokens = []

  loop do
    skip_whitespace
    break if eof?

    tokens << next_token
  end

  tokens << eof_token
  tokens
end