Lexer Semantics

type Token
  = oneof {
           identifier: String;
           reservedWord: String;
           punctuator: String;
           number: Double;
           string: String;
           lineBreaks;
           end}

action Token[NextToken] : Token

Token[NextToken WhiteSpace Token] = Token[Token]

action Token[Token] : Token

Token[Token LineBreaks] = lineBreaks

Token[Token IdentifierOrReservedWord] = Token[IdentifierOrReservedWord]

Token[Token Punctuator] = punctuator Punctuator[Punctuator]

Token[Token NumericLiteral] = number DoubleValue[NumericLiteral]

Token[Token StringLiteral] = string StringValue[StringLiteral]

Token[Token EndOfInput] = end

Keywords

Syntax

IdentifierName

IdentifierLetter

| IdentifierName IdentifierLetter

| IdentifierName DecimalDigit

IdentifierLetter

A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z

| a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z

| $ | _

DecimalDigit 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9

Semantics

action Name[IdentifierName] : String

Name[IdentifierName IdentifierLetter] = [CharacterValue[IdentifierLetter]]

Name[IdentifierName IdentifierName₁ IdentifierLetter]
= Name[IdentifierName₁] [CharacterValue[IdentifierLetter]]

Name[IdentifierName IdentifierName₁ DecimalDigit]
= Name[IdentifierName₁] [CharacterValue[DecimalDigit]]

action CharacterValue[IdentifierLetter] : Character = nil(IdentifierLetter)

action CharacterValue[DecimalDigit] : Character = nil(DecimalDigit)

action DecimalValue[DecimalDigit] : Integer = digitValue(DecimalDigit)

keywords : String[]
  = [“break”,
      “case”,
      “catch”,
      “continue”,
      “default”,
      “delete”,
      “do”,
      “else”,
      “finally”,
      “for”,
      “function”,
      “if”,
      “in”,
      “new”,
      “return”,
      “switch”,
      “this”,
      “throw”,
      “try”,
      “typeof”,
      “var”,
      “void”,
      “while”,
      “with”]

futureReservedWords : String[]
= [“class”, “const”, “debugger”, “enum”, “export”, “extends”, “import”, “super”]

literals : String[] = [“null”, “true”, “false”]

reservedWords : String[] = keywords futureReservedWords literals

member(id: String, list: String[]) : Boolean
  = if |list| = 0
     then false
     else let s: String = list[0]
           in if id = s
               then true
               else member(id, list[1 ...])

Syntax

IdentifierOrReservedWord IdentifierName

Semantics

action Token[IdentifierOrReservedWord] : Token

Token[IdentifierOrReservedWord  IdentifierName]
  = let id: String = Name[IdentifierName]
     in if member(id, reservedWords)
         then reservedWord id
         else identifier id

Punctuators

Syntax

Punctuator

=

| >

| <

| = =

| = = =

| < =

| > =

| ! =

| ! = =

| ,

| !

| ~

| ?

| :

| .

| & &

| | |

| + +

| - -

| +

| -

| *

| /

| &

| |

| ^

| %

| < <

| > >

| > > >

| + =

| - =

| * =

| / =

| & =

| | =

| ^ =

| % =

| < < =

| > > =

| > > > =

| (

| )

| {

| }

| [

| ]

| ;

Semantics

action Punctuator[Punctuator] : String

Punctuator[Punctuator =] = “=”

Punctuator[Punctuator >] = “>”

Punctuator[Punctuator <] = “<”

Punctuator[Punctuator = =] = “==”

Punctuator[Punctuator = = =] = “===”

Punctuator[Punctuator < =] = “<=”

Punctuator[Punctuator > =] = “>=”

Punctuator[Punctuator ! =] = “!=”

Punctuator[Punctuator ! = =] = “!==”

Punctuator[Punctuator ,] = “,”

Punctuator[Punctuator !] = “!”

Punctuator[Punctuator ~] = “~”

Punctuator[Punctuator ?] = “?”

Punctuator[Punctuator :] = “:”

Punctuator[Punctuator .] = “.”

Punctuator[Punctuator & &] = “&&”

Punctuator[Punctuator | |] = “||”

Punctuator[Punctuator + +] = “++”

Punctuator[Punctuator - -] = “--”

Punctuator[Punctuator +] = “+”

Punctuator[Punctuator -] = “-”

Punctuator[Punctuator *] = “*”

Punctuator[Punctuator /] = “/”

Punctuator[Punctuator &] = “&”

Punctuator[Punctuator |] = “|”

Punctuator[Punctuator ^] = “^”

Punctuator[Punctuator %] = “%”

Punctuator[Punctuator < <] = “<<”

Punctuator[Punctuator > >] = “>>”

Punctuator[Punctuator > > >] = “>>>”

Punctuator[Punctuator + =] = “+=”

Punctuator[Punctuator - =] = “-=”

Punctuator[Punctuator * =] = “*=”

Punctuator[Punctuator / =] = “/=”

Punctuator[Punctuator & =] = “&=”

Punctuator[Punctuator | =] = “|=”

Punctuator[Punctuator ^ =] = “^=”

Punctuator[Punctuator % =] = “%=”

Punctuator[Punctuator < < =] = “<<=”

Punctuator[Punctuator > > =] = “>>=”

Punctuator[Punctuator > > > =] = “>>>=”

Punctuator[Punctuator (] = “(”

Punctuator[Punctuator )] = “)”

Punctuator[Punctuator {] = “{”

Punctuator[Punctuator }] = “}”

Punctuator[Punctuator [] = “[”

Punctuator[Punctuator ]] = “]”

Punctuator[Punctuator ;] = “;”

Numeric literals

Syntax

NumericLiteral

DecimalLiteral

| HexIntegerLiteral

| OctalIntegerLiteral

Semantics

action DoubleValue[NumericLiteral] : Double

DoubleValue[NumericLiteral DecimalLiteral]
= rationalToDouble(RationalValue[DecimalLiteral])

DoubleValue[NumericLiteral HexIntegerLiteral]
= rationalToDouble(IntegerValue[HexIntegerLiteral])

DoubleValue[NumericLiteral OctalIntegerLiteral]
= rationalToDouble(IntegerValue[OctalIntegerLiteral])

expt(base: Rational, exponent: Integer) : Rational
  = if exponent = 0
     then 1
     else if exponent < 0
     then 1/expt(base, -exponent)
     else base*expt(base, exponent - 1)

Syntax

DecimalLiteral Mantissa Exponent

Mantissa

DecimalIntegerLiteral

| DecimalIntegerLiteral .

| DecimalIntegerLiteral . Fraction

| . Fraction

DecimalIntegerLiteral

0

| NonZeroDecimalDigits

NonZeroDecimalDigits

NonZeroDigit

| NonZeroDecimalDigits DecimalDigit

NonZeroDigit 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9

Fraction DecimalDigits

Semantics

action RationalValue[DecimalLiteral] : Rational

RationalValue[DecimalLiteral Mantissa Exponent]
= RationalValue[Mantissa]*expt(10, IntegerValue[Exponent])

action RationalValue[Mantissa] : Rational

RationalValue[Mantissa DecimalIntegerLiteral] = IntegerValue[DecimalIntegerLiteral]

RationalValue[Mantissa DecimalIntegerLiteral .] = IntegerValue[DecimalIntegerLiteral]

RationalValue[Mantissa DecimalIntegerLiteral . Fraction]
= IntegerValue[DecimalIntegerLiteral] + RationalValue[Fraction]

RationalValue[Mantissa . Fraction] = RationalValue[Fraction]

action IntegerValue[DecimalIntegerLiteral] : Integer

IntegerValue[DecimalIntegerLiteral 0] = 0

IntegerValue[DecimalIntegerLiteral NonZeroDecimalDigits]
= IntegerValue[NonZeroDecimalDigits]

action IntegerValue[NonZeroDecimalDigits] : Integer

IntegerValue[NonZeroDecimalDigits NonZeroDigit] = DecimalValue[NonZeroDigit]

IntegerValue[NonZeroDecimalDigits NonZeroDecimalDigits₁ DecimalDigit]
= 10*IntegerValue[NonZeroDecimalDigits₁] + DecimalValue[DecimalDigit]

action DecimalValue[NonZeroDigit] : Integer = digitValue(NonZeroDigit)

action RationalValue[Fraction] : Rational

RationalValue[Fraction DecimalDigits]
= IntegerValue[DecimalDigits]/expt(10, NDigits[DecimalDigits])

Syntax

Exponent

«empty»

| ExponentIndicator SignedInteger

ExponentIndicator E | e

SignedInteger

DecimalDigits

| + DecimalDigits

| - DecimalDigits

Semantics

action IntegerValue[Exponent] : Integer

IntegerValue[Exponent «empty»] = 0

IntegerValue[Exponent ExponentIndicator SignedInteger] = IntegerValue[SignedInteger]

action IntegerValue[SignedInteger] : Integer

IntegerValue[SignedInteger DecimalDigits] = IntegerValue[DecimalDigits]

IntegerValue[SignedInteger + DecimalDigits] = IntegerValue[DecimalDigits]

IntegerValue[SignedInteger - DecimalDigits] = -IntegerValue[DecimalDigits]

Syntax

DecimalDigits

DecimalDigit

| DecimalDigits DecimalDigit

Semantics

action IntegerValue[DecimalDigits] : Integer

action NDigits[DecimalDigits] : Integer

IntegerValue[DecimalDigits DecimalDigit] = DecimalValue[DecimalDigit]

NDigits[DecimalDigits DecimalDigit] = 1

IntegerValue[DecimalDigits DecimalDigits₁ DecimalDigit]
= 10*IntegerValue[DecimalDigits₁] + DecimalValue[DecimalDigit]

NDigits[DecimalDigits DecimalDigits₁ DecimalDigit] = NDigits[DecimalDigits₁] + 1

Syntax

HexIntegerLiteral

0 HexIndicator HexDigit

| HexIntegerLiteral HexDigit

HexIndicator X | x

HexDigit 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | A | B | C | D | E | F | a | b | c | d | e | f

OctalIntegerLiteral

0 OctalDigit

| OctalIntegerLiteral OctalDigit

OctalDigit 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7

Semantics

action IntegerValue[HexIntegerLiteral] : Integer

IntegerValue[HexIntegerLiteral 0 HexIndicator HexDigit] = HexValue[HexDigit]

IntegerValue[HexIntegerLiteral HexIntegerLiteral₁ HexDigit]
= 16*IntegerValue[HexIntegerLiteral₁] + HexValue[HexDigit]

action HexValue[HexDigit] : Integer = digitValue(HexDigit)

action IntegerValue[OctalIntegerLiteral] : Integer

IntegerValue[OctalIntegerLiteral 0 OctalDigit] = OctalValue[OctalDigit]

IntegerValue[OctalIntegerLiteral OctalIntegerLiteral₁ OctalDigit]
= 8*IntegerValue[OctalIntegerLiteral₁] + OctalValue[OctalDigit]

action CharacterValue[OctalDigit] : Character = nil(OctalDigit)

action OctalValue[OctalDigit] : Integer = digitValue(OctalDigit)

String literals

Syntax

Quote {single, double}

StringLiteral

' StringChars^single '

| " StringChars^double "

Semantics

action StringValue[StringLiteral] : String

StringValue[StringLiteral ' StringChars^single '] = StringValue[StringChars^single]

StringValue[StringLiteral " StringChars^double "] = StringValue[StringChars^double]

Syntax

StringChars^Quote

OrdinaryStringChars^Quote

| StringChars^Quote \ ShortOctalEscape

OrdinaryStringChars^Quote

«empty»

| StringChars^Quote PlainStringChar

| StringChars^Quote PlainStringQuote^Quote

| OrdinaryStringChars^Quote OctalDigit

| StringChars^Quote \ OrdinaryEscape

PlainStringChar UnicodeCharacter except ' | " | \ | OctalDigit | LineTerminator

PlainStringQuote^single "

PlainStringQuote^double '

Semantics

action StringValue[StringChars^Quote] : String

StringValue[StringChars^Quote OrdinaryStringChars^Quote]
= StringValue[OrdinaryStringChars^Quote]

StringValue[StringChars^Quote StringChars^Quote₁ \ ShortOctalEscape]
= StringValue[StringChars^Quote₁] [CharacterValue[ShortOctalEscape]]

action StringValue[OrdinaryStringChars^Quote] : String

StringValue[OrdinaryStringChars^Quote «empty»] = “”

StringValue[OrdinaryStringChars^Quote StringChars^Quote PlainStringChar]
= StringValue[StringChars^Quote] [CharacterValue[PlainStringChar]]

StringValue[OrdinaryStringChars^Quote StringChars^Quote PlainStringQuote^Quote]
= StringValue[StringChars^Quote] [CharacterValue[PlainStringQuote^Quote]]

StringValue[OrdinaryStringChars^Quote OrdinaryStringChars^Quote₁ OctalDigit]
= StringValue[OrdinaryStringChars^Quote₁] [CharacterValue[OctalDigit]]

StringValue[OrdinaryStringChars^Quote StringChars^Quote \ OrdinaryEscape]
= StringValue[StringChars^Quote] [CharacterValue[OrdinaryEscape]]

action CharacterValue[PlainStringChar] : Character = nil(PlainStringChar)

action CharacterValue[PlainStringQuote^Quote] : Character

CharacterValue[PlainStringQuote^single "] = ‘"’

CharacterValue[PlainStringQuote^double '] = ‘'’

Syntax

OrdinaryEscape

StringCharEscape

| FullOctalEscape

| HexEscape

| UnicodeEscape

| StringNonEscape

StringNonEscape NonTerminator except OctalDigit | x | u | ' | " | \ | b | f | n | r | t | v

Semantics

action CharacterValue[OrdinaryEscape] : Character

CharacterValue[OrdinaryEscape StringCharEscape] = CharacterValue[StringCharEscape]

CharacterValue[OrdinaryEscape FullOctalEscape] = CharacterValue[FullOctalEscape]

CharacterValue[OrdinaryEscape HexEscape] = CharacterValue[HexEscape]

CharacterValue[OrdinaryEscape UnicodeEscape] = CharacterValue[UnicodeEscape]

CharacterValue[OrdinaryEscape StringNonEscape] = CharacterValue[StringNonEscape]

action CharacterValue[StringNonEscape] : Character = nil(StringNonEscape)

Syntax

StringCharEscape

'

| "

| \

| b

| f

| n

| r

| t

| v

Semantics

action CharacterValue[StringCharEscape] : Character

CharacterValue[StringCharEscape '] = ‘'’

CharacterValue[StringCharEscape "] = ‘"’

CharacterValue[StringCharEscape \] = ‘\’

CharacterValue[StringCharEscape b] = ‘«BS»’

CharacterValue[StringCharEscape f] = ‘«FF»’

CharacterValue[StringCharEscape n] = ‘«LF»’

CharacterValue[StringCharEscape r] = ‘«CR»’

CharacterValue[StringCharEscape t] = ‘«TAB»’

CharacterValue[StringCharEscape v] = ‘«VT»’

Syntax

ShortOctalEscape

OctalDigit

| ZeroToThree OctalDigit

FullOctalEscape

FourToSeven OctalDigit

| ZeroToThree OctalDigit OctalDigit

ZeroToThree 0 | 1 | 2 | 3

FourToSeven 4 | 5 | 6 | 7

HexEscape x HexDigit HexDigit

UnicodeEscape u HexDigit HexDigit HexDigit HexDigit

Semantics

action CharacterValue[ShortOctalEscape] : Character

CharacterValue[ShortOctalEscape OctalDigit] = codeToCharacter(OctalValue[OctalDigit])

CharacterValue[ShortOctalEscape ZeroToThree OctalDigit]
= codeToCharacter(8*OctalValue[ZeroToThree] + OctalValue[OctalDigit])

action CharacterValue[FullOctalEscape] : Character

CharacterValue[FullOctalEscape FourToSeven OctalDigit]
= codeToCharacter(8*OctalValue[FourToSeven] + OctalValue[OctalDigit])

CharacterValue[FullOctalEscape  ZeroToThree OctalDigit₁ OctalDigit₂]
  = codeToCharacter(
         64*OctalValue[ZeroToThree] + 8*OctalValue[OctalDigit₁] + OctalValue[OctalDigit₂])

action OctalValue[ZeroToThree] : Integer = digitValue(ZeroToThree)

action OctalValue[FourToSeven] : Integer = digitValue(FourToSeven)

action CharacterValue[HexEscape] : Character

CharacterValue[HexEscape x HexDigit₁ HexDigit₂]
= codeToCharacter(16*HexValue[HexDigit₁] + HexValue[HexDigit₂])

action CharacterValue[UnicodeEscape] : Character

CharacterValue[UnicodeEscape  u HexDigit₁ HexDigit₂ HexDigit₃ HexDigit₄]
  = codeToCharacter(
         4096*HexValue[HexDigit₁] + 256*HexValue[HexDigit₂] + 16*HexValue[HexDigit₃] +
         HexValue[HexDigit₄])

Waldemar Horwat
Last modified Thursday, July 22, 1999