Unable to create string literal union

Lezer
1

Hi, I am pretty new to Lezer and my intention is to create a simple query language for my expense manager app. So far I have this grammar:

@top Query {
  expression* // * because there may not be anything
  sorting?
}

@tokens {
  String { '"' (!["\\] | "\\" _)* '"' }

  Number { '-'? int frac? exp?  }
  int  { '0' | $[1-9] @digit* }
  frac { '.' @digit+ }
  exp  { $[eE] $[+\-]? @digit+ }

  LineComment { ";" ![\n]* }

  space { @whitespace+ }

  "("
  ")"
}

@skip { space | LineComment }

expression {
  Or | And | Not |
  Group |

  // Custom commands
  IsAutomatic |
  Name |
  AmountOver
}

@precedence { leftPrecedence @left }

Or {
  expression !leftPrecedence ":or" expression
}

And {
  expression !leftPrecedence ":and" expression
}

Not {
  ":not" expression
}

Group {
  "(" expression+ ")"
}

IsAutomatic {
  ":is-automatic"
}

Name {
  ":name" String
}

AmountOver {
  ":amount-over" Number
}

sorting {
  SortAsc | SortDesc
}

SortAsc {
  ":sort-asc-by" sortExpression
}

SortDesc {
  ":sort-desc-by" sortExpression
}

sortExpression {
  @specialize<String, "name"> | @specialize<String, "date">
}

@detectDelim

The grammar compiles, but when I try to parse:

:sort-asc-by "name"

I get SyntaxError: No parse at 13 and I am not sure what could be wrong…

The idea behind my language is pretty simple. I want to have commands (:command-name ), what I would love is to have a way to specify enums for some of the commands. For example, specific sorting methods - so far, all of my attempts with Lezer has been unsuccessful :(.

2

The quotes in "name"/"date" are part of Lezer’s syntax, so the strings you are specializing are just name/date. Strings always include quotes so these will never match any actual token. Using '"name"' instead should work.

(Also, using simple quoted tokens for things like your : keywords is not a good idea—it’ll match them even when there’s further letters after them, and blow up the size of your tokenizer tables. It’s better to define a token for colon-followed-by-identifier, and then @specialize that for each instance.)

3

Thanks. That helped me a lot :).

I even changed my language grammar a bit, so it’s more strict. Pasting it here in case anyone is interested.

@top Query {
  expression? | SortAscBy | SortDescBy
}

@tokens {
  String { '"' (!["\\] | "\\" _)* '"' }

  Number { '-'? int frac? exp?  }
  int  { '0' | $[1-9] @digit* }
  frac { '.' @digit+ }
  exp  { $[eE] $[+\-]? @digit+ }

  LineComment { ";" ![\n]* }

  space { @whitespace+ }

  keyword { ":" $[-a-z]+ }

  "("
  ")"
}

@skip { space | LineComment }

expression {
  Or | And | Not |
  Group |

  // Custom commands
  IsAutomatic |
  Name |
  AmountOver |
  TotalAmountOver
}

@precedence { leftPrecedence @left }

Or {
  expression !leftPrecedence @specialize<keyword, ":or"> expression
}

And {
  expression !leftPrecedence @specialize<keyword, ":and"> expression
}

Not {
  @specialize<keyword, ":not"> expression
}

Group {
  "(" expression ")"
}

IsAutomatic {
  @specialize<keyword, ":is-automatic">
}

Name {
  @specialize<keyword, ":name"> String
}

AmountOver {
  @specialize<keyword, ":amount-over"> Number String
}

TotalAmountOver {
  @specialize<keyword, ":total-amount-over"> Number String
}

SortAscBy {
  expression? @specialize<keyword, ":sort-asc-by"> SortMethod
}

SortDescBy {
  expression? @specialize<keyword, ":sort-desc-by"> SortMethod
}

SortMethod {
  @specialize<String,'"name"'> | @specialize<String, '"tx-count"'> | @specialize<String, '"total-amount"'>
}

@detectDelim