Parser loses the folding and closing tags highlight after the custom templating highlight

[wesleyguirra]

Hi, I’m new to CodeMirror, actually I’m using the v6, I’m trying to make it work with my custom templating language below (something very similar to template literals ${})

templating.grammar

@top TemplateString { (Expression | JavaScriptContent)* }

@local tokens {
    ExpressionStart[closedBy=ExpressionEnd]{ "$(" }
    @else JavaScriptContent
}

@precedence {
    Expression
    JavaScriptContent
}

@skip {} {
    Expression {
        ExpressionStart (ScriptText* | "") ExpressionEnd
    }
}

@local tokens {
  ExpressionEnd[openedBy=ExpressionStart] { ")" }
  ScriptText { Identifier ":" Type "="? (Identifier | stringLiteral)? }
}

@tokens {
  Identifier { $[a-zA-Z_\-0-9]+ }
  Type { (Component | Number | String | Image | Video | Color) }
  Component { "Component" }
  Number { "Number" }
  String { "String" }
  Image { "Image" }
  Video { "Video" }
  Color { "Color" }
  stringLiteral { '"' (!["\\] | "\\" _)* '"' }
  "$(" ")"
}

templating.ts

import {parser as templatingParser} from "./templating.js"
import {foldNodeProp, foldInside, indentNodeProp, LRLanguage, LanguageSupport} from "@codemirror/language"
import {styleTags, tags as t} from "@lezer/highlight"
import { Diagnostic } from '@codemirror/lint'
import { EditorView } from '@codemirror/view'
import { javascript, javascriptLanguage, jsxLanguage } from '@codemirror/lang-javascript'
import { parseMixed } from '@lezer/common'

let parserWithMetadata = templatingParser.configure({
  props: [
    styleTags({
      Identifier: t.variableName,
      Type: t.typeName,
      Boolean: t.bool,
      String: t.string,
      LineComment: t.lineComment,
      // "( )": t.paren,
      "$( )": t.lineComment,
    }),
  ],
  wrap: parseMixed((node) => {
    return node.type.name === 'JavaScriptContent' ? {
      parser: jsxLanguage.parser,
      // overlay: node => node.type.name == "JavaScriptContent"
    } : null
    // if (node.type.name === 'JavaScriptContent') {
    //   return {
    //     parser: jsxLanguage.parser,
    //     overlay: (node) => {
    //       console.log(node)
    //       return node.type.name === 'JavaScriptContent'
    //     }
    //   }
    // }
    // if (node.type.name === 'Expression') {
    //   return {
    //     parser: templatingParser,
    //   }
    // }
    // return null
  }),
})

export const templatingLanguage = LRLanguage.define({
  parser: parserWithMetadata,
})

export function templating() {
  return new LanguageSupport(templatingLanguage)
}

export function templatingLinter(view: EditorView) {
  const code = view.state.doc.toString();
  const tree = templatingParser.parse(code);
  let messages: Diagnostic[] = [];

  tree.iterate({
    enter: (type, start, end) => {
      if (type.name === 'Identifier' && end - start > 10) {
        messages.push({
          from: start,
          to: end,
          message: 'Identifier names should be no more than 10 characters long',
          severity: 'warning'
        });
      }
    }
  });

  return messages;
}

CodeEditor.tsx

import React, { useEffect, useRef } from 'react'
import styled from 'styled-components'
import { basicSetup } from 'codemirror'
import { EditorState, Compartment } from '@codemirror/state'
import { EditorView, keymap } from '@codemirror/view'
import { search, searchKeymap } from '@codemirror/search'
import { esLint, javascript, javascriptLanguage } from '@codemirror/lang-javascript'
import { linter, lintGutter, lintKeymap } from '@codemirror/lint'
import { Linter } from 'eslint-linter-browserify'
import { monokai } from '@uiw/codemirror-theme-monokai'
import { templating, templatingLanguage, templatingLinter } from './templating'

const Container = styled.div`
    border-radius: 4px;
    overflow-y: scroll;
    max-height: 300px;
    margin-top: 20px;
    margin-bottom: 12px;
`

interface CodeEditorProps {
  onChange: (value: string) => void
  onLintError?: (errorLength: number) => void
  value?: string
  width?: number
  height?: number
  theme?: string
}

const CodeEditor: React.FC<CodeEditorProps> = ({ value, onChange }) => {
  const editor = useRef<HTMLDivElement>(null)
  const tabSize = new Compartment()
  const state = EditorState.create({
    doc: value,
    extensions: [
      basicSetup,
      tabSize.of(EditorState.tabSize.of(4)),
      keymap.of([...searchKeymap, ...lintKeymap]),
      search({ top: true }),
      lintGutter(),
      linter(esLint(
        new Linter(),
        {
          parserOptions: {
            ecmaVersion: 2019,
            sourceType: 'module',
            ecmaFeatures: { jsx: true }
          }
        }
      )),
      monokai,
      templating(),
      // linter(templatingLinter),
      EditorView.updateListener.of(function (update) {
        if (update.docChanged) {
          onChange(update.state.doc.toString())
        }
      })
    ],
  })

  useEffect(() => {
    if (!editor.current) return
    const view = new EditorView({
      state,
      parent: editor.current
    })

    view.dispatch({ changes: { from: 0, to: view.state.doc.length, insert: value || '\n\n\n\n\n\n\n\n\n\n' } })

    return () => {
      view.destroy()
    }
  }, [editor.current])

  return (
    <>
      <Container ref={editor} />
    </>
  )
}
export default CodeEditor

Not sure what I’m doing wrong, but the tags are losing the highlight and folding after my custom language.

[marijn]

I didn’t get into this too deeply, but at a glance, a @local token block without an @else token and with a rule that looks like it is a nonterminal rather than an actual token suggests you aren’t applying that feature in the right way.

[wesleyguirra]

I’m new to this grammar language, not sure what’s wrong, this is the root of the issue?

[wesleyguirra]

@marijn do you have any idea on what’s wrong with the grammar?