If someone is still interested, it turned out to be not too slow.
I mainly check only PointDecorations offsetheight and then adjust the scaling of brackets/parenthesis:
{
getBracketDecorations(view) {
const { doc } = view.state;
const decorations = [];
const stack = [];
const rangeSets = view.state.facet(EditorView.atomicRanges).map((f)=>f(view));
let height = -1;
for (let pos = 0; pos < doc.length; pos += 1) {
const char = doc.sliceString(pos, pos + 1);
if (char === '(' || char === '[' || char === '{') {
stack.push({ type: char, from: pos });
} else if (char === ')' || char === ']' || char === '}') {
const open = stack.pop();
if (open && open.type === this.getMatchingBracket(char)) {
height = -1;
rangeSets.forEach((rangeSet) => {
if (rangeSet.size > 0) {
const cursor = rangeSet.iter();
let v;
do {
v = cursor.value;
if (!v) break;
const start = v.widget.visibleValue.pos;
const end = v.widget.visibleValue.pos + v.widget.visibleValue.length;
if (start > open.from && end < pos+1 && v.widget.DOMElement) {
height = Math.max(height, v.widget.DOMElement.offsetHeight)
}
cursor.next();
} while(v);
}
});
if (height > 14) {
decorations.push(
Decoration.mark({ attributes: {style: `transform: scaleY(${height/14}); display:inline-block`} }).range(open.from, open.from + 1),
Decoration.mark({ attributes: {style: `transform: scaleY(${height/14}); display:inline-block`} }).range(pos, pos + 1),
);
}
}
}
}
decorations.sort((a, b) => a.from - b.from || a.startSide - b.startSide);
return Decoration.set(decorations);
}
getMatchingBracket(closingBracket) {
switch (closingBracket) {
case ')': return '(';
case ']': return '[';
case '}': return '{';
default: return null;
}
}
}
However now there is another issue: scaled glyphs are horrible.
Therefore, I believe I need to recreate them using SVG/CSS magic with ReplacingDecorations instead of MarkDecorations. But this is another story…