Ferrite.TL.Schema/Lexer.cs
Rama main · réplica de lectura del repositorio original.
// SPDX-License-Identifier: AGPL-3.0-or-later
// Copyright (C) 2022-2026 Aykut Alparslan KOC
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
namespace Ferrite.TL.Schema
{
public class Lexer
{
private TextWindow _textWindow;
private TokenType previousTokenType = TokenType.None;
public Lexer(string source)
{
_textWindow = new TextWindow(source.ToCharArray());
}
public Token Lex()
{
switch (_textWindow.Peek())
{
case '\r':
_textWindow.AdvancePastNewLine();
previousTokenType = TokenType.EOL;
return new Token()
{
Type = TokenType.EOL,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '\n':
_textWindow.AdvancePastNewLine();
previousTokenType = TokenType.EOL;
return new Token()
{
Type = TokenType.EOL,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '/':
if (ScanComment(out Token token)) return token;
_textWindow.Advance();
previousTokenType = TokenType.Slash;
return new Token()
{
Type = TokenType.Slash,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '-':
if (ScanFunctions(out token)) return token;
if (ScanTypes(out token)) return token;
_textWindow.Advance(1);
previousTokenType = TokenType.Minus;
return new Token()
{
Type = TokenType.Minus,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case ' ':
int count = 0;
while (_textWindow.Peek(++count) == ' '){}
_textWindow.Advance(count);
previousTokenType = TokenType.Spaces;
return new Token()
{
Type = TokenType.Spaces,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-count,
};
case '[':
_textWindow.Advance(1);
previousTokenType = TokenType.OpenBracket;
return new Token()
{
Type = TokenType.OpenBracket,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case ']':
_textWindow.Advance(1);
previousTokenType = TokenType.CloseBracket;
return new Token()
{
Type = TokenType.CloseBracket,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '{':
_textWindow.Advance(1);
previousTokenType = TokenType.OpenBrace;
return new Token()
{
Type = TokenType.OpenBrace,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '}':
_textWindow.Advance(1);
previousTokenType = TokenType.CloseBrace;
return new Token()
{
Type = TokenType.CloseBrace,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '(':
_textWindow.Advance(1);
previousTokenType = TokenType.OpenParen;
return new Token()
{
Type = TokenType.OpenParen,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case ')':
_textWindow.Advance(1);
previousTokenType = TokenType.CloseParen;
return new Token()
{
Type = TokenType.CloseParen,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '=':
_textWindow.Advance(1);
previousTokenType = TokenType.Equal;
return new Token()
{
Type = TokenType.Equal,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '.':
_textWindow.Advance(1);
previousTokenType = TokenType.Dot;
return new Token()
{
Type = TokenType.Dot,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case ':':
_textWindow.Advance(1);
previousTokenType = TokenType.Colon;
return new Token()
{
Type = TokenType.Colon,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn,
};
case ';':
_textWindow.Advance(1);
previousTokenType = TokenType.Semicolon;
return new Token()
{
Type = TokenType.Semicolon,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '?':
_textWindow.Advance(1);
previousTokenType = TokenType.QuestionMark;
return new Token()
{
Type = TokenType.QuestionMark,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn,
};
case '!':
_textWindow.Advance(1);
previousTokenType = TokenType.ExclamationMark;
return new Token()
{
Type = TokenType.ExclamationMark,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '#':
_textWindow.Advance(1);
previousTokenType = TokenType.Hash;
return new Token()
{
Type = TokenType.Hash,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '%':
_textWindow.Advance(1);
previousTokenType = TokenType.Percent;
return new Token()
{
Type = TokenType.Percent,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '<':
_textWindow.Advance(1);
previousTokenType = TokenType.Langle;
return new Token()
{
Type = TokenType.Langle,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '>':
_textWindow.Advance(1);
previousTokenType = TokenType.Rangle;
return new Token()
{
Type = TokenType.Rangle,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '*':
_textWindow.Advance(1);
previousTokenType = TokenType.Multiply;
return new Token()
{
Type = TokenType.Multiply,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '`':
_textWindow.Advance(1);
previousTokenType = TokenType.BackTick;
return new Token()
{
Type = TokenType.BackTick,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case '+':
_textWindow.Advance(1);
previousTokenType = TokenType.Plus;
return new Token()
{
Type = TokenType.Plus,
Line = _textWindow.CurrentLine,
Column = _textWindow.CurrentColumn-1,
};
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
if (ScanHexConstant(out token)) return token;
if (ScanNamespaceIdentifier(out token)) return token;
if (ScanCombinatorIdentifier(out token)) return token;
if ((previousTokenType == TokenType.Spaces ||
previousTokenType == TokenType.OpenBrace) &&
ScanVariableIdentifier(out token)) return token;
if (ScanConditionalIdentifier(out token)) return token;
if (ScanBareTypeIdentifier(out token)) return token;
if (ScanLowercaseIdentifier(out token)) return token;
previousTokenType = TokenType.EOF;
return new Token()
{
Type = TokenType.EOF
};
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
if (ScanHexConstant(out token)) return token;
if (ScanNumber(out token)) return token;
previousTokenType = TokenType.EOF;
return new Token()
{
Type = TokenType.EOF
};
default:
if (ScanHexConstant(out token)) return token;
if (previousTokenType == TokenType.Spaces &&
ScanVariableIdentifier(out token)) return token;
if (ScanTypeIdentifier(out token)) return token;
previousTokenType = TokenType.EOF;
return new Token()
{
Type = TokenType.EOF
};
}
}
private bool ScanComment(out Token token)
{
if (_textWindow.Peek(1) == '/')
{
int count = 0;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
char c = _textWindow.Peek(count + 2);
if (c is '\r' or '\n')
{
break;
}
}
int column = _textWindow.CurrentColumn;
_textWindow.Advance(2);
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.LineComment;
token = new Token()
{
Type = TokenType.LineComment,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
if (_textWindow.Peek(1) == '*')
{
int count = 0;
while (_textWindow.Peek(++count + 1) != TextWindow.InvalidChar)
{
if (_textWindow.Peek(count + 2) == '*' &&
_textWindow.Peek(count + 3) == '/')
{
int column = _textWindow.CurrentColumn;
_textWindow.Advance(2);
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count + 2);
previousTokenType = TokenType.MultilineComment;
token = new Token()
{
Type = TokenType.MultilineComment,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
}
}
token = default;
return false;
}
private bool ScanFunctions(out Token token)
{
if (_textWindow.Peek(1) == '-' &&
_textWindow.Peek(2) == '-')
{
int pos = 2;
bool found = false;
while (_textWindow.Peek(++pos) != TextWindow.InvalidChar)
{
if (char.IsWhiteSpace(_textWindow.Peek(pos)))
{
continue;
}
else if (_textWindow.IsExactMatch("functions", pos))
{
pos += 8;
found = true;
continue;
}
else if (found && _textWindow.IsExactMatch("---", pos))
{
pos += 3;
}
else if (found && !char.IsWhiteSpace(_textWindow.Peek(pos)))
{
found = false;
}
break;
}
if (found)
{
int column = _textWindow.CurrentColumn;
_textWindow.Advance(pos);
previousTokenType = TokenType.Functions;
token = new Token()
{
Type = TokenType.Functions,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
}
token = default;
return false;
}
private bool ScanTypes(out Token token)
{
if (_textWindow.Peek(1) == '-' &&
_textWindow.Peek(2) == '-')
{
int pos = 2;
bool found = false;
while (_textWindow.Peek(++pos) != TextWindow.InvalidChar)
{
if (char.IsWhiteSpace(_textWindow.Peek(pos)))
{
continue;
}
else if (_textWindow.IsExactMatch("types", pos))
{
pos += 4;
found = true;
continue;
}
else if (found && _textWindow.IsExactMatch("---", pos))
{
pos += 3;
}
else if (found && !char.IsWhiteSpace(_textWindow.Peek(pos)))
{
found = false;
}
break;
}
if (found)
{
int column = _textWindow.CurrentColumn;
_textWindow.Advance(pos);
previousTokenType = TokenType.Types;
token = new Token()
{
Type = TokenType.Types,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
}
token = default;
return false;
}
private bool ScanHexConstant(out Token token)
{
int count = 0;
char c = _textWindow.Peek();
while (c != TextWindow.InvalidChar)
{
switch (c)
{
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
c = _textWindow.Peek(++count);
break;
default:
c = TextWindow.InvalidChar;
break;
}
}
if (count is >= 6 and <= 8)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.HexConstant;
token = new Token()
{
Type = TokenType.HexConstant,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanNamespaceIdentifier(out Token token)
{
int count = 0;
bool found = false;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
if(char.IsWhiteSpace(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '<' ||
_textWindow.Peek(count) == ':') break;
if (_textWindow.Peek(count) == '.' &&
!char.IsLetter(_textWindow.Peek(count + 1))) break;
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
_textWindow.Peek(count + 1) == '.' &&
char.IsLetter(_textWindow.Peek(count + 2)))
{
count++;
found = true;
break;
}
}
if (found)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.NamespaceIdentifier;
token = new Token()
{
Type = TokenType.NamespaceIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanCombinatorIdentifier(out Token token)
{
int count = 0;
bool found = false;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
if(char.IsWhiteSpace(_textWindow.Peek(count))) break;
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
_textWindow.Peek(count + 1) == '#' &&
char.IsLetterOrDigit(_textWindow.Peek(count + 2)))
{
count++;
found = true;
break;
}
}
if (found)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.CombinatorIdentifier;
token = new Token()
{
Type = TokenType.CombinatorIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanVariableIdentifier(out Token token)
{
int count = 0;
bool found = false;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
if(char.IsWhiteSpace(_textWindow.Peek(count))) break;
if (_textWindow.Peek(count) == ':')
{
found = true;
break;
}
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
_textWindow.Peek(count + 1) == ':')
{
count++;
found = true;
break;
}
}
if (found)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.VariableIdentifier;
token = new Token()
{
Type = TokenType.VariableIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanConditionalIdentifier(out Token token)
{
int count = 0;
bool found = false;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
if(char.IsWhiteSpace(_textWindow.Peek(count))) break;
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
_textWindow.Peek(count + 1) == '.' &&
((char.IsDigit(_textWindow.Peek(count + 2)) &&
_textWindow.Peek(count + 3) == '?') || (char.IsDigit(_textWindow.Peek(count + 2)) &&
char.IsDigit(_textWindow.Peek(count + 3)) &&
_textWindow.Peek(count + 4) == '?')))
{
count++;
found = true;
break;
}
}
if (found)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.ConditionalIdentifier;
token = new Token()
{
Type = TokenType.ConditionalIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanTypeIdentifier(out Token token)
{
int count = 0;
bool found = false;
while (_textWindow.Peek(count) != TextWindow.InvalidChar)
{
if(char.IsWhiteSpace(_textWindow.Peek(count))) break;
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
(char.IsWhiteSpace(_textWindow.Peek(count + 1)) ||
_textWindow.Peek(count + 1) == ':' ||
_textWindow.Peek(count + 1) == ';' ||
_textWindow.Peek(count + 1) == '}' ||
_textWindow.Peek(count + 1) == '<') ||
_textWindow.Peek(count + 1) == '>')
{
count++;
found = true;
break;
}
count++;
}
if (found)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.TypeIdentifier;
token = new Token()
{
Type = TokenType.TypeIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanBareTypeIdentifier(out Token token)
{
int count = 0;
bool found = false;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
if(char.IsWhiteSpace(_textWindow.Peek(count))) break;
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
(char.IsWhiteSpace(_textWindow.Peek(count + 1)) ||
_textWindow.Peek(count + 1) == '<' ||
_textWindow.Peek(count + 1) == '>'))
{
count++;
found = true;
break;
}
}
if (found)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.BareTypeIdentifier;
token = new Token()
{
Type = TokenType.BareTypeIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
private bool ScanLowercaseIdentifier(out Token token)
{
int count = 0;
while (_textWindow.Peek(++count) != TextWindow.InvalidChar)
{
if (char.IsWhiteSpace(_textWindow.Peek(count)) &&
_textWindow.Peek(count + 2) != ']') break;
if (_textWindow.Peek(count) == ';')
{
break;
}
if ((char.IsLetterOrDigit(_textWindow.Peek(count)) ||
_textWindow.Peek(count) == '_') &&
!char.IsLower(_textWindow.Peek(count + 1)))
{
count++;
break;
}
}
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.LowercaseIdentifier;
token = new Token()
{
Type = TokenType.LowercaseIdentifier,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
private bool ScanNumber(out Token token)
{
int count = 0;
while (char.IsDigit(_textWindow.Peek(count)))
{
count++;
}
if (count > 0)
{
int column = _textWindow.CurrentColumn;
var val = _textWindow.StringSlice(count);
_textWindow.Advance(count);
previousTokenType = TokenType.Number;
token = new Token()
{
Type = TokenType.Number,
Value = val,
Line = _textWindow.CurrentLine,
Column = column,
};
return true;
}
token = default;
return false;
}
}
}