packages/sql/tokenizer.test.ts

Rama main · réplica de lectura del repositorio original.

import { describe, expect, it } from 'bun:test'

import { Dialect } from '@tamery/shared/enums/connection-type'

import { dialects } from './dialects'
import { tokenize } from './tokenizer'

const pg = dialects[Dialect.Postgres]
const mysql = dialects[Dialect.MySQL]
const kinds = (sql: string, dialect = pg) =>
  tokenize(sql, dialect).tokens.map((token) => `${token.kind}:${token.text}`)
const names = (sql: string, dialect = pg) =>
  tokenize(sql, dialect).tokens.map((token) => token.name ?? token.text)

describe('tokenize', () => {
  it('resolves identifier names the way the engine does', () => {
    expect(names('"a""b" Foo')).toEqual(['a"b', 'foo'])
    expect(names('[a]]b] Foo', dialects[Dialect.MSSQL])).toEqual(['a]b', 'Foo'])
  })

  it('classifies words, literals and operators', () => {
    expect(
      kinds('SELECT count(*) AS n, "Col" FROM t WHERE a->>\'k\' = 1.5e3')
    ).toEqual([
      'keyword:SELECT',
      'function:count',
      'punctuation:(',
      'operator:*',
      'punctuation:)',
      'keyword:AS',
      'identifier:n',
      'punctuation:,',
      'identifier:"Col"',
      'keyword:FROM',
      'identifier:t',
      'keyword:WHERE',
      'identifier:a',
      'operator:->>',
      "string:'k'",
      'operator:=',
      'number:1.5e3',
    ])
  })

  it('keeps a string prefix in the literal', () => {
    expect(kinds("SELECT X'00FF', E'a\\'b', N'n'")).toEqual([
      'keyword:SELECT',
      "string:X'00FF'",
      'punctuation:,',
      "string:E'a\\'b'",
      'punctuation:,',
      "string:N'n'",
    ])
  })

  it('follows dialect quoting', () => {
    expect(kinds('SELECT `a`, "b", @v, # c', mysql)).toEqual([
      'keyword:SELECT',
      'identifier:`a`',
      'punctuation:,',
      'string:"b"',
      'punctuation:,',
      'variable:@v',
      'punctuation:,',
      'comment:# c',
    ])
    expect(kinds('[dbo].[Users]', dialects[Dialect.MSSQL])).toEqual([
      'identifier:[dbo]',
      'punctuation:.',
      'identifier:[Users]',
    ])
    expect(kinds("$$ a; 'b $$ || $q$x$q$ || 'it''s' || $1")).toEqual([
      "string:$$ a; 'b $$",
      'operator:||',
      'string:$q$x$q$',
      'operator:||',
      "string:'it''s'",
      'operator:||',
      'variable:$1',
    ])
  })

  it('resumes across chunks and flags unclosed constructs', () => {
    const first = tokenize("SELECT '", pg)
    expect(first.tokens.at(-1)?.unclosed).toBe(true)
    expect(first.state.kind).toBe('string')
    const second = tokenize("still' FROM t", pg, first.state)
    expect(second.tokens.map((token) => token.kind)).toEqual([
      'string',
      'keyword',
      'identifier',
    ])
    expect(second.state.kind).toBe('none')
    expect(tokenize('/* open', pg).state.kind).toBe('comment')
  })
})