tests/test_table_gfm_compliance.py
Rama main · réplica de lectura del repositorio original.
"""
Unit tests for GFM-compliant markdown table generation.
Tests that html2text generates tables with proper leading and trailing
pipe delimiters as per GitHub Flavored Markdown specification.
Fixes: https://github.com/unclecode/crawl4ai/issues/1731
"""
import pytest
import sys
import os
# Add parent directory to path to import html2text
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from crawl4ai.html2text import HTML2Text
def _table_lines(result: str) -> list[str]:
"""Extract table lines (containing |) from html2text output, stripped."""
return [line.strip() for line in result.split('\n') if '|' in line]
class TestTableGFMCompliance:
"""Test suite for GFM-compliant table generation (pad_tables=False)."""
def test_table_has_leading_pipes(self):
"""All table rows start with |"""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
lines = _table_lines(result)
assert len(lines) > 0, "No table rows found in output"
for i, line in enumerate(lines):
assert line.startswith('|'), f"Line {i+1} missing leading pipe: {repr(line)}"
def test_table_has_trailing_pipes(self):
"""All table rows end with |"""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
lines = _table_lines(result)
assert len(lines) > 0, "No table rows found in output"
for i, line in enumerate(lines):
assert line.endswith('|'), f"Line {i+1} missing trailing pipe: {repr(line)}"
def test_cells_have_space_padding(self):
"""Cell content has spaces on both sides of pipe delimiters.
Correct: | A | B |
Incorrect: | A| B | (missing space after A)
"""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
lines = _table_lines(result)
# Check header and data rows (skip separator)
for line in lines:
if '---' in line:
continue
# Split by | and check that internal cells have spaces
cells = line.split('|')
# cells[0] is '' (before first |), cells[-1] is '' (after last |)
for cell in cells[1:-1]:
if cell.strip(): # Non-empty cell
assert cell.startswith(' '), f"Cell missing leading space: {repr(line)}"
assert cell.endswith(' '), f"Cell missing trailing space: {repr(line)}"
def test_separator_row_format(self):
"""Separator row has format | --- | --- |"""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
separators = [line.strip() for line in result.split('\n') if '---' in line]
assert len(separators) > 0, "No separator row found"
sep = separators[0]
assert sep.startswith('|'), f"Separator missing leading pipe: {repr(sep)}"
assert sep.endswith('|'), f"Separator missing trailing pipe: {repr(sep)}"
assert sep == '| --- | --- |', f"Unexpected separator format: {repr(sep)}"
def test_multirow_table(self):
"""Multi-row tables maintain GFM compliance throughout."""
html = '''<table>
<tr><th>Parameter</th><th>Guideline</th><th>Sources</th></tr>
<tr><td>Arsenic</td><td>0.010</td><td>Naturally occurring</td></tr>
<tr><td>Lead</td><td>0.005</td><td>Plumbing</td></tr>
<tr><td>Mercury</td><td>0.001</td><td>Industrial</td></tr>
</table>'''
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
lines = _table_lines(result)
# 1 header + 1 separator + 3 data rows = 5 rows
assert len(lines) >= 5, f"Expected at least 5 table rows, got {len(lines)}"
for i, line in enumerate(lines):
assert line.startswith('|'), f"Line {i+1} missing leading pipe"
assert line.endswith('|'), f"Line {i+1} missing trailing pipe"
def test_single_column_table(self):
"""Single-column tables are GFM compliant."""
html = '<table><tr><th>Header</th></tr><tr><td>Data</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
lines = _table_lines(result)
assert len(lines) > 0, "No table rows found"
for line in lines:
assert line.startswith('|') and line.endswith('|'), \
f"Single column row not GFM compliant: {repr(line)}"
def test_empty_cells(self):
"""Tables with empty cells are GFM compliant."""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td></td><td>Data</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
lines = _table_lines(result)
assert len(lines) > 0, "No table rows found"
for line in lines:
assert line.startswith('|') and line.endswith('|'), \
f"Row with empty cell not GFM compliant: {repr(line)}"
def test_table_starts_on_own_line(self):
"""Table starts on its own line, not inline with preceding content."""
html = '<p>Text before.</p><table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
# The first table line must start at the beginning of a line
# (no leading whitespace before the pipe)
for line in result.split('\n'):
stripped = line.strip()
if stripped.startswith('|'):
# Check no leading whitespace (table row at column 0)
assert line.startswith('|'), f"Table row not at line start: {repr(line)}"
break
else:
pytest.fail("No table row starting with | found")
def test_nested_table_starts_on_own_line(self):
"""Table nested in list item starts on its own line."""
html = '<ul><li>Item<table><tr><th>X</th></tr><tr><td>1</td></tr></table></li></ul>'
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
# Find the first | line — it should not be on the same line as "Item"
lines = result.split('\n')
for line in lines:
if 'Item' in line:
assert '|' not in line, \
f"Table pipe on same line as 'Item': {repr(line)}"
break
def test_caption_does_not_merge_with_header(self):
"""Table with <caption> renders caption on its own line, not inline with header."""
html = '''<table>
<caption>Table 1. Parameters</caption>
<tr><th>Name</th><th>Value</th></tr>
<tr><td>Lead</td><td>0.005</td></tr>
</table>'''
h = HTML2Text()
h.body_width = 0
result = h.handle(html)
# Caption text should NOT be on the same line as the header pipe row
for line in result.split('\n'):
if 'Table 1' in line:
assert '|' not in line, \
f"Caption on same line as table row: {repr(line)}"
break
# Header row should start with |
table_lines = _table_lines(result)
assert len(table_lines) >= 3, f"Expected at least 3 table rows, got {len(table_lines)}"
assert table_lines[0].startswith('|'), f"Header not starting with pipe: {repr(table_lines[0])}"
class TestPadTablesUnchanged:
"""Verify pad_tables=True behavior is unchanged from upstream."""
def test_pad_tables_produces_aligned_output(self):
"""pad_tables=True produces properly aligned GFM output."""
html = '<table><tr><th>Parameter</th><th>Value</th></tr><tr><td>Lead</td><td>0.005 mg/L</td></tr></table>'
h = HTML2Text()
h.body_width = 0
h.pad_tables = True
result = h.handle(html)
lines = _table_lines(result)
assert len(lines) >= 3, f"Expected at least 3 rows, got {len(lines)}"
# All rows should have leading and trailing pipes
for line in lines:
assert line.startswith('|'), f"Padded row missing leading pipe: {repr(line)}"
assert line.endswith('|'), f"Padded row missing trailing pipe: {repr(line)}"
# All rows should have same width (padded alignment)
widths = [len(line) for line in lines]
assert len(set(widths)) == 1, f"Rows have uneven widths: {widths}"
def test_pad_tables_no_double_pipes(self):
"""pad_tables=True does not produce double pipes | | or | |."""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
h.pad_tables = True
result = h.handle(html)
lines = _table_lines(result)
for line in lines:
# Should not have pipe-space-pipe (double boundary)
assert '| |' not in line, f"Double pipes found: {repr(line)}"
# Line should not start with | | (extra pipe from both systems)
assert not line.startswith('| |'), f"Extra leading pipe: {repr(line)}"
def test_pad_tables_separator_has_dashes(self):
"""pad_tables=True separator row uses dashes with proper alignment."""
html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'
h = HTML2Text()
h.body_width = 0
h.pad_tables = True
result = h.handle(html)
separators = [line.strip() for line in result.split('\n') if '---' in line]
assert len(separators) >= 1, "No separator row found in padded table"
if __name__ == '__main__':
pytest.main([__file__, '-v'])