tests/pdfText.test.ts
Rama main · réplica de lectura del repositorio original.
// THE PAGE STORE (server/pdfText.ts + shared/pdfText.ts): the validator every
// entry read from disk goes through, and one real extraction over a PDF
// written by hand — because "pdf.js runs under Node" is a fact this feature
// stands on and a test is the only place it is asserted.
import assert from "node:assert/strict";
import { readdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { after, before, describe, it } from "node:test";
import {
cleanBookTitle,
cleanPageText,
cleanPdfTextEntry,
cleanPdfTextFailure,
entryChars,
PDFTEXT_BOOK_CHARS_MAX,
PDFTEXT_PAGE_CHARS_MAX,
PDFTEXT_PAGES_MAX,
} from "../shared/pdfText.ts";
import { parseSearchQuery, searchScope } from "../shared/searchQuery.ts";
import { extractAll, initPdfText, pdfSearchEnabled, searchPages } from "../server/pdfText.ts";
import { initIndexer, search } from "../server/indexer.ts";
import { initSite } from "../server/site.ts";
import { initVault } from "../server/vault.ts";
import { makeDir, makeVault, removeVault } from "./helpers/vault.ts";
// ================================================================ the cleaner
describe("cleanPdfTextEntry: what a stored page record must be", () => {
const good = { title: "Ihya", pageCount: 3, pages: ["one", "", "three"], truncated: false, extractedAt: 1700000000000 };
it("keeps a well-formed entry, page slots included", () => {
const entry = cleanPdfTextEntry(good);
assert.ok(entry);
assert.deepEqual(entry.pages, ["one", "", "three"]);
assert.equal(entry.pageCount, 3);
assert.equal(entry.title, "Ihya");
assert.equal(entry.truncated, false);
assert.equal(entry.extractedAt, 1700000000000);
});
it("refuses anything that is not a record of pages", () => {
for (const bad of [null, "x", 3, [], {}, { pages: "no" }, { pages: [], pageCount: -1 }, { pages: [1], pageCount: 1 }]) {
assert.equal(cleanPdfTextEntry(bad), null, JSON.stringify(bad));
}
});
it("refuses more stored pages than the book has — a record that is not about its book", () => {
assert.equal(cleanPdfTextEntry({ ...good, pageCount: 2 }), null);
// Fewer is an honest cap.
assert.ok(cleanPdfTextEntry({ ...good, pageCount: 9 }));
});
it("collapses whitespace and strips control characters on the way in", () => {
const entry = cleanPdfTextEntry({ ...good, pages: [" a \n\n b\u0000c\u000c ", "", ""] });
assert.ok(entry);
assert.equal(entry.pages[0], "a bc");
assert.equal(cleanPageText("x\ty\r\nz"), "x y z");
});
it("applies the page, book and pages caps and says so", () => {
const long = "a".repeat(PDFTEXT_PAGE_CHARS_MAX + 10);
const perPage = cleanPdfTextEntry({ ...good, pageCount: 3, pages: [long, "", ""] });
assert.ok(perPage);
assert.equal(perPage.pages[0].length, PDFTEXT_PAGE_CHARS_MAX);
assert.equal(perPage.truncated, true);
const many = Array.from({ length: PDFTEXT_PAGES_MAX + 5 }, () => "p");
const perBook = cleanPdfTextEntry({ ...good, pageCount: many.length, pages: many });
assert.ok(perBook);
assert.equal(perBook.pages.length, PDFTEXT_PAGES_MAX);
assert.equal(perBook.truncated, true);
const dense = Array.from({ length: Math.ceil(PDFTEXT_BOOK_CHARS_MAX / PDFTEXT_PAGE_CHARS_MAX) + 2 }, () =>
"b".repeat(PDFTEXT_PAGE_CHARS_MAX),
);
const chars = cleanPdfTextEntry({ ...good, pageCount: dense.length, pages: dense });
assert.ok(chars);
assert.ok(entryChars(chars) <= PDFTEXT_BOOK_CHARS_MAX);
assert.equal(chars.truncated, true);
});
it("makes a /Title safe for a row and drops a missing or absurd timestamp", () => {
assert.equal(cleanBookTitle(" The Ihya\n of \u0007Ulum "), "The Ihya of Ulum");
assert.equal(cleanBookTitle(42), "");
assert.equal(cleanBookTitle("t".repeat(500)).length, 200);
const entry = cleanPdfTextEntry({ ...good, title: 7, extractedAt: "soon" });
assert.ok(entry);
assert.equal(entry.title, "");
assert.equal(entry.extractedAt, 0);
});
it("validates a failure record by its reason", () => {
assert.deepEqual(cleanPdfTextFailure({ reason: "parse", at: 5 }), { reason: "parse", at: 5 });
assert.equal(cleanPdfTextFailure({ reason: "bored", at: 5 }), null);
assert.equal(cleanPdfTextFailure({ reason: "parse" }), null);
assert.equal(cleanPdfTextFailure("parse"), null);
});
});
// ============================================================ a real book
/** A PDF written by hand: one text object per page, offsets computed here.
* Enough for pdf.js to open and read, and not a byte more. */
function tinyPdf(pages: string[]): Buffer {
const objs: string[] = [];
const n = pages.length;
const kids = pages.map((_, i) => `${3 + 2 * i} 0 R`).join(" ");
objs.push("<< /Type /Catalog /Pages 2 0 R >>");
objs.push(`<< /Type /Pages /Kids [${kids}] /Count ${n} >>`);
const fontId = 3 + 2 * n;
pages.forEach((text, i) => {
const pageId = 3 + 2 * i;
const content = `BT /F1 24 Tf 72 700 Td (${text}) Tj ET`;
objs.push(
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents ${pageId + 1} 0 R ` +
`/Resources << /Font << /F1 ${fontId} 0 R >> >> >>`,
);
objs.push(`<< /Length ${content.length} >>\nstream\n${content}\nendstream`);
});
objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>");
let out = "%PDF-1.4\n";
const offsets: number[] = [];
objs.forEach((o, i) => {
offsets.push(Buffer.byteLength(out, "latin1"));
out += `${i + 1} 0 obj\n${o}\nendobj\n`;
});
const xref = Buffer.byteLength(out, "latin1");
out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n`;
for (const off of offsets) out += `${String(off).padStart(10, "0")} 00000 n \n`;
out += `trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
return Buffer.from(out, "latin1");
}
const data = makeDir();
const vault = makeVault({
"Notes/Zephyr.md": "# Zephyr\n\nA note that also says zephyr.\n",
"Books/Astrolabe.pdf": tinyPdf(["The astrolabe quadrant and the zephyr", "A second page mentions the gnomon"]).toString("latin1"),
"Books/Broken.pdf": "%PDF-1.7\nthis is not a pdf\n%%EOF\n",
});
before(async () => {
initSite({ ASTROLABE_DATA: data });
initVault(vault);
initPdfText({});
await initIndexer();
});
after(() => {
removeVault(vault);
removeVault(data);
});
describe("the page store, over a vault with a book in it", () => {
it("is on unless PDF_SEARCH says off", () => {
assert.equal(pdfSearchEnabled(), true);
initPdfText({ PDF_SEARCH: "off" });
assert.equal(pdfSearchEnabled(), false);
assert.deepEqual(searchPages("zephyr"), []);
initPdfText({ PDF_SEARCH: "on" });
});
it("reads every page, records the broken book, and writes into ASTROLABE_DATA only", async () => {
const stats = await extractAll();
assert.equal(stats.searchable, 1);
assert.equal(stats.failed, 1);
assert.ok(readdirSync(data).includes("pdftext.json"), "the text lives in ASTROLABE_DATA");
assert.ok(!readdirSync(path.join(vault, "Books")).some((f) => f.endsWith(".json")), "and never in the vault");
const file = JSON.parse(readFileSync(path.join(data, "pdftext.json"), "utf8")) as {
version: number;
texts: Record<string, { pages: string[]; pageCount: number }>;
failures: Record<string, { reason: string }>;
};
assert.equal(file.version, 1);
const [entry] = Object.values(file.texts);
assert.equal(entry.pageCount, 2);
assert.deepEqual(entry.pages, ["The astrolabe quadrant and the zephyr", "A second page mentions the gnomon"]);
const [failure] = Object.values(file.failures);
assert.equal(failure.reason, "parse");
});
it("answers a word with the book, the page and a marked snippet", () => {
const hits = searchPages("gnomon");
assert.equal(hits.length, 1);
assert.equal(hits[0].kind, "book");
assert.equal(hits[0].path, "Books/Astrolabe.pdf");
assert.equal(hits[0].page, 2);
assert.equal(hits[0].title, "Astrolabe");
assert.match(hits[0].snippet, /<mark>gnomon<\/mark>/);
});
it("ANDs the words, folds case, and applies path: but no note operator", () => {
assert.equal(searchPages("ZEPHYR quadrant")[0]?.page, 1);
assert.deepEqual(searchPages("zephyr gnomon"), []);
assert.equal(searchPages("path:books zephyr").length, 1);
assert.deepEqual(searchPages("path:journal zephyr"), []);
assert.deepEqual(searchPages("tag:x zephyr"), []);
assert.deepEqual(searchPages("in:notes zephyr"), []);
});
it("a lone in:books lists the shelf, and the note index stays out of it", () => {
const shelf = searchPages("in:books");
assert.equal(shelf.length, 1);
assert.equal(shelf[0].page, 1);
assert.equal(searchScope(parseSearchQuery("in:books").filters), "books");
assert.deepEqual(search("in:books zephyr", false, null), []);
// And the note index still answers the same word on its own side.
assert.equal(search("in:notes zephyr", false, null).length, 1);
assert.equal(search("zephyr", false, null).length, 1);
});
});