__tests__/google.test.js
Rama main · réplica de lectura del repositorio original.
/**
* Google, read as data rather than driven by hand.
*
* Doing research with look/click/read costs five or six calls per source: sixty links to guess
* among, a click, ten thousand characters of which nine are menus and cookie notices, then back —
* and the results are gone. All of that is the model operating a browser instead of thinking about
* what the pages say.
*
* The two things pinned here are the ones that quietly ruin a research run: Google's consent wall,
* which looks exactly like a search that found nothing, and its redirect wrapper, which stores a
* Google address where a source should be.
*/
import { describe, it, expect, vi } from 'vitest';
import { looksLikeConsent, search, readPage, extractResults, resolveWrapped, landedElsewhere, url } from '../src/sites/google.js';
const pageThat = (results, { text = '', at = 'https://www.google.com/search?q=x', request } = {}) => ({
goto: vi.fn(async () => {}),
url: () => at,
request,
evaluate: vi.fn(async (fn) => (String(fn).includes('querySelectorAll(\'h3\')') || String(fn).includes('h3')
? results : text)),
});
/*
* THE OPAQUE WRAPPER — the defect that emptied every research pass for weeks.
*
* Measured 2026-09-04 in the live browser: the results page was full, and google() reported
* "0 result(s)" on every query. Signed-in Google links each result as /goto?url=<token>; the
* destination is nowhere in the DOM (mousedown only re-wraps the token in /url?url=…), and Google
* produces it as a 302 only when the wrapper is fetched with the search page as referer — a bare
* fetch is a 400. The old extractor wanted an http address and dropped Google's own domain, so a
* page of results read as an empty one, and the master's gate read "zero demand".
*/
describe('the opaque /goto wrapper', () => {
const { JSDOM } = (() => { try { return require('jsdom'); } catch { return {}; } })();
// The markup as it was served (class names generated; the <a><h3> shape and the /goto href are the contract).
const RESULT = (token, title, source) =>
`<div class="MjjYud"><div><a jsname="UWckNb" class="zReHs" href="/goto?url=${token}" data-jsarwt="1" data-usg="x" data-ved="y">`
+ `<h3 class="LC20lb">${title}</h3><div><span>${source}</span><div><cite>12 reacties · 3 maanden geleden</cite></div></div></a>`
+ `<div class="VwiC3b">People here keep asking which builder lets them self-host the result.</div></div></div>`;
const withDom = (html, fn) => {
if (!JSDOM) return null; // jsdom absent — the resolver tests below still guard the fix
const dom = new JSDOM(html, { url: 'https://www.google.com/search?q=x' });
const g = dom.window;
// jsdom has no layout engine, so innerText is undefined — emulate a browser's: block elements break lines.
const BLOCK = /^(DIV|P|H[1-6]|LI|UL|OL|SECTION|ARTICLE|TR|BR)$/;
Object.defineProperty(g.HTMLElement.prototype, 'innerText', { configurable: true, get() {
const walk = (n) => n.nodeType === 3 ? n.textContent
: [...n.childNodes].map(walk).join('').replace(/^|$/g, BLOCK.test(n.tagName) ? '\n' : '');
return walk(this).replace(/\n{2,}/g, '\n').trim();
} });
const prev = { document: global.document, location: global.location };
global.document = g.document; global.location = g.location;
try { return fn(); } finally { global.document = prev.document; global.location = prev.location; }
};
it('keeps a /goto result as `via` (with its source line) instead of dropping it as Google furniture', () => {
const got = withDom(RESULT('CAESabc', "What's the best AI no-code web app builder?", 'Reddit · r/nocode')
+ RESULT('CAESdef', 'I tested 5 AI app builders', 'Reddit · r/SideProject'), extractResults);
if (!got) return;
expect(got).toHaveLength(2);
expect(got[0]).toMatchObject({ title: "What's the best AI no-code web app builder?", url: '',
via: 'https://www.google.com/goto?url=CAESabc', source: 'Reddit · r/nocode' });
expect(got[0].snippet).toMatch(/self-host/);
});
it('unwraps the mousedown form /url?url=%2Fgoto… to the same wrapper, and still reads the old /url?q=REAL', () => {
const got = withDom(
`<a href="http://www.google.com/url?sa=t&url=%2Fgoto%3Furl%3DCAESxyz"><h3>Wrapped twice</h3></a>`
+ `<a href="/url?q=https://example.nl/pricing&sa=U"><h3>Old wrapper</h3></a>`
+ `<a href="https://www.google.com/search?q=more"><h3>More results</h3></a>`, extractResults);
if (!got) return;
expect(got.map((r) => r.via || r.url)).toEqual(['https://www.google.com/goto?url=CAESxyz', 'https://example.nl/pricing']);
});
const answer = (status, location) => ({ status: () => status, headers: () => (location ? { location } : {}) });
it('resolves each wrapper through the context request client — search referer, redirects off — and hands back the real address', async () => {
const get = vi.fn(async (u) => answer(302, u.endsWith('A') ? 'https://www.reddit.com/r/nocode/comments/1mbctla/whats_the_best/' : 'https://www.reddit.com/r/SideProject/comments/1rvddm1/tested/'));
const results = [{ title: 'a', url: '', via: 'https://www.google.com/goto?url=A', source: 'Reddit · r/nocode', snippet: '' },
{ title: 'b', url: '', via: 'https://www.google.com/goto?url=B', source: '', snippet: '' },
{ title: 'c', url: 'https://example.nl/', source: '', snippet: '' }];
const n = await resolveWrapped({ request: { get } }, results, 'https://www.google.com/search?q=x');
expect(n).toBe(2);
expect(results[0].url).toBe('https://www.reddit.com/r/nocode/comments/1mbctla/whats_the_best/');
expect(results[1].url).toMatch(/SideProject/);
expect(results.every((r) => !('via' in r))).toBe(true);
expect(get).toHaveBeenCalledTimes(2); // the plain result costs nothing
expect(get.mock.calls[0][1]).toMatchObject({ maxRedirects: 0, headers: { referer: 'https://www.google.com/search?q=x' } });
});
it('a wrapper that will not resolve keeps the wrapper as its address and says why — never silently vanishes', async () => {
const get = vi.fn(async () => answer(400));
const results = [{ title: 'a', url: '', via: 'https://www.google.com/goto?url=A', source: '', snippet: '' }];
await resolveWrapped({ request: { get } }, results, 'https://www.google.com/');
expect(results[0].url).toBe('https://www.google.com/goto?url=A');
expect(results[0].unresolved).toMatch(/HTTP 400/);
});
it('search() resolves wrappers in place, so the agent sees real addresses and the count is right', async () => {
const get = vi.fn(async () => answer(302, 'https://www.reddit.com/r/nocode/comments/1mbctla/x/'));
const page = pageThat([{ title: 'a', url: '', via: 'https://www.google.com/goto?url=A', source: 'Reddit · r/nocode', snippet: 's' },
{ title: 'a again', url: '', via: 'https://www.google.com/goto?url=B', source: '', snippet: 's' }],
{ request: { get } });
const r = await search(page, 'site:reddit.com builder', { settle: 0 });
expect(r.results).toHaveLength(1); // two wrappers, one page
expect(r.results[0].url).toBe('https://www.reddit.com/r/nocode/comments/1mbctla/x/');
expect(r.error).toBeUndefined();
});
});
/*
* AN ADDRESS TYPED FROM MEMORY IS NOT A SOURCE. With google() dead, the same pass "remembered"
* thread addresses and dug them; Reddit redirected each invented id to whatever post owns it, and
* four unrelated threads were quoted as evidence. Reddit rewrites a thread's slug to its real title,
* so a link that was actually seen keeps its slug and a composed one does not.
*/
describe('a dig that lands somewhere else than it asked', () => {
it('reddit: a different post id, or a slug rewritten to another title, is refused as not-a-real-link', () => {
expect(landedElsewhere('https://www.reddit.com/r/nocode/comments/1abc123/best_ai_app_builder_self_hosted/',
'https://www.reddit.com/r/nocode/comments/1abc123/my_cat_learned_to_open_doors/')).toMatch(/not a real link/);
expect(landedElsewhere('https://old.reddit.com/r/nocode/comments/1abc123/best_ai_app_builder/',
'https://www.reddit.com/r/SideProject/comments/1zzz999/best_ai_app_builder/')).toMatch(/DIFFERENT thread/);
});
it('reddit: the ordinary redirects a real link goes through are fine', () => {
// old → www, trailing-slash, a slug Reddit merely extended, or a query string
expect(landedElsewhere('https://old.reddit.com/r/nocode/comments/1abc123/best_ai_app_builder/',
'https://www.reddit.com/r/nocode/comments/1abc123/best_ai_app_builder_for_startups/?rdt=1')).toBe('');
expect(landedElsewhere('https://www.reddit.com/r/nocode/comments/1abc123/',
'https://www.reddit.com/r/nocode/comments/1abc123/best_ai_app_builder/')).toBe('');
expect(landedElsewhere('https://example.nl/pricing', 'https://example.nl/pricing/', 'Pricing')).toBe('');
});
it('a "page not found" title is refused; an article about 404 pages is not', () => {
expect(landedElsewhere('https://x.nl/a', 'https://x.nl/a', 'Page not found')).toMatch(/does not exist/);
expect(landedElsewhere('https://x.nl/a', 'https://x.nl/a', '404 Not Found')).toMatch(/does not exist/);
expect(landedElsewhere('https://x.nl/a', 'https://x.nl/a', 'How we redesigned our 404 page and cut bounce rate by 30% — a long engineering write-up about not found pages and what error copy should say')).toBe('');
});
it('readPage() refuses to read the wrong thread and names the composed address in the error', async () => {
const page = {
goto: vi.fn(async () => {}),
url: () => 'https://www.reddit.com/r/nocode/comments/1abc123/completely_other_thread/',
title: async () => 'Completely other thread : r/nocode',
evaluate: vi.fn(async () => ({ title: 'x', description: '', text: 'not this', emails: [], phones: [], length: 8 })),
};
const r = await readPage(page, 'https://www.reddit.com/r/nocode/comments/1abc123/best_ai_app_builder_self_hosted/', { settle: 0 });
expect(r.elsewhere).toBe(true);
expect(r.error).toMatch(/not a real link/);
expect(page.evaluate).not.toHaveBeenCalled(); // the unrelated page is never read as evidence
});
});
describe('the consent wall', () => {
/*
* In Europe the first request is a full-page "before you continue" and no results at all. It is
* indistinguishable from a search that found nothing unless somebody says so — and a person can
* clear it in one click, but only if they know that is what they are looking at.
*/
it('is recognised by its address', () => {
expect(looksLikeConsent('', 'https://consent.google.com/m?continue=')).toBe(true);
});
it('is recognised by what it says', () => {
expect(looksLikeConsent('Voordat je verdergaat naar Google\nAlles accepteren', 'https://www.google.com/')).toBe(true);
expect(looksLikeConsent('Before you continue to Google — Accept all', 'https://www.google.com/')).toBe(true);
});
it('is not confused with a page that merely mentions accepting something', () => {
expect(looksLikeConsent('x'.repeat(6000) + ' accept all cookies', 'https://example.com')).toBe(false);
});
it('is reported as itself, with what to do about it', async () => {
const page = pageThat([], { text: 'Voordat je verdergaat naar Google — alles accepteren',
at: 'https://consent.google.com/m' });
const r = await search(page, 'anything', { settle: 0 });
expect(r.consent).toBe(true);
expect(r.error).toMatch(/Accept it once in the live view/);
});
});
describe('searching', () => {
const hit = (n) => ({ title: `Result ${n}`, url: `https://example${n}.nl/`, snippet: `about thing ${n}` });
it('hands back the results as data', async () => {
const page = pageThat([hit(1), hit(2)]);
const r = await search(page, 'autobedrijf alkmaar', { settle: 0 });
expect(r.results).toHaveLength(2);
expect(page.goto.mock.calls[0][0]).toContain('q=autobedrijf%20alkmaar');
});
it('can ask for recent results only, using Google’s own parameter', async () => {
const page = pageThat([hit(1)]);
await search(page, 'x', { recentDays: 7, settle: 0 });
expect(page.goto.mock.calls[0][0]).toContain('tbs=qdr:w');
});
/* An exception reads exactly like a search that found nothing, and that lie is expensive. */
it('says it could not read the page rather than returning nothing', async () => {
const page = { goto: async () => {}, url: () => 'https://www.google.com/',
evaluate: async () => { throw new Error('detached'); } };
const r = await search(page, 'x', { settle: 0 });
expect(r.error).toMatch(/could not read the results/);
});
it('escapes what people actually type', () => {
expect(url.search('garage "geen website" -marktplaats'))
.toContain('q=garage%20%22geen%20website%22%20-marktplaats');
});
});
describe('following a result', () => {
it('reads the page and hands back what it says', async () => {
const page = {
goto: vi.fn(async () => {}),
url: () => 'https://example.nl/contact',
evaluate: async () => ({ title: 'Contact', description: '', text: 'Bel ons op 072 123 4567',
emails: ['info@example.nl'], phones: ['072 123 4567'], length: 24 }),
};
const r = await readPage(page, 'https://example.nl/contact', { settle: 0 });
expect(r).toMatchObject({ title: 'Contact', emails: ['info@example.nl'] });
expect(page.goto).toHaveBeenCalledWith('https://example.nl/contact', expect.anything());
});
it('reports a page it could not read instead of pretending it was empty', async () => {
const page = { goto: async () => {}, url: () => 'x', evaluate: async () => { throw new Error('nope'); } };
const r = await readPage(page, 'https://example.nl', { settle: 0 });
expect(r.error).toMatch(/could not read that page/);
});
});