scan-seek.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* scan-seek.mjs — SEEK NZ keyword scanner over the SEO path form.
*
* Fetches https://nz.seek.com/<keyword>-jobs[/in-<Location>] and reads the
* job records out of the page's inlined `window.SEEK_REDUX_DATA` (id, title,
* advertiser, location, salary label, listing date, teaser, work types).
*
* Unlike `providers/jobstreet.mjs` (which queries `/api/jobsearch/v5/search`
* for known companies in `companies:`), this scanner discovers roles by
* keyword across NZ-domestic employers and recruiters with no public ATS
* board. It never fetches a `/job/<id>` detail page: the posting URL is built
* for the pipeline, not requested during the scan.
*
* PAGINATION & FRESHNESS (#6694). Paginates `?page=N` (from page 2 onward) up
* to `max_pages` per search (default 5, cap 50, or `--pages N` on the CLI),
* stopping early when a page yields no new postings. `--daterange N` (or
* `daterange: N` on an entry) appends `daterange=N` so results are scoped to
* the last N days of ads. (Until #6691, query strings were omitted to stay
* inside `nz.seek.com/robots.txt`'s `*?` disallow; rule 6 no longer treats
* `robots.txt` as a gate.)
*
* Config (portals.yml): `seek_searches`, a list of
* `{ keyword, location?, daterange?, max_pages? }`.
* title_filter, location_filter and data/blacklist.md apply as in scan.mjs.
*/
try {
const { config } = await import('dotenv');
config();
} catch (e) {}
import { readFileSync, existsSync } from 'fs';
import { loadYaml } from './lib/yaml-document.mjs';
import { appendToPipeline, appendToScanHistory, PORTALS_PATH } from './scan.mjs';
import { localToday } from './lib/local-today.mjs';
import { isMainModule } from './lib/is-main-module.mjs';
import { printScanSummaryHeader } from './lib/scan-summary-marker.mjs';
import { validateFlags, flagValue } from './lib/cli-flags.mjs';
import { makeHttpCtx, fetchTextWithRetry } from './providers/_http.mjs';
import {
sleep, slugify, decodeHtml,
createLocalGate, formatLocalSkips, writeRunLog,
} from './scan-local-lib.mjs';
const KNOWN_FLAGS = ['--help', '-h', '--daterange', '--pages', '--max-pages', '--delay-ms', '--dry-run'];
const VALUE_FLAGS = ['--daterange', '--pages', '--max-pages', '--delay-ms'];
const USAGE = `Usage:
node scan-seek.mjs # scan seek_searches from portals.yml
node scan-seek.mjs --daterange 7 # restrict to the last 7 days (?daterange=7)
node scan-seek.mjs --pages 3 # fetch up to 3 pages per search (?page=N)
node scan-seek.mjs --dry-run # print matches, write nothing
Reads seek_searches from portals.yml:
seek_searches:
- keyword: "data analyst"
location: "Wellington" # optional; omit for all of NZ
daterange: 7 # optional; last N days
max_pages: 5 # optional; default 5 (cap 50)
Fetches the SEO path (nz.seek.com/<keyword>-jobs[/in-<Location>]) and reads
the inlined window.SEEK_REDUX_DATA results, paging ?page=N until a page yields
no new postings or max_pages is reached.
Flags:
--daterange N append daterange=N (last N days of ads)
--pages N max pages per search (default 5, cap 50; alias --max-pages)
--delay-ms N pause between requests (default 2000)
--dry-run do not touch pipeline, scan history or scan-runs
--help, -h show this help and exit`;
const BASE = 'https://nz.seek.com';
export const DEFAULT_MAX_PAGES = 5;
export const MAX_PAGES_CAP = 50;
function coercePositiveInt(v) {
if (typeof v === 'number' && Number.isSafeInteger(v) && v >= 1) return v;
if (typeof v === 'string' && /^[1-9]\d*$/.test(v.trim())) {
const n = Number(v.trim());
if (Number.isSafeInteger(n)) return n;
}
return v;
}
function parsePositiveInt(val, name) {
const n = coercePositiveInt(val);
if (typeof n === 'number' && Number.isSafeInteger(n) && n >= 1) return n;
throw new Error(`Invalid ${name} ${JSON.stringify(val)}; must be a positive integer`);
}
export function loadSearches(portals) {
const list = Array.isArray(portals?.seek_searches) ? portals.seek_searches : [];
return list
.map((s) => (typeof s === 'string' ? { keyword: s } : s))
.filter((s) => s && typeof s.keyword === 'string' && s.keyword.trim())
.map((s) => {
const entry = { keyword: s.keyword.trim(), location: (s.location || '').trim() };
const maxPages = s.max_pages ?? s.pages;
if (maxPages !== undefined && maxPages !== null && maxPages !== '') {
entry.max_pages = coercePositiveInt(maxPages);
}
if (s.daterange !== undefined && s.daterange !== null && s.daterange !== '') {
entry.daterange = coercePositiveInt(s.daterange);
}
return entry;
});
}
const NZ_LOCATION_SLUGS = {
auckland: 'in-All-Auckland',
wellington: 'in-All-Wellington',
canterbury: 'in-All-Canterbury',
christchurch: 'in-All-Christchurch-Canterbury',
waikato: 'in-All-Waikato',
hamilton: 'in-All-Hamilton-Waikato',
'bay-of-plenty': 'in-All-Bay-of-Plenty',
tauranga: 'in-All-Tauranga-Bay-of-Plenty',
rotorua: 'in-All-Rotorua-Bay-of-Plenty',
otago: 'in-All-Otago',
dunedin: 'in-All-Dunedin-Otago',
queenstown: 'in-Queenstown-Otago',
manawatu: 'in-All-Manawatu',
'palmerston-north': 'in-All-Palmerston-North-Manawatu',
'hawkes-bay': 'in-All-Hawkes-Bay',
napier: 'in-All-Napier-Hawkes-Bay',
taranaki: 'in-All-Taranaki',
'new-plymouth': 'in-All-New-Plymouth-Taranaki',
northland: 'in-All-Northland',
whangarei: 'in-All-Whangarei-Northland',
southland: 'in-All-Southland',
invercargill: 'in-All-Invercargill-Southland',
nelson: 'in-All-Nelson-Tasman',
tasman: 'in-All-Tasman',
marlborough: 'in-All-Marlborough',
gisborne: 'in-All-Gisborne',
'west-coast': 'in-All-West-Coast',
masterton: 'in-Masterton-Wellington',
};
/**
* Resolve a configured location ("Wellington", "Masterton", or "in-All-Wellington")
* to SEEK's canonical `in-...` path segment, or throw if an unmapped plain
* location is passed instead of guessing a broken `in-All-...` URL.
*/
export function formatSeekLocation(location) {
const raw = String(location ?? '')
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.trim();
if (!raw) return '';
if (/^in-[A-Za-z0-9-]+$/.test(raw)) {
return raw.replace(/^in-(all-)?/i, (_, all) => (all ? 'in-All-' : 'in-'));
}
const key = slugify(raw);
if (key === 'new-zealand' || key === 'nz') return '';
if (NZ_LOCATION_SLUGS[key]) return NZ_LOCATION_SLUGS[key];
throw new Error(`Unknown SEEK NZ location ${JSON.stringify(location)}; use a known NZ region/city or a canonical "in-..." SEEK path slug`);
}
/**
* SEO path URL for one search. `daterange` (positive integer days) and `page`
* (1-based; omitted on page 1) are appended as query parameters when set.
* Accepts either `(search, daterange, page)` or `(search, { daterange, page })`.
*/
export function buildSeekUrl({ keyword, location, daterange: entryDaterange }, optsOrDaterange = null, pageArg = 1) {
const kw = slugify(keyword);
if (!kw) throw new Error(`Empty keyword: ${JSON.stringify(keyword)}`);
const loc = formatSeekLocation(location);
const isOpts = optsOrDaterange !== null && typeof optsOrDaterange === 'object';
const daterange = isOpts ? (optsOrDaterange.daterange ?? entryDaterange ?? null) : (optsOrDaterange ?? entryDaterange ?? null);
const page = isOpts ? (optsOrDaterange.page ?? 1) : (pageArg ?? 1);
const params = new URLSearchParams();
if (daterange !== null && daterange !== undefined && daterange !== '') {
const days = parsePositiveInt(daterange, 'daterange');
params.set('daterange', String(days));
}
if (page !== null && page !== undefined && page !== 1) {
const p = parsePositiveInt(page, 'page');
if (p > 1) params.set('page', String(p));
}
const base = `${BASE}/${kw}-jobs${loc ? `/${loc}` : ''}`;
const qs = params.toString();
return qs ? `${base}?${qs}` : base;
}
/** Text of the JSON object literal assigned to window.SEEK_REDUX_DATA, or null. */
export function extractReduxJson(html) {
const marker = html.indexOf('window.SEEK_REDUX_DATA');
if (marker === -1) return null;
const start = html.indexOf('{', marker);
if (start === -1) return null;
let depth = 0;
let inStr = false;
for (let i = start; i < html.length; i++) {
const c = html[i];
if (inStr) {
if (c === '\\') i++;
else if (c === '"') inStr = false;
} else if (c === '"') inStr = true;
else if (c === '{') depth++;
else if (c === '}' && --depth === 0) return html.slice(start, i + 1);
}
return null;
}
function* walkJobs(node) {
if (Array.isArray(node)) {
for (const x of node) yield* walkJobs(x);
} else if (node && typeof node === 'object') {
if (node.id !== undefined && typeof node.title === 'string' && node.advertiser) yield node;
else for (const v of Object.values(node)) yield* walkJobs(v);
}
}
function toArray(v) {
return Array.isArray(v) ? v : v ? [v] : [];
}
/**
* Job records of one results page, normalized to scanner offers. The records
* are located by shape (id + title + advertiser) rather than by a fixed path
* into the Redux tree, so a rearranged tree does not silently return nothing.
*/
export function parseSeekPage(html) {
const raw = extractReduxJson(html);
if (!raw) throw new Error('SEEK_REDUX_DATA not found — page may be blocked or layout changed');
let data;
try {
// Redux dumps serialize JS undefined as a bare token.
data = JSON.parse(raw.replace(/:\s*undefined\b/g, ':null'));
} catch (err) {
throw new Error(`SEEK_REDUX_DATA is not valid JSON: ${err.message}`);
}
const seen = new Set();
const offers = [];
for (const j of walkJobs(data)) {
const id = String(j.id);
if (seen.has(id)) continue;
seen.add(id);
const date = typeof j.listingDate === 'string' ? j.listingDate.slice(0, 10) : '';
offers.push({
id,
url: `${BASE}/job/${id}`, // recorded, never fetched during scan
title: decodeHtml(j.title),
company: decodeHtml(j.advertiser?.description ?? j.companyName ?? ''),
location: decodeHtml(j.location ?? j.locationLabel ?? ''),
salary: decodeHtml(j.salaryLabel ?? ''),
teaser: decodeHtml(j.teaser ?? ''),
workTypes: toArray(j.workTypes).map(String),
postedAt: /^\d{4}-\d{2}-\d{2}$/.test(date) ? date : '',
source: 'seek-nz',
});
}
return offers;
}
/**
* Page through one SEEK search (`?page=1..maxPages`), stopping early when a
* page returns no unseen job IDs.
*/
export async function scanSeekSearch(
search,
{
fetchText,
daterange = null,
maxPages = null,
delayMs = 0,
sleepImpl = sleep,
} = {},
) {
const rawMax = maxPages ?? search?.max_pages ?? DEFAULT_MAX_PAGES;
const pages = parsePositiveInt(rawMax, 'max_pages');
const pageLimit = Math.min(pages, MAX_PAGES_CAP);
const effectiveDaterange = daterange ?? search?.daterange ?? null;
const seenIds = new Set();
const collected = [];
let lastUrl = search?.keyword ?? '';
for (let page = 1; page <= pageLimit; page++) {
try {
lastUrl = buildSeekUrl(search, effectiveDaterange, page);
if (page > 1 && delayMs > 0) await sleepImpl(delayMs);
const html = await fetchText(lastUrl);
const offers = parseSeekPage(html);
const fresh = offers.filter((o) => !seenIds.has(o.id));
if (fresh.length === 0) break;
for (const o of fresh) {
seenIds.add(o.id);
collected.push(o);
}
} catch (err) {
if (err && typeof err === 'object' && !err.seekUrl) err.seekUrl = lastUrl;
if (page === 1) throw err;
collected.incomplete = true;
collected.error = err;
break;
}
}
return collected;
}
function loadPortals() {
if (!existsSync(PORTALS_PATH)) return {};
try {
return loadYaml(readFileSync(PORTALS_PATH, 'utf-8')) || {};
} catch (err) {
console.error(`Error: failed to parse ${PORTALS_PATH}: ${err.message}`);
process.exit(1);
}
}
async function main(args) {
const dryRun = args.includes('--dry-run');
const daterangeRaw = flagValue(args, '--daterange');
if (daterangeRaw !== undefined && !/^[1-9]\d*$/.test(daterangeRaw)) {
console.error('Error: --daterange needs a positive whole number of days');
process.exit(1);
}
const daterange = daterangeRaw !== undefined ? Number(daterangeRaw) : null;
for (const flag of ['--pages', '--max-pages']) {
const raw = flagValue(args, flag);
if (raw !== undefined && !/^[1-9]\d*$/.test(raw)) {
console.error(`Error: ${flag} needs a positive whole number of pages`);
process.exit(1);
}
}
const pagesRaw = flagValue(args, '--pages') ?? flagValue(args, '--max-pages');
const maxPages = pagesRaw !== undefined ? Number(pagesRaw) : null;
const delayRaw = flagValue(args, '--delay-ms');
const delayMs = delayRaw !== undefined ? Number(delayRaw) : 2000;
if (delayRaw !== undefined && (delayRaw.trim() === '' || !Number.isFinite(delayMs) || delayMs < 0)) {
console.error('Error: --delay-ms needs a non-negative number');
process.exit(1);
}
const portals = loadPortals();
const searches = loadSearches(portals);
if (searches.length === 0) {
console.error('No seek_searches in portals.yml. Add entries like { keyword: "data analyst", location: "Wellington" }.');
process.exit(1);
}
const ctx = makeHttpCtx();
const gate = createLocalGate({ config: portals });
const added = [];
let found = 0;
let errors = 0;
for (let i = 0; i < searches.length; i++) {
if (i > 0) await sleep(delayMs);
try {
const offers = await scanSeekSearch(searches[i], {
fetchText: (url) => fetchTextWithRetry(ctx, url, { headers: { 'accept-language': 'en' } }),
daterange,
maxPages,
delayMs,
});
found += offers.length;
for (const offer of offers) {
if (gate.check(offer)) continue;
added.push(offer);
console.log(` ✅ ${offer.company} — ${offer.title}`);
}
if (offers.error) {
errors++;
console.warn(`⚠️ ${offers.error.seekUrl || searches[i].keyword}: ${offers.error.message}`);
}
} catch (err) {
errors++;
console.warn(`⚠️ ${err.seekUrl || searches[i].keyword}: ${err.message}`);
}
}
if (!dryRun) {
if (added.length > 0) {
await appendToPipeline(added);
await appendToScanHistory(added, localToday(), 'added');
}
await writeRunLog({ found, counts: gate.counts, newAdded: added.length, queries: searches.length, errors });
}
printScanSummaryHeader('SEEK NZ Scan', localToday());
console.log(`Queries: ${searches.length}`);
console.log(`Postings fetched: ${found}`);
console.log(`New offers: ${added.length}${dryRun ? ' (dry run, nothing written)' : ''}`);
const skipped = formatLocalSkips(gate.counts);
if (skipped) console.log(`Skipped: ${skipped}`);
if (errors) console.log(`Errors: ${errors} of ${searches.length} queries`);
if (errors === searches.length) process.exitCode = 1;
}
if (isMainModule(import.meta.url)) {
const args = process.argv.slice(2);
validateFlags(args, KNOWN_FLAGS, USAGE, { valueFlags: VALUE_FLAGS, requireOperand: true });
main(args).catch((err) => { console.error('Fatal:', err.message); process.exit(1); });
}