scan-interamt.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* scan-interamt.mjs — Interamt.de scanner via Playwright
*
* Interamt uses Apache Wicket (stateful Java framework) — no REST API exists.
* Playwright maintains a browser session so the wicket-crypt token and cookies
* are handled transparently.
*
* Reads `interamt_searches` from portals.yml. Falls back to a generic set of
* German IT keywords if the section is absent.
*
* Usage:
* node scan-interamt.mjs
* node scan-interamt.mjs --dry-run
* node scan-interamt.mjs --all # skip date filter (use for first scan)
* node scan-interamt.mjs --keyword "Softwareentwickler"
* node scan-interamt.mjs --debug # save a screenshot and the page HTML to output/
*/
import { chromium } from 'playwright';
import { readFileSync, existsSync, mkdirSync } from 'fs';
import { join } from 'path';
import { loadYaml } from './lib/yaml-document.mjs';
import {
appendToPipeline,
appendToScanHistory,
appendUnrecordedToScanHistory,
BROWSER_PROCESS_ENV,
DATA_ROOT as SCAN_DATA_ROOT,
PORTALS_PATH,
SCAN_HISTORY_PATH,
} from './scan.mjs';
import { createAdmissionGate } from './lib/admission-gate.mjs';
import { getCareerOpsRoot } from './path-resolver.mjs';
import { localToday } from './lib/local-today.mjs';
import { printScanSummaryHeader } from './lib/scan-summary-marker.mjs';
import { isMainModule } from './lib/is-main-module.mjs';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { scanHistoryDataLines } from './lib/scan-history-columns.mjs';
import { browserLaunchOptions } from './lib/browser-launch.mjs';
// ── Config ───────────────────────────────────────────────────────────
// Both imported from scan.mjs (#3510). This scanner appends through
// scan.mjs's appendToScanHistory — anchored to the data root — while reading its
// own bare-relative copy to work out the last scan date, so within a single run
// it could write one history file and read another. Its portals copy also
// ignored CAREER_OPS_PORTALS, which #2271 added so a second search lane does not
// poison the first one's dedup history.
const SCAN_HISTORY = SCAN_HISTORY_PATH;
const DATA_ROOT = getCareerOpsRoot();
const INTERAMT_HOME = 'https://interamt.de/koop/app/';
// Direct offer URL — constructed from StellenangebotId.
// Wicket adds a session version number (?28&id=...) during live navigation,
// but the bookmarkable form (?id=...) works without a session.
// If a specific offer fails to load from pipeline, open Interamt manually
// and navigate to it — Wicket will assign a valid version for that session.
const OFFER_BASE_URL = 'https://interamt.de/koop/app/stelle?id=';
// Generic fallback — configure interamt_searches in portals.yml for your target roles
const DEFAULT_KEYWORDS = [
'Informatiker',
'Softwareentwickler',
'IT-Spezialist',
'Datenwissenschaftler',
'Systemadministrator',
'Datenanalyst',
];
// ── Args ─────────────────────────────────────────────────────────────
// Checked by validateFlags() in the main-module block below, before main()
// runs (#4599). Without it, --help or a mistyped --dryrun fell through to a
// live scan that appended to data/pipeline.md.
const KNOWN_FLAGS = ['--dry-run', '--debug', '--all', '--keyword', '--help', '-h'];
const VALUE_FLAGS = ['--keyword'];
const USAGE = `Usage:
node scan-interamt.mjs
node scan-interamt.mjs --dry-run
node scan-interamt.mjs --all # skip date filter (use for first scan)
node scan-interamt.mjs --keyword "Softwareentwickler"
node scan-interamt.mjs --debug # save a screenshot and the page HTML to output/
node scan-interamt.mjs --help|-h # print this usage block and exit`;
const args = process.argv.slice(2);
const DRY_RUN = args.includes('--dry-run');
const DEBUG = args.includes('--debug');
const NO_DATE_FILTER = args.includes('--all');
// flagValue/hasFlag rather than indexOf: validateFlags accepts --keyword=value
// for a value flag, and indexOf() cannot see that form, so the scan would run
// every keyword instead of the one asked for.
const SINGLE_KEYWORD = flagValue(args, '--keyword') ?? null;
if (hasFlag(args, '--keyword') && (!SINGLE_KEYWORD || SINGLE_KEYWORD.startsWith('--'))) {
console.error('Error: --keyword requires a value, e.g. --keyword "Softwareentwickler"');
process.exit(1);
}
// ── Load portals.yml ─────────────────────────────────────────────────
let config = {};
if (existsSync(PORTALS_PATH)) {
config = loadYaml(readFileSync(PORTALS_PATH, 'utf-8')) || {};
}
const interamtSearches = config.interamt_searches || DEFAULT_KEYWORDS.map(k => ({ was: k }));
const keywords = SINGLE_KEYWORD
? [SINGLE_KEYWORD]
: interamtSearches.map(s => s.was).filter(Boolean);
// ── Filters ──────────────────────────────────────────────────────────
// title_filter, location_filter, data/blacklist.md and the seen set all go
// through the shared admission gate in main(), built from the same scan.mjs
// matchers every other scanner uses. The private `includes()` matchers that
// lived here read `word:Intern` as literal text, matched "KI" inside "Kita",
// never matched an AND group and ignored always_allow/block_hard (#5478).
// ── Date helpers ─────────────────────────────────────────────────────
// Parse Interamt date format DD.MM.YYYY → Date (midnight UTC)
function parseDE(str) {
if (!str) return null;
const [d, m, y] = str.trim().split('.');
if (!d || !m || !y) return null;
return new Date(`${y}-${m.padStart(2,'0')}-${d.padStart(2,'0')}T00:00:00Z`);
}
// Returns the most recent first_seen date for 'interamt' portal entries, or null
function loadLastScanDate() {
if (!existsSync(SCAN_HISTORY)) return null;
let latest = null;
scanHistoryDataLines(readFileSync(SCAN_HISTORY, 'utf-8')).forEach(line => {
const parts = line.split('\t');
if (parts[2] !== 'interamt') return;
const d = new Date((parts[1] || '') + 'T00:00:00Z');
if (!isNaN(d) && (!latest || d > latest)) latest = d;
});
return latest;
}
// ── Stable selectors (Wicket name attributes don't change between sessions) ──
const SEL_EDIT_BTN = 'a.ia-e-link--icon:has-text("Suchkriterien ändern"), a:has-text("Suchkriterien ändern")';
const SEL_NEXT_INPUT = 'input[name="stellensucheFilterAttributes.suchtextContainer:stellensucheFilterAttributes.suchText"]';
const SEL_NEXT_SUBMIT = 'button.ia-e-button--primary:has-text("Detailsuche")';
// ── Scraper ──────────────────────────────────────────────────────────
async function extractRows(page) {
return page.$$eval('tr.ia-e-table__row', rows =>
rows.map(row => {
const cell = field => row.querySelector(`td[data-field="${field}"]`)?.textContent?.trim() || '';
const title = cell('Stellenbezeichnung');
const id = row.querySelector('td[data-field="StellenangebotId"] span')?.textContent?.trim() || '';
const company = cell('Behoerde') || 'Interamt';
const modality = cell('Dienstort');
const city = cell('PLZOrte');
const publishedDate = cell('Von');
const deadline = cell('Bewerbungsfrist');
return { title, id, company, modality, city, publishedDate, deadline };
}).filter(r => r.title && r.id)
);
}
async function searchInteramt(page, keyword, isFirst) {
const found = [];
if (isFirst) {
// Initial search: navigate to homepage and use the landing form
await page.goto(INTERAMT_HOME, { waitUntil: 'networkidle', timeout: 30000 });
// Dismiss cookie modal — it has a backdrop overlay that blocks all clicks
const cookieModal = page.locator('#ia-m-cookie-modal');
const isModalVisible = await cookieModal.isVisible().catch(() => false);
if (isModalVisible) {
await page.locator('#ia-m-cookie-modal .ia-m-modal__close').click();
await cookieModal.waitFor({ state: 'hidden', timeout: 10000 }).catch(() => null);
}
await page.waitForSelector('#idOrSuchtext', { state: 'visible', timeout: 15000 });
await page.fill('#idOrSuchtext', keyword);
// #ida is Wicket auto-generated; fallback to class selector if ID changes
const submitBtn = page.locator('#ida, button.ia-e-button--primary.ia-e-button--icon').first();
await submitBtn.waitFor({ state: 'visible', timeout: 10000 });
await submitBtn.click();
} else {
// Subsequent search: use "Suchkriterien ändern" to reuse the Wicket session.
// Fall back to a fresh homepage load if the button isn't found (e.g. after an error).
const editBtn = page.locator(SEL_EDIT_BTN).first();
const editVisible = await editBtn.isVisible({ timeout: 5000 }).catch(() => false);
if (editVisible) {
// Dismiss any modal/backdrop that Wicket may have raised during the previous search
const anyModal = page.locator('.ia-m-modal__close').first();
if (await anyModal.isVisible({ timeout: 2000 }).catch(() => false)) {
await anyModal.evaluate(el => el.click());
await page.locator('.ia-m-modal__container').waitFor({ state: 'hidden', timeout: 5000 }).catch(() => null);
}
await page.locator('.ia-e-backdrop').waitFor({ state: 'hidden', timeout: 10000 }).catch(() => null);
await editBtn.evaluate(el => el.click());
await page.waitForSelector(SEL_NEXT_INPUT, { state: 'visible', timeout: 15000 });
await page.fill(SEL_NEXT_INPUT, keyword);
const submitBtn = page.locator(SEL_NEXT_SUBMIT);
await submitBtn.waitFor({ state: 'visible', timeout: 10000 });
await submitBtn.click();
} else {
// Fallback: reload homepage (no cookie modal on subsequent loads)
await page.goto(INTERAMT_HOME, { waitUntil: 'networkidle', timeout: 30000 });
await page.waitForSelector('#idOrSuchtext', { state: 'visible', timeout: 15000 });
await page.fill('#idOrSuchtext', keyword);
const submitBtn = page.locator('#ida, button.ia-e-button--primary.ia-e-button--icon').first();
await submitBtn.waitFor({ state: 'visible', timeout: 10000 });
await submitBtn.click();
}
}
// Wait for results table or "no results" message
await Promise.race([
page.waitForSelector('tr.ia-e-table__row', { timeout: 20000 }),
page.waitForSelector('.ia-e-no-results, .ia-no-results, [class*="no-result"]', { timeout: 20000 }),
]).catch(() => null);
if (DEBUG) {
// output/ is User Layer, so the debug dump follows the data root rather than
// appearing in whatever directory the scan was launched from (#3510).
const debugDir = join(DATA_ROOT, 'output');
const debugPng = join(debugDir, 'debug-interamt.png');
const debugHtml = join(debugDir, 'debug-interamt.html');
mkdirSync(debugDir, { recursive: true });
await page.screenshot({ path: debugPng, fullPage: true });
const { writeFileSync: wf } = await import('fs');
wf(debugHtml, await page.content());
console.log(` [debug] screenshot → ${debugPng}`);
console.log(` [debug] html → ${debugHtml}`);
console.log(' [debug] url:', page.url());
const rowCount = await page.$$eval('tr', rows => rows.length);
console.log(` [debug] total <tr> on page: ${rowCount}`);
}
// Wait for Wicket AJAX to fully settle — networkidle ensures no pending requests
await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => null);
// "mehr laden" appends rows to the same page — click until gone, then extract all at once
const loadMoreBtn = page.locator('button.ia-m-searchresults__btn-load').first();
let pageNum = 1;
while (pageNum <= 50) {
const visible = await loadMoreBtn.isVisible().catch(() => false);
if (!visible) break;
const beforeCount = await page.$$eval('tr.ia-e-table__row', rows => rows.length);
await loadMoreBtn.evaluate(el => el.click());
// Wait for backdrop to clear, then for new rows to appear
await page.locator('.ia-e-backdrop').waitFor({ state: 'hidden', timeout: 10000 }).catch(() => null);
await page.waitForFunction(
n => document.querySelectorAll('tr.ia-e-table__row').length > n,
beforeCount,
{ timeout: 15000 }
).catch(() => null);
pageNum++;
}
const rows = await extractRows(page);
for (const row of rows) {
found.push({
title: row.title,
url: `${OFFER_BASE_URL}${row.id}`,
company: row.company,
modality: row.modality,
city: row.city,
publishedDate: row.publishedDate,
deadline: row.deadline,
});
}
return found;
}
// ── Main ─────────────────────────────────────────────────────────────
async function main() {
mkdirSync(join(DATA_ROOT, 'data'), { recursive: true });
const gate = createAdmissionGate({ config });
const date = localToday();
const lastScanDate = NO_DATE_FILTER ? null : loadLastScanDate();
if (NO_DATE_FILTER) {
console.log(` --all: date filter disabled — fetching all available offers`);
} else if (lastScanDate) {
console.log(` Last Interamt scan: ${lastScanDate.toISOString().slice(0,10)} — skipping older offers`);
}
let totalFound = 0;
const newOffers = [];
const titleSkipped = [];
const locationSkipped = [];
const dateSkipped = [];
const dupeSkipped = [];
const errors = [];
// Searches overlap, so one offer often comes back under several keywords.
// Its first sighting already got the right verdict and scan-history row; a
// repeat must not add a skipped_dup row, which pins the URL, on top of a
// blacklist, location or date skip that is meant to be revisited later.
const handledThisRun = new Set();
const browser = await chromium.launch(browserLaunchOptions(
{ headless: true },
{ projectRoot: SCAN_DATA_ROOT, processEnv: BROWSER_PROCESS_ENV },
));
const context = await browser.newContext({ locale: 'de-DE', timezoneId: 'Europe/Berlin' });
const page = await context.newPage();
try {
for (let i = 0; i < keywords.length; i++) {
const kw = keywords[i];
process.stdout.write(` Searching "${kw}"... `);
try {
const found = await searchInteramt(page, kw, i === 0);
totalFound += found.length;
process.stdout.write(`${found.length} found\n`);
for (const offer of found) {
const location = [offer.modality, offer.city].filter(Boolean).join(' ')
+ (offer.deadline ? ` (Frist: ${offer.deadline})` : '');
const pubDate = parseDE(offer.publishedDate);
const canonical = {
url: offer.url,
company: offer.company,
title: offer.title,
location,
source: 'interamt',
postedAt: pubDate ? pubDate.getTime() : undefined,
};
if (handledThisRun.has(canonical.url)) continue;
handledThisRun.add(canonical.url);
const reason = gate.check(canonical);
if (reason === 'seen') { dupeSkipped.push(canonical); continue; }
// Blacklist skips are counted, not written to scan-history, as in
// scan.mjs: removing the entry later lets the posting through again.
if (reason === 'blacklisted') continue;
if (reason === 'title') { titleSkipped.push(canonical); continue; }
if (reason === 'location') { locationSkipped.push(canonical); continue; }
// Same-day offers pass: lastScanDate is the day of the last run, and an
// offer published later that same day should not be treated as stale.
if (lastScanDate && pubDate && pubDate < lastScanDate) { dateSkipped.push(canonical); continue; }
newOffers.push(canonical);
}
} catch (err) {
process.stdout.write(`ERROR\n`);
errors.push({ keyword: kw, error: err.message });
}
}
} finally {
await context.close();
await browser.close();
}
if (!DRY_RUN) {
if (newOffers.length > 0) await appendToPipeline(newOffers);
if (newOffers.length > 0) await appendToScanHistory(newOffers, date, 'added');
// Every search hands the same skipped postings back, so each is recorded
// once per status, not once per run (#2695, as scan.mjs records its own).
await appendUnrecordedToScanHistory(titleSkipped, date, 'skipped_title');
await appendUnrecordedToScanHistory(locationSkipped, date, 'skipped_location');
await appendUnrecordedToScanHistory(dateSkipped, date, 'skipped_date');
await appendUnrecordedToScanHistory(dupeSkipped, date, 'skipped_dup');
}
// Summary
printScanSummaryHeader('Interamt Scan', date);
console.log(`Keywords searched: ${keywords.length}`);
console.log(`Total found: ${totalFound}`);
console.log(`Filtered by title: ${titleSkipped.length}`);
console.log(`Filtered location: ${locationSkipped.length}`);
console.log(`Blacklisted: ${gate.counts.blacklisted}`);
console.log(`Filtered by date: ${dateSkipped.length}`);
console.log(`Duplicates: ${dupeSkipped.length}`);
console.log(`New offers: ${newOffers.length}`);
if (errors.length > 0) {
console.log(`\nErrors (${errors.length}):`);
for (const e of errors) console.log(` ✗ "${e.keyword}": ${e.error}`);
}
if (newOffers.length > 0) {
console.log('\nNew offers:');
for (const o of newOffers) {
console.log(` + ${o.company} | ${o.title} | ${o.location || 'N/A'}`);
}
if (DRY_RUN) {
console.log('\n(dry run — not saved)');
} else {
console.log(`\nSaved to data/pipeline.md`);
}
}
console.log('\n→ Run /career-ops pipeline to evaluate new offers.');
}
// Guarded like every sibling scanner (scan-hn.mjs:122, scan-ats-full.mjs:1115).
// Unguarded, merely IMPORTING this module drove a live browser scan of
// interamt.de and appended its results to the user's pipeline and scan history —
// so the file could not be imported by a test, or by anything else, without
// doing that. A module should not scan the internet as a side effect of being
// loaded (#3510).
if (isMainModule(import.meta.url)) {
validateFlags(args, KNOWN_FLAGS, USAGE, { valueFlags: VALUE_FLAGS, requireOperand: true });
main().catch(err => {
console.error('Fatal:', err.message);
process.exit(1);
});
}