scripts/update-bot-patterns.js

Rama main · réplica de lectura del repositorio original.

const fs = require("fs");
const path = require("path");

const SOURCE = "https://raw.githubusercontent.com/omrilotan/isbot/main/src/patterns.json";
const BROWSER_FIXTURES = "https://raw.githubusercontent.com/omrilotan/isbot/main/fixtures/browsers.yml";
const DIR = path.join(__dirname, "..", "backend", "src", "bot_detection");
const ENFORCED_FILE = path.join(DIR, "bot_patterns.txt");
const SHADOW_FILE = path.join(DIR, "bot_patterns_shadow.txt");
const LOCAL_FILE = path.join(DIR, "bot_patterns_local.txt");

function readPatterns(file) {
  return fs
    .readFileSync(file, "utf8")
    .split(/\r?\n/)
    .filter((line) => line && !line.startsWith("#"));
}

async function main() {
  const res = await fetch(SOURCE);
  if (!res.ok) throw new Error(`Failed to fetch patterns: ${res.status}`);
  const patterns = await res.json();

  const invalid = patterns.filter((p) => typeof p !== "string" || p.includes("\n") || p.startsWith("#"));
  if (invalid.length > 0) throw new Error(`Unsupported patterns: ${JSON.stringify(invalid)}`);

  // Rust's regex parser rejects a bare '-' following a class escape inside [...];
  // escaping the hyphen is semantics-preserving in both JS and Rust
  const upstream = patterns.map((p) => p.replace(/\\([wdsWDS])-/g, "\\$1\\-"));
  const upstreamSet = new Set(upstream);

  // Enforced patterns must still exist upstream
  const enforcedSet = new Set(readPatterns(ENFORCED_FILE));
  const gone = [...enforcedSet].filter((p) => !upstreamSet.has(p));
  if (gone.length > 0) {
    throw new Error(
      `Enforced patterns no longer exist upstream — remove them or move them to ${path.basename(LOCAL_FILE)}:\n` +
        gone.join("\n"),
    );
  }

  // Unpromoted upstream patterns go to shadow
  const enforced = upstream.filter((p) => enforcedSet.has(p));
  const shadow = upstream.filter((p) => !enforcedSet.has(p));
  const local = readPatterns(LOCAL_FILE);

  // Guard against upstream regressions and local mistakes: no pattern in any tier
  // may match a real browser UA from upstream's human-browser fixture corpus
  const fixturesRes = await fetch(BROWSER_FIXTURES);
  if (!fixturesRes.ok) throw new Error(`Failed to fetch browser fixtures: ${fixturesRes.status}`);
  const humanUas = (await fixturesRes.text())
    .split("\n")
    .filter((line) => line.startsWith("  - "))
    .map((line) => line.slice(4).trim())
    .filter(Boolean);
  if (humanUas.length < 100) throw new Error(`Browser fixture corpus suspiciously small: ${humanUas.length}`);

  for (const [tier, tierPatterns] of [["enforced", enforced], ["local", local], ["shadow", shadow]]) {
    const combined = new RegExp(tierPatterns.join("|"), "i");
    const falsePositives = humanUas.filter((ua) => combined.test(ua));
    if (falsePositives.length > 0) {
      throw new Error(
        `Refusing to write: ${falsePositives.length} human browser UAs match the ${tier} tier:\n` +
          falsePositives.slice(0, 10).join("\n"),
      );
    }
  }
  console.log(`Verified 0 false positives against ${humanUas.length} human browser UAs`);

  const date = new Date().toISOString().slice(0, 10);
  const enforcedHeader = [
    "# Enforced bot user-agent patterns: the curated subset of the vendored isbot list",
    "# (omrilotan/isbot, Unlicense) that names a specific product, vendor, or bot class.",
    "# The upstream remainder runs in shadow (bot_patterns_shadow.txt); promote a pattern",
    "# by moving its line here once bot_events shadow data shows it only hits bots.",
    `# Source: ${SOURCE}`,
    `# Generated: ${date} (${enforced.length} of ${upstream.length} upstream patterns enforced)`,
    "# Regenerate with: node scripts/update-bot-patterns.js",
  ];
  const shadowHeader = [
    "# Shadow bot user-agent patterns, generated by scripts/update-bot-patterns.js — do",
    "# not hand-edit. Upstream isbot patterns not promoted in bot_patterns.txt: format",
    "# heuristics and generic words that could conceivably match a real client. Hits are",
    "# recorded to bot_events as shadow:ua-heuristic; the event is processed as human.",
    `# Source: ${SOURCE}`,
    `# Generated: ${date} (${shadow.length} patterns)`,
  ];
  fs.writeFileSync(ENFORCED_FILE, enforcedHeader.join("\n") + "\n" + enforced.join("\n") + "\n");
  fs.writeFileSync(SHADOW_FILE, shadowHeader.join("\n") + "\n" + shadow.join("\n") + "\n");

  console.log(
    `Wrote ${enforced.length} enforced + ${shadow.length} shadow patterns (${local.length} local additions untouched)`,
  );
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});