agentbuilds.js
Rama master · réplica de lectura del repositorio original.
'use strict';
// Default agent release downloads: the pinned release manifest, the verified download cache and a
// local copy of each new agent release. Agent architecture definitions stay in agenttypes.js so new
// agents can be added there.
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const dns = require('dns');
const net = require('net');
const http = require('http');
const https = require('https');
const tls = require('tls');
const { Transform } = require('stream');
const { pipeline } = require('stream/promises');
// HTTPS downloads and GitHub API access with private-address guards
const httpsDownloads = (function () {
const blocked = new net.BlockList();
for (const [address, prefix] of [['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.88.99.0', 24], ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24], ['224.0.0.0', 3]]) {
blocked.addSubnet(address, prefix, 'ipv4');
blocked.addSubnet('64:ff9b::' + address, 96 + prefix, 'ipv6');
}
const global6 = new net.BlockList();
global6.addSubnet('2000::', 3, 'ipv6');
// DNS64 on IPv6-only networks answers with the IPv4 address inside the NAT64 well-known prefix. Those count
// as public when the IPv4 address does, which the blocked list above checks.
global6.addSubnet('64:ff9b::', 96, 'ipv6');
for (const [address, prefix] of [['2001::', 23], ['2001:db8::', 32], ['2002::', 16], ['3fff::', 20]]) blocked.addSubnet(address, prefix, 'ipv6');
function publicAddress(address) {
const family = net.isIP(address);
return family === 4 ? !blocked.check(address, 'ipv4') : family === 6 && global6.check(address, 'ipv6') && !blocked.check(address, 'ipv6');
}
function parseUrl(value) {
if (typeof value !== 'string' || value.length > 4096) throw new Error('Enter a public HTTPS download URL.');
let url;
try { url = new URL(value); } catch (ex) { throw new Error('Enter a public HTTPS download URL.'); }
if (url.protocol !== 'https:' || url.username || url.password || url.hash || (url.port && url.port !== '443')) throw new Error('Use HTTPS on port 443 without a username, password or fragment.');
const host = url.hostname.replace(/^\[|\]$/g, '');
if (net.isIP(host) && !publicAddress(host)) throw new Error('Downloads from private or reserved addresses are not allowed.');
return url;
}
function sourceUrl(value) {
const url = parseUrl(value);
url.search = '';
return url.href;
}
function createClient(transport = https, lookup = dns.lookup, proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY || process.env.https_proxy || process.env.http_proxy) {
// A forward proxy, from npmProxy in config.json or from the environment, carries each request through a
// CONNECT tunnel. The proxy resolves the host itself, so the address checks on DNS answers do not apply behind it.
function tunnel(options, callback) {
let target;
try { target = new URL(proxy); } catch (ex) { return callback(new Error('The proxy URL is not valid.')); }
const host = options.hostname || options.host, authority = (net.isIPv6(host) ? '[' + host + ']' : host) + ':' + options.port, headers = { Host: authority };
if (target.username) headers['Proxy-Authorization'] = 'Basic ' + Buffer.from(decodeURIComponent(target.username) + ':' + decodeURIComponent(target.password)).toString('base64');
const secure = target.protocol === 'https:';
const request = (secure ? https : http).request({ host: target.hostname.replace(/^\[|\]$/g, ''), port: target.port || (secure ? 443 : 80), method: 'CONNECT', path: authority, headers, agent: false, timeout: 10000 });
request.on('connect', (res, socket) => {
if (res.statusCode !== 200) { socket.destroy(); return callback(new Error('The proxy refused the connection (HTTP ' + res.statusCode + ').')); }
callback(null, tls.connect({ socket, host, servername: net.isIP(host) ? undefined : host }));
});
request.on('timeout', () => request.destroy(new Error('The proxy timed out.')));
request.on('error', err => callback(new Error(/proxy/.test(err.message) ? err.message : 'Unable to reach the proxy' + (/^[A-Z0-9_]+$/.test(err.code) ? ' (' + err.code + ')' : '') + '.')));
request.end();
}
async function response(value, options, redirects = 0, authenticated = true) {
const url = parseUrl(value), hostname = url.hostname.replace(/^\[|\]$/g, '');
const headers = { 'User-Agent': 'MeshCentral-AgentBuilds', Accept: options.json ? 'application/vnd.github+json' : 'application/octet-stream', 'Accept-Encoding': 'identity' };
if (url.hostname === 'api.github.com') {
headers['X-GitHub-Api-Version'] = '2022-11-28';
if (/\/actions\/artifacts\/\d+\/zip$/.test(url.pathname)) headers.Accept = 'application/vnd.github+json';
if (options.token && authenticated) headers.Authorization = 'Bearer ' + options.token;
if (options.etag) headers['If-None-Match'] = options.etag;
}
const res = await new Promise((resolve, reject) => {
let timer, connecting;
const request = transport.get(url, { headers, agent: proxy ? undefined : false, createConnection: proxy ? tunnel : undefined, defaultPort: 443, signal: options.signal, lookup: function (host, settings, callback) {
lookup(host, { all: true, verbatim: true }, (err, addresses) => {
if (err || !addresses || !addresses.length) return callback(new Error('Unable to resolve the download host.'));
if (addresses.some(x => !publicAddress(x.address))) return callback(new Error('Downloads from private or reserved addresses are not allowed.'));
// Return the checked addresses to the socket so a second lookup cannot change the destination. All
// of them, so a host without IPv4 still reaches a DNS64 address after the IPv4 one fails.
callback(null, settings.all ? addresses : addresses[0].address, addresses[0].family);
});
} }, res => { clearTimeout(timer); clearTimeout(connecting); resolve(res); });
timer = setTimeout(() => request.destroy(new Error('Download timed out.')), 20000);
// A firewall that drops packets leaves the connection pending rather than refusing it, so connecting
// gets less time than a slow response.
connecting = setTimeout(() => request.destroy(new Error('Connection timed out.')), 10000);
request.once('socket', socket => socket.once('secureConnect', () => clearTimeout(connecting)));
request.setTimeout(20000, () => request.destroy(new Error('Download timed out.')));
request.on('error', err => {
clearTimeout(timer); clearTimeout(connecting);
if (err.name === 'AbortError') return reject(Object.assign(new Error('Download cancelled or timed out.'), { name: 'AbortError' }));
// Nothing came back, so other downloads from this host will fail the same way.
const code = /^[A-Z0-9_]+$/.test(err.code) ? ' (' + err.code + ')' : '';
reject(Object.assign(new Error(/private or reserved|resolve the download|timed out|proxy/.test(err.message) ? err.message : 'Unable to download from this host' + code + '.'), { unreachable: true }));
});
});
if ([301, 302, 303, 307, 308].includes(res.statusCode)) {
res.destroy();
if (redirects >= 5 || options.json) throw new Error('Too many redirects or an unexpected API redirect.');
const next = parseUrl(new URL(res.headers.location || '', url).href);
return response(next.href, options, redirects + 1, authenticated && next.origin === url.origin);
}
if (res.statusCode === 304 && options.conditional) {
res.destroy();
return { res, url: sourceUrl(url.href) };
}
if (res.statusCode !== 200) {
res.destroy();
if (options.conditional && (res.statusCode === 403 || res.statusCode === 429)) {
const error = new Error('GitHub denied the release check or its rate limit was reached. The next check has been delayed.');
error.retryAt = Math.min(Date.now() + 7 * 86400000, Math.max(Date.now() + 3600000, (Number(res.headers['x-ratelimit-reset']) || 0) * 1000, Date.now() + (Number(res.headers['retry-after']) || 0) * 1000));
throw error;
}
let message = 'Download failed (HTTP ' + res.statusCode + ').';
if (res.statusCode === 401 || res.statusCode === 403) message = 'Download denied or API rate limit reached. Check the GitHub token and repository read permissions.';
if (res.statusCode === 404) message = 'File or GitHub resource not found, or the token cannot access it.';
if (res.statusCode === 410) message = 'This GitHub artifact has expired. Select a newer build.';
const error = new Error(message);
error.statusCode = res.statusCode;
throw error;
}
if ((res.headers['content-encoding'] && res.headers['content-encoding'] !== 'identity') || Number(res.headers['content-length']) > options.limit) {
res.destroy(); throw new Error('Download exceeds the size limit or uses unsupported HTTP compression.');
}
return { res, url: sourceUrl(url.href) };
}
async function download(url, filename, options) {
const { res, url: finalUrl } = await response(url, options), hash = crypto.createHash('sha256');
let size = 0;
const meter = new Transform({ transform(chunk, encoding, callback) {
size += chunk.length;
if (size > options.limit) return callback(new Error('Download exceeds the size limit.'));
hash.update(chunk);
if (options.progress) options.progress(size, Number(res.headers['content-length']) || null);
callback(null, chunk);
} });
await pipeline(res, meter, fs.createWriteStream(filename, { flags: 'wx', mode: 0o600 }), { signal: options.signal });
const sha256 = hash.digest('hex');
if (options.sha256 && sha256 !== options.sha256) throw new Error('The downloaded SHA256 does not match the expected digest.');
return { url: sourceUrl(url), finalUrl, size, sha256, digestVerified: !!options.sha256 };
}
async function readJsonBody(res) {
const chunks = []; let length = 0;
try {
for await (const chunk of res) {
length += chunk.length;
if (length > 4 * 1024 * 1024) throw new Error('GitHub response exceeds the size limit.');
chunks.push(chunk);
}
return JSON.parse(Buffer.concat(chunks).toString('utf8'));
} catch (ex) { res.destroy(); throw new Error('Unable to read the GitHub response.'); }
}
async function getJson(url, token, signal) {
const { res } = await response(url, { json: true, token, signal, limit: 4 * 1024 * 1024 });
return readJsonBody(res);
}
async function getJsonIfChanged(url, etag, signal) {
const { res } = await response(url, { json: true, conditional: true, etag, signal, limit: 4 * 1024 * 1024 });
return { unchanged: res.statusCode === 304, etag: res.headers.etag, data: res.statusCode === 304 ? null : await readJsonBody(res) };
}
return { download, getJson, getJsonIfChanged };
}
return { publicAddress, parseUrl, sourceUrl, createClient };
})();
// Release folders: one per tag under meshcentral-data/agentbuilds, with a SHA256SUMS listing every verified file
const releaseFolder = (function () {
const writes = new Map();
function hashes(filename) {
return new Promise((resolve, reject) => {
const sha256 = crypto.createHash('sha256'), sha384 = crypto.createHash('sha384');
let size = 0;
fs.createReadStream(filename).on('data', chunk => { size += chunk.length; sha256.update(chunk); sha384.update(chunk); })
.on('error', reject).on('end', () => resolve({ size, sha256: sha256.digest('hex'), sha384: sha384.digest('hex') }));
});
}
async function directory(filename) {
await fs.promises.mkdir(filename, { recursive: true, mode: 0o700 });
if (!(await fs.promises.lstat(filename)).isDirectory()) throw new Error('Invalid agent release folder.');
}
async function sums(folder) {
const result = new Map();
let text;
try { text = await fs.promises.readFile(path.join(folder, 'SHA256SUMS'), 'utf8'); } catch (ex) { if (ex.code === 'ENOENT') return result; throw ex; }
for (const line of text.split('\n')) {
const match = /^([a-f0-9]{64}) [ *](\S+)$/.exec(line.trim());
if (match) result.set(match[2], match[1]);
}
return result;
}
// Downloads run a few at a time, so updates to one folder's listing are queued behind each other.
function record(folder, filename, sha256) {
const previous = writes.get(folder) || Promise.resolve();
const task = previous.catch(() => {}).then(async function () {
const entries = await sums(folder);
entries.set(filename, sha256);
const temp = path.join(folder, '.' + crypto.randomBytes(16).toString('hex') + '.tmp');
await fs.promises.writeFile(temp, Array.from(entries, ([name, hash]) => hash + ' ' + name + '\n').join(''), { flag: 'wx', mode: 0o600 });
await fs.promises.rename(temp, path.join(folder, 'SHA256SUMS'));
});
writes.set(folder, task);
return task.finally(() => { if (writes.get(folder) === task) writes.delete(folder); });
}
// Free space on the volume holding the folder, or null when the runtime cannot say.
async function freeSpace(folder) {
if (typeof fs.promises.statfs !== 'function') return null;
const stat = await fs.promises.statfs(folder);
return Number(stat.bavail) * Number(stat.bsize);
}
function reserve(settings) {
const mib = Number.isInteger(settings.minfreespacemib) && settings.minfreespacemib >= 0 && settings.minfreespacemib <= 1048576 ? settings.minfreespacemib : 256;
return mib * 1048576;
}
async function ensureSpace(folder, bytes, reserved, probe) {
const free = await (probe || freeSpace)(folder);
const mib = value => Math.ceil(value / 1048576);
if (free != null && free < bytes + reserved) throw new Error('Not enough free disk space: ' + mib(bytes) + ' MiB to download, ' + mib(free) + ' MiB free, ' + mib(reserved) + ' MiB kept in reserve.');
}
return { hashes, directory, sums, record, freeSpace, reserve, ensureSpace };
})();
// Scheduled GitHub release checks, and a local copy of each new release for administrators to try
function CreateAgentReleaseUpdates(settings, client, entries, options = {}) {
const hours = Number.isInteger(settings.checkintervalhours) && settings.checkintervalhours >= 0 && settings.checkintervalhours <= 168 ? settings.checkintervalhours : 24;
const enabled = settings.enabled !== false;
const fetching = (settings.fetchreleases !== false) && (typeof options.root === 'string');
const prereleases = settings.includeprereleases === true;
const keep = Number.isInteger(settings.keepreleases) && settings.keepreleases >= 0 && settings.keepreleases <= 100 ? settings.keepreleases : 3;
const names = options.names || new Set(), pinned = options.pinned || new Set(), log = options.log || function () { }, reserved = releaseFolder.reserve(settings);
const repositories = new Map();
for (const file of entries) {
if (!repositories.has(file.repository)) repositories.set(file.repository, { repository: file.repository, tags: new Set(), files: new Set(), fetched: new Set(), failures: 0, nextCheck: 0 });
const repository = repositories.get(file.repository);
repository.tags.add(file.tag); repository.files.add(file.filename);
}
let pending = null, timer = null, stopped = false;
function version(tag) {
const match = /^v?(\d{1,6})\.(\d{1,6})\.(\d{1,6})$/.exec(tag);
return match ? match.slice(1).map(Number) : null;
}
function compare(a, b) {
for (let i = 0; i < 3; i++) { if (a[i] !== b[i]) return a[i] - b[i]; }
return 0;
}
function safeName(value) { return typeof value === 'string' && /^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$/.test(value); }
function latest(releases, repository) {
if (!Array.isArray(releases)) throw new Error('Invalid GitHub release response.');
// Releases from before the fixed asset names, such as the versioned-only Android APKs, carry none of these files.
const candidates = releases.filter(release => release && !release.draft && !release.prerelease && version(release.tag_name) &&
Array.isArray(release.assets) && release.assets.some(asset => asset && asset.size > 0 && repository.files.has(asset.name)));
candidates.sort((a, b) => compare(version(b.tag_name), version(a.tag_name)));
if (!candidates.length) return null;
const release = candidates[0], tag = release.tag_name;
const files = release.assets.filter(asset => asset && safeName(asset.name) && asset.size > 0).map(asset => asset.name);
const current = Array.from(repository.tags).map(version).filter(Boolean);
return { tag, sourceUrl: 'https://github.com/' + repository.repository + '/releases/tag/' + encodeURIComponent(tag),
publishedAt: release.published_at, availableFiles: Array.from(repository.files).filter(name => files.includes(name)),
updateAvailable: !current.length || current.some(value => compare(version(tag), value) > 0) };
}
// Releases worth a local copy: published, carrying at least one agent file under its served name, and not
// already pinned as a default. Only the newest few, so a fresh server does not fetch years of history.
function candidates(releases, repository) {
if (!Array.isArray(releases)) return [];
return releases.filter(release => release && !release.draft && (prereleases || !release.prerelease) && safeName(release.tag_name) && !repository.tags.has(release.tag_name) &&
Array.isArray(release.assets) && release.assets.some(asset => asset && asset.size > 0 && names.has(asset.name)))
.sort((a, b) => (Date.parse(b.published_at) || 0) - (Date.parse(a.published_at) || 0)).slice(0, keep || undefined);
}
// Copies that are no longer among the newest for any repository are removed. Pinned defaults are never
// touched, nor is a folder without a listing, which is not ours. Nothing is removed until every
// repository has answered, or a repository that is down would lose its copies.
async function prune() {
if (!keep) return;
const wanted = new Set();
for (const repository of repositories.values()) {
if (!repository.releases) return;
for (const release of candidates(repository.releases, repository)) wanted.add(release.tag_name);
}
let entries;
try { entries = await fs.promises.readdir(options.root, { withFileTypes: true }); } catch (ex) { return; }
for (const entry of entries) {
if (!entry.isDirectory() || !safeName(entry.name) || pinned.has(entry.name) || wanted.has(entry.name)) continue;
const listing = await fs.promises.lstat(path.join(options.root, entry.name, 'SHA256SUMS')).catch(() => null);
if (!listing || !listing.isFile()) continue;
await fs.promises.rm(path.join(options.root, entry.name), { recursive: true, force: true });
log('Agent release ' + entry.name + ' removed, keeping the newest ' + keep + ' releases');
}
}
// A file counts as present once it matches its SHA256SUMS line, so an interrupted copy keeps what it has
// and fetches the rest. Returns the folder when anything was downloaded.
async function pickup(repository, release, signal) {
const target = path.join(options.root, release.tag_name);
const listed = await releaseFolder.sums(target), missing = [];
for (const asset of release.assets) {
if (!asset || !names.has(asset.name) || !(asset.size > 0) || (asset.size > 128 * 1024 * 1024)) continue;
const digest = (typeof asset.digest === 'string' && /^sha256:[a-f0-9]{64}$/.test(asset.digest)) ? asset.digest.slice(7) : undefined;
const info = listed.has(asset.name) ? await releaseFolder.hashes(path.join(target, asset.name)).catch(() => null) : null;
if (info && (info.size === asset.size) && (info.sha256 === listed.get(asset.name)) && (!digest || info.sha256 === digest)) continue;
missing.push({ asset, digest });
}
if (!missing.length) return false;
await releaseFolder.directory(target);
await releaseFolder.ensureSpace(target, missing.reduce((total, item) => total + item.asset.size, 0), reserved, options.freeSpace);
for (const { asset, digest } of missing) {
const filename = path.join(target, asset.name);
const temp = path.join(target, '.' + crypto.randomBytes(16).toString('hex') + '.tmp');
try {
await client.download('https://github.com/' + repository.repository + '/releases/download/' + encodeURIComponent(release.tag_name) + '/' + encodeURIComponent(asset.name), temp, { limit: asset.size, signal, sha256: digest });
const fetched = await releaseFolder.hashes(temp);
if (fetched.size !== asset.size) throw new Error(asset.name + ': the downloaded size does not match the release.');
await fs.promises.chmod(temp, 0o600);
await fs.promises.rename(temp, filename);
await releaseFolder.record(target, asset.name, fetched.sha256);
} finally { await fs.promises.unlink(temp).catch(() => {}); }
}
return target;
}
function status() {
return { enabled, intervalHours: hours, fetching, prereleases, checking: !!pending, repositories: Array.from(repositories.values(), item => ({
repository: item.repository, tags: Array.from(item.tags), latest: item.latest || null, fetched: Array.from(item.fetched),
lastCheck: item.lastCheck, lastSuccess: item.lastSuccess, nextCheck: item.nextCheck, error: item.error || item.copyError || null
})) };
}
function schedule() {
if (timer) clearTimeout(timer);
timer = null;
if (stopped || !enabled || !hours || !repositories.size) return;
const next = Math.min(...Array.from(repositories.values(), item => item.nextCheck));
timer = setTimeout(() => { check(false).catch(() => {}); }, Math.max(1000, next - Date.now()));
timer.unref();
}
function check(manual) {
if (pending) return pending;
if (!enabled || stopped) return Promise.resolve();
pending = (async function () {
const controller = new AbortController(), deadline = setTimeout(() => controller.abort(), 120000);
try {
for (const repository of repositories.values()) {
if (repository.retryAt > Date.now() || (repository.lastCheck && Date.now() - repository.lastCheck < 60000) || (!manual && repository.nextCheck > Date.now())) continue;
repository.lastCheck = Date.now();
try {
const response = await client.getJsonIfChanged('https://api.github.com/repos/' + repository.repository + '/releases?per_page=100', repository.etag, controller.signal);
if (!response.unchanged) { repository.latest = latest(response.data, repository); repository.releases = response.data; }
repository.etag = response.etag || repository.etag;
repository.lastSuccess = Date.now(); repository.error = null; repository.failures = 0;
} catch (ex) {
repository.error = ex.message; repository.failures++;
repository.retryAt = ex.retryAt || 0;
}
const delay = repository.failures ? Math.min(24 * 3600000, 3600000 * Math.pow(2, Math.min(5, repository.failures - 1))) : (hours || 24) * 3600000;
repository.nextCheck = Math.max(repository.retryAt || 0, Date.now() + delay + Math.floor(Math.random() * 300000));
}
} finally { clearTimeout(deadline); }
if (!fetching) return;
// Copies get their own deadline: a release is tens of megabytes and the API check above is not.
const copies = new AbortController(), limit = setTimeout(() => copies.abort(), 600000);
try {
for (const repository of repositories.values()) {
// A copy that failed last time is retried from the cached release list, so its error clears on success.
repository.copyError = null;
for (const release of candidates(repository.releases, repository)) {
if (repository.fetched.has(release.tag_name)) continue;
try {
const target = await pickup(repository, release, copies.signal);
repository.fetched.add(release.tag_name);
if (target) log('Agent release ' + repository.repository + ' ' + release.tag_name + ' downloaded to ' + target);
} catch (ex) { repository.copyError = release.tag_name + ': ' + ex.message; }
}
}
await prune().catch(ex => log('Unable to remove old agent release copies: ' + ex.message));
} finally { clearTimeout(limit); }
})().finally(() => { pending = null; schedule(); });
return pending;
}
function start() {
if (timer || stopped) return;
for (const repository of repositories.values()) repository.nextCheck = Date.now() + 30000 + Math.floor(Math.random() * 120000);
schedule();
}
function stop() { stopped = true; if (timer) clearTimeout(timer); timer = null; }
return { start, stop, check, status };
}
// Default agent release downloads
function CreateAgentDefaults(parent, options = {}) {
const root = path.join(parent.datapath, 'agentbuilds');
const bundled = options.directory || path.join(__dirname, 'agents');
// npmProxy comes ahead of the environment, as it does for npm and the time stamp server.
const npmProxy = (parent.args && (typeof parent.args.npmproxy == 'string') && parent.args.npmproxy) ? parent.args.npmproxy : undefined;
const client = options.client || httpsDownloads.createClient(undefined, undefined, npmProxy), log = options.log || (message => console.log(message));
const settings = (parent.config.settings || {}).agentdownloads || {};
const names = new Set(Object.values(parent.meshAgentsArchitectureNumbers).map(x => x.localname));
for (const name of ['MeshCentralRouter.exe', 'MeshCentralRouter.dmg', 'MeshCentralAssistant.exe']) names.add(name);
const ready = new Map(), entries = new Map(), rejected = new Set();
const mismatch = 'The release file does not match its expected size and SHA384.';
let busy = null, errors = [], loaded = false, activating = null, updates = null, activePaths = null, announced = false, retryTimer = null, retries = 0;
function regular(filename) {
try { return fs.lstatSync(filename).isFile(); } catch (ex) { return false; }
}
// Where a name resolves right now: a verified release file, else a local or bundled file for names
// outside the manifest. Nothing outside the manifest stands in for a release file that is not ready.
function current(name) {
if (ready.has(name)) return ready.get(name);
if (entries.has(name)) return null;
const local = path.join(root, name);
if (regular(local)) return local;
const original = path.join(bundled, name);
return regular(original) ? original : null;
}
function sourcePath(name) {
if (!names.has(name)) return null;
return activePaths ? activePaths.get(name) : current(name);
}
function changed() {
return Array.from(names).some(name => activePaths.get(name) !== current(name));
}
// Swap in what changed since the paths were frozen. The server calls this at the start of an
// activation pass so signing and the table rebuild see the same files.
function activate() {
if (!activePaths || !changed()) return false;
activePaths = new Map(Array.from(names, name => [name, current(name)]));
return true;
}
function notify() {
if (typeof parent.activateAgentDefaults !== 'function') { activate(); return; }
const task = new Promise(resolve => parent.activateAgentDefaults(resolve)).catch(() => {}).finally(() => { if (activating === task) activating = null; });
activating = task;
}
function validate(manifest) {
if (!manifest || manifest.schemaVersion !== 1 || !Array.isArray(manifest.releases) || manifest.releases.length > 32) throw new Error('Invalid default agent release manifest.');
const files = new Map(), skipped = new Set();
for (const release of manifest.releases) {
if (!release || typeof release.repository !== 'string' || !/^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/.test(release.repository) ||
typeof release.tag !== 'string' || !/^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$/.test(release.tag) ||
!Array.isArray(release.files) || !release.files.length) throw new Error('Specify a repository, release tag and files for each default release.');
const base = 'https://github.com/' + release.repository + '/releases/';
for (const file of release.files) {
if (!file || typeof file.filename !== 'string' || !/^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$/.test(file.filename)) throw new Error('Invalid default release file name in ' + release.tag + '.');
// Newer releases can carry agent types this server does not know. Nothing would serve them, so they
// are left out instead of failing the whole manifest.
if (!names.has(file.filename)) { skipped.add(file.filename); continue; }
if (files.has(file.filename)) throw new Error(file.filename + ' is listed more than once in the default releases.');
if (typeof file.asset !== 'string' || !/^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$/.test(file.asset) ||
!Number.isSafeInteger(file.size) || file.size <= 0 || file.size > 128 * 1024 * 1024 ||
typeof file.sha384 !== 'string' || !/^[a-f0-9]{96}$/.test(file.sha384)) throw new Error('Invalid asset, size or SHA384 for default release file ' + file.filename + '.');
files.set(file.filename, {
filename: file.filename, asset: file.asset, size: file.size, sha384: file.sha384,
repository: release.repository, tag: release.tag, sourceUrl: base + 'tag/' + encodeURIComponent(release.tag),
url: base + 'download/' + encodeURIComponent(release.tag) + '/' + encodeURIComponent(file.asset), status: 'Not checked'
});
}
}
if (!files.size) throw new Error('The default releases list no agent files this server uses.');
if (skipped.size) log('Ignoring ' + Array.from(skipped).join(', ') + ' in the default agent releases: no agent type on this server uses ' + (skipped.size === 1 ? 'that file name.' : 'those file names.'));
return files;
}
async function read(filename) {
try {
const stat = await fs.promises.lstat(filename);
if (!stat.isFile() || stat.size > 1024 * 1024) throw new Error('Invalid default agent release manifest.');
return validate(JSON.parse(await fs.promises.readFile(filename, 'utf8')));
} catch (ex) { if (ex.code === 'ENOENT') throw ex; throw new Error(filename + ': ' + ex.message); }
}
async function load() {
const shipped = path.resolve(bundled, 'agent-defaults.json');
let custom = settings.manifest ? path.resolve(parent.datapath, settings.manifest) : null, files;
if (custom === shipped) custom = null;
// check-releases.js passes the shipped manifest as the custom one, from a folder that holds no copy of it.
try { files = await read(shipped); } catch (ex) { if (!custom || ex.code !== 'ENOENT') throw ex; files = new Map(); }
// Laid over the shipped pins file by file, so a release that carries only some agents, like a testing
// build, does not leave the others with nothing to serve.
if (custom) for (const [name, file] of await read(custom)) files.set(name, file);
entries.clear();
for (const [name, file] of files) entries.set(name, file);
loaded = true;
updates = CreateAgentReleaseUpdates(settings, client, entries.values(), { root: root, names: names, pinned: new Set(Array.from(entries.values(), file => file.tag)), freeSpace: options.freeSpace, log: message => console.log(message) });
if (activePaths) updates.start();
}
async function verified(filename, file) {
let handle;
try {
const entry = await fs.promises.lstat(filename);
if (!entry.isFile() || entry.size !== file.size) return false;
handle = await fs.promises.open(filename, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0) | (fs.constants.O_NONBLOCK || 0));
const stat = await handle.stat();
if (!stat.isFile() || stat.size !== file.size) return false;
const hash = crypto.createHash('sha384');
for await (const chunk of handle.createReadStream({ autoClose: false })) hash.update(chunk);
const after = await handle.stat();
return hash.digest('hex') === file.sha384 && stat.size === after.size && stat.mtimeMs === after.mtimeMs && stat.ctimeMs === after.ctimeMs;
} catch (ex) { if (ex.code === 'ENOENT' || ex.code === 'ELOOP') return false; throw ex; }
finally { if (handle) await handle.close(); }
}
async function directory(filename) {
await fs.promises.mkdir(filename, { recursive: true, mode: 0o700 });
if (!(await fs.promises.lstat(filename)).isDirectory()) throw new Error('Invalid default agent cache directory.');
}
async function resolve(file, signal, localFiles) {
const cache = path.join(root, file.tag), target = path.join(cache, file.filename);
if (await verified(target, file)) {
// A file supplied by hand is listed on first use, so SHA256SUMS covers the whole folder.
if (!(await releaseFolder.sums(cache)).has(file.filename)) await releaseFolder.record(cache, file.filename, (await releaseFolder.hashes(target)).sha256);
file.status = 'Ready'; return target;
}
let local;
if (await verified(path.join(bundled, file.filename), file)) local = path.join(bundled, file.filename);
const imported = localFiles && localFiles.find(x => x.sha384 === file.sha384 && x.size === file.size);
if (!local && imported && await verified(imported.path, file)) local = imported.path;
if (!local && settings.enabled === false) return null;
// Checked only now so files already on disk still load after the pass has given up on GitHub.
if (!local) signal.throwIfAborted();
// A published asset that failed verification would fail again, so it is fetched once per start.
if (!local && rejected.has(file.filename)) throw new Error(mismatch);
await directory(root); await directory(cache);
await releaseFolder.ensureSpace(cache, file.size, releaseFolder.reserve(settings), options.freeSpace);
const temp = path.join(cache, '.' + crypto.randomBytes(16).toString('hex') + '.tmp');
try {
if (local) {
await fs.promises.copyFile(local, temp, fs.constants.COPYFILE_EXCL);
} else {
announce();
file.status = 'Downloading'; file.received = 0;
try {
await client.download(file.url, temp, { limit: file.size, signal, progress: received => { file.received = received; } });
} catch (ex) {
if (ex.statusCode === 404) throw new Error('Release asset not found. Publish the configured release and asset before downloading it.');
if (ex.statusCode === 401 || ex.statusCode === 403) throw new Error('GitHub denied the public release download. Check that the release is public or retry later.');
throw ex;
}
}
if (!await verified(temp, file)) { if (!local) rejected.add(file.filename); throw new Error(mismatch); }
await fs.promises.chmod(temp, 0o600);
await fs.promises.rename(temp, target);
await releaseFolder.record(cache, file.filename, (await releaseFolder.hashes(target)).sha256);
file.status = 'Ready';
return target;
} finally { await fs.promises.unlink(temp).catch(() => {}); }
}
function status() {
return {
busy: !!busy, activating: !!activating, enabled: settings.enabled !== false, errors: errors.slice(),
updates: updates ? updates.status() : null,
files: Array.from(entries.values(), ({ url, ...file }) => ({ ...file }))
};
}
// One line when the first download starts, so a slow link explains a quiet startup. Background retries
// stay quiet until their files load.
function announce() {
if (announced) return;
announced = true;
const waiting = Array.from(entries.values()).filter(file => file.status !== 'Ready');
log('Downloading ' + waiting.length + ' default agent files (' + Math.ceil(waiting.reduce((total, file) => total + file.size, 0) / 1048576) + ' MiB) from GitHub releases...');
}
// One warning for all the files that could not be fetched, with the release pages to get them from and
// the folders they go in.
function missingWarning(files, cause) {
const folders = new Map();
for (const file of files) {
if (!folders.has(file.tag)) folders.set(file.tag, new Set());
folders.get(file.tag).add(file.sourceUrl);
}
const one = files.length === 1, what = (!one && files.length === entries.size) ? 'the ' + files.length + ' default agent files' : files.map(file => file.filename).join(', ');
const where = (one ? 'it' : 'them') + ' from ' + Array.from(folders, ([tag, pages]) => Array.from(pages).join(' and ') + ' into "' + path.join(root, tag) + '"').join(', and from ') + '.';
if (!cause) return 'Agent downloads are disabled and ' + what + (one ? ' is' : ' are') + ' missing. Download ' + where;
return 'Unable to download ' + what + ' from GitHub: ' + cause.message + ' Retrying in the background. On a server without internet access, download ' + where;
}
// Missing files are looked for again in the background, so a network that was down at startup or files
// copied in by hand need no restart. The wait doubles from a minute up to an hour.
function retry() {
clearTimeout(retryTimer); retryTimer = null;
if (!activePaths || Array.from(entries.keys()).every(name => ready.has(name) || rejected.has(name))) { retries = 0; return; }
retryTimer = setTimeout(() => { retryTimer = null; prepare(); }, Math.min(3600000, 60000 * Math.pow(2, retries++)));
retryTimer.unref();
}
// Releases before 1.3.0 signed and served whatever sat in the package's agents folder, and a custom
// file left there now counts for nothing unless it matches the pins. The release CLI runs without a
// server, and a mismatch in the folder it is pointed at already fails that run.
async function strays() {
if (typeof parent.activateAgentDefaults !== 'function') return;
for (const [name, file] of entries) {
const local = path.join(bundled, name);
try { if (!regular(local) || await verified(local, file)) continue; } catch (ex) { continue; }
errors.push('Ignoring "' + local + '": it does not match the pinned release file. Custom agents belong in "' + path.join(parent.datapath, 'agents') + '".');
}
for (const type of Object.values(parent.meshAgentsArchitectureNumbers)) {
const local = path.join(bundled, type.localname);
if ((type.base == null) || !regular(local)) continue;
errors.push('Ignoring "' + local + '": ' + type.localname + ' is built from ' + type.base + ' and meshcmd.js. Custom agents belong in "' + path.join(parent.datapath, 'agents') + '".');
}
}
function prepare(localFiles) {
if (busy) return busy;
busy = (async function () {
errors = [];
try { if (!loaded) await load(); }
catch (ex) { errors.push('Unable to read default agent releases: ' + ex.message); return; }
await strays();
const controller = new AbortController(), timer = setTimeout(() => controller.abort(new Error('Downloads did not finish within 10 minutes.')), 600000);
const missing = new Set();
let cause = null;
try {
const pending = Array.from(entries.values()).filter(file => !ready.has(file.filename));
await Promise.all(Array.from({ length: Math.min(3, pending.length) }, async function () {
while (pending.length) {
const file = pending.shift();
delete file.error; delete file.received;
try {
const target = await resolve(file, controller.signal, localFiles);
if (target) { ready.set(file.filename, target); continue; }
file.status = 'Unavailable'; file.error = 'Automatic downloads are disabled. Supply this release file locally.';
missing.add(file);
} catch (ex) {
// Without GitHub every other download would wait out the same timeout, so the rest are stopped
// and report why rather than their cancellation.
if (ex.unreachable && !controller.signal.aborted) controller.abort(ex);
const stopped = controller.signal.aborted && (ex === controller.signal.reason || ex.name === 'AbortError');
const reason = ex.unreachable ? ex : stopped ? controller.signal.reason : null;
file.status = 'Unavailable'; file.error = (reason || ex).message;
if (reason) { cause = cause || reason; missing.add(file); } else errors.push(file.filename + ': ' + ex.message);
}
}
}));
} finally { clearTimeout(timer); }
// Listed in manifest order, not the order the downloads happened to fail in.
if (missing.size) errors.unshift(missingWarning(Array.from(entries.values()).filter(file => missing.has(file)), cause));
// Files that landed after start() froze the paths are switched in by the server, not by a restart.
if (activePaths && changed()) notify();
retry();
})().finally(() => { busy = null; });
return busy;
}
function info(name) {
const file = entries.get(name);
if (!file || !ready.has(name) || (activePaths && activePaths.get(name) !== ready.get(name))) return null;
return { repository: file.repository, tag: file.tag, sourceUrl: file.sourceUrl, sha384: file.sha384 };
}
function start() {
// Signing and table loading must see the same files after the startup download wait ends.
if (!activePaths) activePaths = new Map(Array.from(names, name => [name, current(name)]));
if (updates) updates.start();
if (!busy) retry();
}
return { prepare, status, sourcePath, info, start, activate, checkUpdates: () => updates ? updates.check(true) : Promise.resolve() };
}
module.exports = { CreateAgentDefaults, CreateAgentReleaseUpdates, httpsDownloads };