frontend/build-products.mjs
Rama master · réplica de lectura del repositorio original.
// Build frontend/src/products.{json,tsx} from manifest.tsx files
import * as ps from 'child_process'
import { createHash } from 'crypto'
import fse from 'fs-extra'
import path from 'path'
import { cloneNode } from 'ts-clone-node'
import ts from 'typescript'
import { fileURLToPath } from 'url'
export const __dirname = path.dirname(fileURLToPath(import.meta.url))
// Manifest fields of lazy loaders, each generated into its own module so the shell reaches products by dynamic import.
const LAZY_REGISTRIES = [
{
field: 'activityDescribers',
file: 'productActivityDescribers.tsx',
imports:
"import type { Describer } from 'lib/components/ActivityLog/humanizeActivity'\n\nimport { ActivityScope } from '~/types'",
declaration: 'productActivityDescribers: Partial<Record<ActivityScope, () => Promise<Describer>>>',
},
{
field: 'globalModals',
file: 'productGlobalModals.tsx',
imports: "import type { ComponentType } from 'react'",
declaration: 'productGlobalModals: Record<string, () => Promise<ComponentType>>',
},
]
// Run directly
buildProductManifests()
function buildProductManifests() {
// 1. Scan for manifest files
const productsDir = path.join(__dirname, '../products')
const products = fse.readdirSync(productsDir).filter((p) => !['__pycache__', 'README.md'].includes(p))
const sourceFiles = products
.map((p) => path.resolve(productsDir, `${p}/manifest.tsx`))
.filter((p) => fse.existsSync(p))
.sort()
// Also include the schema file so we can resolve some enum values
const enumFile = path.resolve(__dirname, '../frontend/src/queries/schema/schema-general.ts')
const allSourceFiles = [...sourceFiles, enumFile]
const outputTsx = path.join(__dirname, 'src/products.tsx')
const outputScenesTsx = path.join(__dirname, 'src/productScenes.tsx')
const outputJson = path.join(__dirname, 'src/products.json')
const cacheDir = path.join(__dirname, '.cache')
const cacheFile = path.join(cacheDir, 'products.hash')
const buildScript = fileURLToPath(import.meta.url)
const hash = createHash('sha256')
// hash the build script first
hash.update(fse.readFileSync(buildScript))
// hash all input file content
for (const f of allSourceFiles) {
hash.update(fse.readFileSync(f))
}
const digest = hash.digest('hex')
// if we've already seen this hash digest, we have no recent changes
if (!process.env.BUILD_PRODUCTS_NO_CACHE) {
try {
if (
fse.readFileSync(cacheFile, 'utf8') === digest &&
fse.existsSync(outputTsx) &&
fse.existsSync(outputScenesTsx) &&
LAZY_REGISTRIES.every(({ file }) => fse.existsSync(path.join(__dirname, 'src', file))) &&
fse.existsSync(outputJson)
) {
return
}
} catch {
// Cache file missing or unreadable, so continue with build
}
}
const program = ts.createProgram(allSourceFiles, {
target: 1, // ES5
module: 1, // CommonJS
noEmit: true,
noErrorTruncation: true,
})
// 2. Gather manifest properties
const urls = []
const scenes = []
const sceneConfigs = []
const routes = []
const redirects = []
const fileSystemTypes = []
const setupProbes = []
const lazyRegistryEntries = Object.fromEntries(LAZY_REGISTRIES.map(({ field }) => [field, new Map()]))
const treeItemsNew = {}
const treeItemsGames = {}
const treeItemsMetadata = {}
const treeItemsProducts = {}
const visitManifests = (sourceFile) => {
const manifestSceneKeys = [] // collect the scene keys used in this manifest file
const manifestTreeItems = []
ts.forEachChild(sourceFile, function walk(node) {
if (ts.isPropertyAssignment(node) && ts.isObjectLiteralExpression(node.initializer)) {
const { text: name } = node.name
const list = {
urls,
routes,
redirects,
fileSystemTypes,
}[name]
if (list) {
node.initializer.properties.forEach((p) => list.push(cloneNode(p)))
} else if (name === 'scenes') {
node.initializer.properties.forEach((prop) => {
const sceneName = prop.name?.text ?? prop.name?.escapedText
if (sceneName && !manifestSceneKeys.includes(sceneName)) {
manifestSceneKeys.push(sceneName)
}
const imp = keepOnlyImport(prop, sourceFile.fileName)
if (imp) {
scenes.push(imp)
}
const cfg = withoutImport(prop)
if (cfg) {
sceneConfigs.push(cfg)
}
})
} else if (name in lazyRegistryEntries) {
const entries = lazyRegistryEntries[name]
node.initializer.properties.forEach((prop) => {
const key = prop.name.getText(sourceFile)
if (entries.has(key)) {
throw new Error(`${name} key ${key} appears in more than one manifest`)
}
entries.set(key, withImportPathsFromSrc(cloneNode(prop), sourceFile.fileName))
})
} else if (name === 'setupProbe') {
// One object per manifest, aggregated into the `productSetupProbes` array.
setupProbes.push(cloneNode(node.initializer))
} else {
ts.forEachChild(node, walk)
}
} else if (
ts.isPropertyAssignment(node) &&
ts.isArrayLiteralExpression(node.initializer) &&
['treeItemsNew', 'treeItemsProducts', 'treeItemsMetadata', 'treeItemsGames'].includes(node.name.text)
) {
// only annotate apps and data for now
const shouldAnnotate = node.name.text === 'treeItemsProducts' || node.name.text === 'treeItemsMetadata'
const dict =
node.name.text === 'treeItemsNew'
? treeItemsNew
: node.name.text === 'treeItemsProducts'
? treeItemsProducts
: node.name.text === 'treeItemsMetadata'
? treeItemsMetadata
: treeItemsGames
node.initializer.elements.forEach((el) => {
if (!ts.isObjectLiteralExpression(el)) {
return
}
const pathProp = el.properties.find((p) => p.name?.text === 'path')
const thePath = pathProp?.initializer?.text
if (thePath) {
const cloned = cloneNode(el)
if (shouldAnnotate) {
manifestTreeItems.push(cloned)
}
dict[thePath] = cloned
}
})
} else {
ts.forEachChild(node, walk)
}
})
// go through all tree items
manifestTreeItems.forEach((item) => {
// skip if the tree item already contains "sceneKeys"
if (item.properties.some((p) => p.name?.text === 'sceneKeys')) {
return
}
// add collected "sceneKeys" to the tree item
item.properties = ts.factory.createNodeArray([
...item.properties,
ts.factory.createPropertyAssignment(
ts.factory.createIdentifier('sceneKeys'),
ts.factory.createArrayLiteralExpression(
manifestSceneKeys.map((key) => ts.factory.createStringLiteral(key))
)
),
])
})
}
for (const sf of program.getSourceFiles()) {
if (sourceFiles.includes(sf.fileName)) {
visitManifests(sf)
}
}
// 3. Convert AST → printable code
const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed })
const makeObjExpr = (nodes) =>
printer.printNode(
ts.EmitHint.Unspecified,
ts.factory.createObjectLiteralExpression(nodes),
ts.createSourceFile('', '', ts.ScriptTarget.ESNext)
)
const makeArrExpr = (dict) =>
printer.printNode(
ts.EmitHint.Unspecified,
ts.factory.createArrayLiteralExpression(
Object.keys(dict)
.sort()
.map((k) => dict[k])
),
ts.createSourceFile('', '', ts.ScriptTarget.ESNext)
)
// 4. Harvest *all* imports from ASTs
const gathered = {}
const globalNames = new Set()
const addImport = (mod, kind, spec) => {
// Kludge
if ((mod === './types' || mod === '~/types') && spec === 'ProductManifest') {
return
}
if (!gathered[mod]) {
gathered[mod] = { default: null, namespace: null, named: new Set(), typeNamed: new Set() }
}
const entry = gathered[mod]
const localName =
kind === 'default'
? spec
: kind === 'namespace'
? spec
: spec.includes(' as ')
? spec.split(' as ').pop()
: spec
if (globalNames.has(localName)) {
return
}
globalNames.add(localName)
if (kind === 'default') {
entry.default = spec
} else if (kind === 'namespace') {
entry.namespace = spec
} else if (kind === 'named') {
entry.named.add(spec)
} else {
entry.typeNamed.add(spec)
}
}
for (const manifestPath of sourceFiles) {
const sf = program.getSourceFile(manifestPath)
sf.statements.filter(ts.isImportDeclaration).forEach((decl) => {
if (!ts.isStringLiteral(decl.moduleSpecifier)) {
return
}
const rawModule = decl.moduleSpecifier.text
const modulePath = rawModule.startsWith('.')
? path
.relative(path.resolve(__dirname, 'src'), path.resolve(path.dirname(manifestPath), rawModule))
.replace(/^[^.]/, (m) => `./${m}`)
: rawModule
const ic = decl.importClause
if (!ic) {
return
}
if (ic.name) {
addImport(modulePath, 'default', ic.name.text)
}
const pushSpecifiers = (list, typeOnly) => {
list.forEach((n) => {
const original = n.propertyName ? n.propertyName.text : n.name.text
const local = n.name.text
const alias = original === local ? original : `${original} as ${local}`
addImport(modulePath, typeOnly ? 'typeNamed' : 'named', alias)
})
}
if (ic.namedBindings) {
if (ts.isNamespaceImport(ic.namedBindings)) {
addImport(modulePath, 'namespace', ic.namedBindings.name.text)
} else if (ts.isNamedImports(ic.namedBindings)) {
pushSpecifiers(ic.namedBindings.elements, ic.isTypeOnly)
}
}
})
}
// Ensure critical helpers are always present
if (!globalNames.has('Params')) {
addImport('scenes/sceneTypes', 'typeNamed', 'Params')
}
if (!globalNames.has('FileSystemImport')) {
addImport('~/queries/schema/schema-general', 'typeNamed', 'FileSystemImport')
}
if (!globalNames.has('ProductSetupProbe')) {
addImport('lib/components/ProductEmptyState/setupProbes', 'typeNamed', 'ProductSetupProbe')
}
// 5. Serialise gathered imports → valid TypeScript code
// (no duplicate names, type/value kept separate)
const importLines = Object.entries(gathered)
.flatMap(([mod, spec]) => {
const lines = []
const named = [...spec.named].sort()
const typeNamed = [...spec.typeNamed].sort()
// value imports
if (spec.namespace) {
const head = spec.default
? `import ${spec.default}, * as ${spec.namespace} from '${mod}';`
: `import * as ${spec.namespace} from '${mod}';`
lines.push(head)
if (named.length) {
lines.push(`import { ${named.join(', ')} } from '${mod}';`)
}
} else {
if (spec.default && named.length) {
lines.push(`import ${spec.default}, { ${named.join(', ')} } from '${mod}';`)
} else if (spec.default) {
lines.push(`import ${spec.default} from '${mod}';`)
} else if (named.length) {
lines.push(`import { ${named.join(', ')} } from '${mod}';`)
}
}
// type-only imports
if (typeNamed.length) {
lines.push(`import type { ${typeNamed.join(', ')} } from '${mod}';`)
}
return lines
})
.sort()
.join('\n')
// 6. Serialize Typescript objects to printable code
const serializedProductScenes = makeObjExpr(scenes)
const serializedProductRoutes = makeObjExpr(routes)
const serializedProductRedirects = makeObjExpr(redirects)
const serializedProductConfiguration = makeObjExpr(sceneConfigs)
const serializedProductUrls = makeObjExpr(urls)
const serializedFileSystemTypes = makeObjExpr(
fileSystemTypes.sort((a, b) => a.name.text.localeCompare(b.name.text))
)
const serializedSetupProbes = printer.printNode(
ts.EmitHint.Unspecified,
ts.factory.createArrayLiteralExpression(setupProbes),
ts.createSourceFile('', '', ts.ScriptTarget.ESNext)
)
const serializedTreeItemsNew = makeArrExpr(treeItemsNew)
const serializedTreeItemsProducts = makeArrExpr(treeItemsProducts)
// A literal union of product tree paths, so code that hardcodes a path (e.g. sidebar companion
// maps) fails to compile on a typo instead of silently matching nothing.
const serializedProductTreePaths = Object.keys(treeItemsProducts)
.sort((a, b) => a.localeCompare(b))
.map((p) => JSON.stringify(p))
.join(' | ')
const serializedTreeItemsGames = makeArrExpr(treeItemsGames)
const serializedTreeItemsMetadata = makeArrExpr(treeItemsMetadata)
// 6. Assemble `products.tsx` + `productScenes.tsx`, write, format, move to src/
// The scene lazy-import map lives in its own module: products.tsx is (transitively) imported
// by scenes/urls.ts and therefore by nearly every module in the app, and jest's
// --findRelatedTests counts dynamic import() as a dependency edge — keeping the scene map
// here would make any change to any product scene "related" to almost the entire test suite.
const autogenDisclaimer = '/** This const is auto-generated, as is the whole file */'
const productsTsx = `
/* eslint @typescript-eslint/explicit-module-boundary-types: 0 */
// Generated by build-products.mjs – DO NOT EDIT BY HAND.
${importLines}
${autogenDisclaimer}
export const productRoutes: Record<string, [string, string]> = ${serializedProductRoutes}
${autogenDisclaimer}
export const productRedirects: Record<string, string | ((params: Params, searchParams: Params, hashParams: Params) => string)> = ${serializedProductRedirects}
${autogenDisclaimer}
export const productConfiguration: Record<string, any> = ${serializedProductConfiguration}
${autogenDisclaimer}
export const productUrls = ${serializedProductUrls}
${autogenDisclaimer}
export const fileSystemTypes = ${serializedFileSystemTypes}
${autogenDisclaimer}
export const productSetupProbes: ProductSetupProbe[] = ${serializedSetupProbes}
${autogenDisclaimer}
export const getTreeItemsNew = (): FileSystemImport[] => ${serializedTreeItemsNew}
${autogenDisclaimer}
export type ProductTreePath = ${serializedProductTreePaths}
${autogenDisclaimer}
export const getTreeItemsProducts = (): FileSystemImport[] => ${serializedTreeItemsProducts}
${autogenDisclaimer}
export const getTreeItemsGames = (): FileSystemImport[] => ${serializedTreeItemsGames}
${autogenDisclaimer}
export const getTreeItemsMetadata = (): FileSystemImport[] => ${serializedTreeItemsMetadata}
`
const scenesTsx = `
/* eslint @typescript-eslint/explicit-module-boundary-types: 0 */
// Generated by build-products.mjs – DO NOT EDIT BY HAND.
//
// Kept separate from products.tsx on purpose: this map's dynamic imports reach every
// product scene, and products.tsx is imported app-wide via scenes/urls.ts. In one module,
// jest's --findRelatedTests would mark ~the whole test suite as related to any scene change.
// Import this only from scene-mounting code (scenes/appScenes.ts).
${autogenDisclaimer}
export const productScenes: Record<string, () => Promise<any>> = ${serializedProductScenes}
`
const tsxTmpDir = path.join(__dirname, 'tmp')
fse.mkdirSync(tsxTmpDir, { recursive: true })
const tsxTmpFile = path.join(tsxTmpDir, 'products.tsx')
fse.writeFileSync(tsxTmpFile, productsTsx)
ps.execFileSync('oxfmt', [tsxTmpFile])
fse.renameSync(tsxTmpFile, outputTsx)
const scenesTmpFile = path.join(tsxTmpDir, 'productScenes.tsx')
fse.writeFileSync(scenesTmpFile, scenesTsx)
ps.execFileSync('oxfmt', [scenesTmpFile])
fse.renameSync(scenesTmpFile, outputScenesTsx)
for (const { field, file, imports, declaration } of LAZY_REGISTRIES) {
const entries = lazyRegistryEntries[field]
const serialized = makeObjExpr([...entries.keys()].sort().map((key) => entries.get(key)))
const registryTmpFile = path.join(tsxTmpDir, file)
fse.writeFileSync(
registryTmpFile,
`
/* eslint @typescript-eslint/explicit-module-boundary-types: 0 */
// Generated by build-products.mjs – DO NOT EDIT BY HAND.
${imports}
${autogenDisclaimer}
export const ${declaration} = ${serialized}
`
)
ps.execFileSync('oxfmt', [registryTmpFile])
fse.renameSync(registryTmpFile, path.join(__dirname, 'src', file))
}
// 8. Assemble `products.json`, write, format, move to src/
// A much simplified version of `products.tsx` to simplify consumption from Python
// without any of the AST/TSX code generation logic
//
// NOTE: The structure of products.json must match the TypeScript types defined in
// frontend/src/queries/schema/schema-general.ts (ProductItem and ProductsData).
// These types are used to generate Pydantic models in posthog/schema.py.
// If you change the keys here (keysToKeep), make sure to update the TypeScript types.
const keysToKeep = ['path', 'category', 'iconType', 'type']
const keysToKeepArray = ['intents']
const productKeyEnumMap = buildEnumMapping(program, 'ProductKey')
const productItemCategoryEnumMap = buildEnumMapping(program, 'ProductItemCategory')
const enumMap = new Map([...productKeyEnumMap, ...productItemCategoryEnumMap]) // Join both enum maps, will fail if they have same keys, not the case right now
const productsJson = {
products: extractKeys(treeItemsProducts, { keysToKeep, keysToKeepArray, enumMap }),
games: extractKeys(treeItemsGames, { keysToKeep, keysToKeepArray, enumMap }),
metadata: extractKeys(treeItemsMetadata, { keysToKeep, keysToKeepArray, enumMap }),
}
const jsonTmpDir = path.join(__dirname, 'tmp')
fse.mkdirSync(jsonTmpDir, { recursive: true })
const jsonTmpFile = path.join(jsonTmpDir, 'products.json')
fse.writeFileSync(jsonTmpFile, JSON.stringify(productsJson))
ps.execFileSync('oxfmt', [jsonTmpFile])
fse.renameSync(jsonTmpFile, outputJson)
fse.mkdirSync(cacheDir, { recursive: true })
fse.writeFileSync(cacheFile, digest)
}
function keepOnlyImport(prop, manifestPath) {
if (!ts.isPropertyAssignment(prop) || !ts.isObjectLiteralExpression(prop.initializer)) {
return null
}
const imp = prop.initializer.properties.find((p) => p.name?.text === 'import')
if (!imp) {
return null
}
const fn = cloneNode(imp.initializer)
if (
ts.isFunctionLike(fn) &&
ts.isCallExpression(fn.body) &&
fn.body.arguments.length === 1 &&
ts.isStringLiteralLike(fn.body.arguments[0])
) {
const importText = fn.body.arguments[0].text
if (importText.startsWith('./')) {
const newPath = path.relative('./src/', path.join(path.dirname(manifestPath), importText))
fn.body.arguments[0] = ts.factory.createStringLiteral(newPath)
}
return ts.factory.createPropertyAssignment(prop.name, fn)
}
return null
}
// A manifest writes \`import('./frontend/...')\` relative to itself; the generated file lives in frontend/src.
function withImportPathsFromSrc(node, manifestPath) {
const srcDir = path.resolve(__dirname, 'src')
const rewrite = (context) => (root) => {
const visit = (child) => {
if (
ts.isCallExpression(child) &&
child.expression.kind === ts.SyntaxKind.ImportKeyword &&
child.arguments.length === 1 &&
ts.isStringLiteralLike(child.arguments[0]) &&
child.arguments[0].text.startsWith('./')
) {
const target = path.relative(srcDir, path.resolve(path.dirname(manifestPath), child.arguments[0].text))
return context.factory.updateCallExpression(child, child.expression, child.typeArguments, [
context.factory.createStringLiteral(target),
])
}
return ts.visitEachChild(child, visit, context)
}
return ts.visitNode(root, visit)
}
return ts.transform(node, [rewrite]).transformed[0]
}
function withoutImport(prop) {
if (!ts.isPropertyAssignment(prop) || !ts.isObjectLiteralExpression(prop.initializer)) {
return null
}
const clone = cloneNode(prop)
clone.initializer.properties = clone.initializer.properties.filter((p) => p.name?.text !== 'import')
return clone
}
// Build a mapping of enum values by parsing the enum definition file
function buildEnumMapping(program, enumName) {
const enumMap = new Map()
for (const sourceFile of program.getSourceFiles()) {
if (!sourceFile.fileName.includes('schema-general.ts')) {
continue
}
ts.forEachChild(sourceFile, function walk(node) {
if (ts.isEnumDeclaration(node) && node.name.text === enumName) {
node.members.forEach((member) => {
const memberName = member.name?.text || member.name?.escapedText
if (member.initializer) {
if (ts.isStringLiteral(member.initializer)) {
enumMap.set(memberName, member.initializer.text)
} else if (ts.isNumericLiteral(member.initializer)) {
enumMap.set(memberName, Number(member.initializer.text))
}
}
})
} else {
ts.forEachChild(node, walk)
}
})
}
return enumMap
}
// Helper to resolve enum values from property access expressions (e.g., ProductKey.ENDPOINTS -> 'endpoints')
function resolveEnumValue(node, enumMap) {
if (!ts.isPropertyAccessExpression(node)) {
return null
}
// Get the enum name (e.g., "ENDPOINTS" from ProductKey.ENDPOINTS)
const memberName = node.name?.text || node.name?.escapedText
if (memberName && enumMap.has(memberName)) {
return enumMap.get(memberName)
}
return null
}
// Helper to extract only specific keys from AST nodes
function extractKeys(
dict /* : Record<any, ASTNode> */,
{ keysToKeep /* : string[] */, keysToKeepArray /* : string[] */, enumMap /* : Map<string, string | number> */ } = {
keysToKeep: [],
keysToKeepArray: [],
enumMap: null,
}
) {
return Object.values(dict).map((node /* : ASTNode */) => {
if (!ts.isObjectLiteralExpression(node)) {
return {}
}
const result /* : Record<string, string | boolean> */ = {}
const processKey = (key, { isArray }) => {
const prop /* : PropertyAssignment */ = node.properties.find((p) => p.name?.text === key)
result[key] = isArray ? [] : null // default to empty array/null
if (prop && prop.initializer) {
if (ts.isStringLiteral(prop.initializer)) {
result[key] = prop.initializer.text
} else if (ts.isNoSubstitutionTemplateLiteral(prop.initializer)) {
result[key] = prop.initializer.text
} else if (ts.isLiteralExpression(prop.initializer)) {
result[key] = prop.initializer.text
} else if (ts.isNumericLiteral(prop.initializer)) {
result[key] = Number(prop.initializer.text)
} else if (ts.isPropertyAccessExpression(prop.initializer)) {
// Set the escaped text by default
result[key] = prop.initializer.name.escapedText
// Attempt to set the resolved enum value if found
if (enumMap) {
const enumValue = resolveEnumValue(prop.initializer, enumMap)
if (enumValue !== null) {
result[key] = enumValue
}
}
} else if (ts.isArrayLiteralExpression(prop.initializer)) {
result[key] = prop.initializer.elements.map((e) => {
// Resolve enum values first
if (enumMap && ts.isPropertyAccessExpression(e)) {
const enumValue = resolveEnumValue(e, enumMap)
if (enumValue !== null) {
return enumValue
}
}
// Else just return the escaped text
return e.name.escapedText
})
} else if (prop.initializer.kind === ts.SyntaxKind.TrueKeyword) {
result[key] = true
} else if (prop.initializer.kind === ts.SyntaxKind.FalseKeyword) {
result[key] = false
}
}
}
keysToKeep.forEach((key) => processKey(key, { isArray: false }))
keysToKeepArray.forEach((key) => processKey(key, { isArray: true }))
return result
})
}