builder/lib/processors/minifier.js

  1. import {fileURLToPath} from "node:url";
  2. import posixPath from "node:path/posix";
  3. import {promisify} from "node:util";
  4. import os from "node:os";
  5. import workerpool from "workerpool";
  6. import Resource from "@ui5/fs/Resource";
  7. import {getLogger} from "@ui5/logger";
  8. const log = getLogger("builder:processors:minifier");
  9. import {setTimeout as setTimeoutPromise} from "node:timers/promises";
  11. const debugFileRegex = /((?:\.view|\.fragment|\.controller|\.designtime|\.support)?\.js)$/;
  13. const MIN_WORKERS = 2;
  14. const MAX_WORKERS = 4;
  15. const osCpus = os.cpus().length || 1;
  16. const maxWorkers = Math.max(Math.min(osCpus - 1, MAX_WORKERS), MIN_WORKERS);
  18. const sourceMappingUrlPattern = /\/\/# sourceMappingURL=(\S+)\s*$/;
  19. const httpPattern = /^https?:\/\//i;
  21. // Shared workerpool across all executions until the taskUtil cleanup is triggered
  22. let pool;
  24. function getPool(taskUtil) {
  25. 	if (!pool) {
  26. 		log.verbose(`Creating workerpool with up to ${maxWorkers} workers (available CPU cores: ${osCpus})`);
  27. 		const workerPath = fileURLToPath(new URL("./minifierWorker.js", import.meta.url));
  28. 		pool = workerpool.pool(workerPath, {
  29. 			workerType: "auto",
  30. 			maxWorkers
  31. 		});
  32. 		taskUtil.registerCleanupTask((force) => {
  33. 			const attemptPoolTermination = async () => {
  34. 				log.verbose(`Attempt to terminate the workerpool...`);
  36. 				if (!pool) {
  37. 					return;
  38. 				}
  40. 				// There are many stats that could be used, but these ones seem the most
  41. 				// convenient. When all the (available) workers are idle, then it's safe to terminate.
  42. 				let {idleWorkers, totalWorkers} = pool.stats();
  43. 				while (idleWorkers !== totalWorkers && !force) {
  44. 					await setTimeoutPromise(100); // Wait a bit workers to finish and try again
  45. 					({idleWorkers, totalWorkers} = pool.stats());
  46. 				}
  48. 				const poolToBeTerminated = pool;
  49. 				pool = null;
  50. 				return poolToBeTerminated.terminate(force);
  51. 			};
  53. 			return attemptPoolTermination();
  54. 		});
  55. 	}
  56. 	return pool;
  57. }
  59. async function minifyInWorker(options, taskUtil) {
  60. 	return getPool(taskUtil).exec("execMinification", [options]);
  61. }
  63. async function extractAndRemoveSourceMappingUrl(resource) {
  64. 	const resourceContent = await resource.getString();
  65. 	const resourcePath = resource.getPath();
  66. 	const sourceMappingUrlMatch = resourceContent.match(sourceMappingUrlPattern);
  67. 	if (sourceMappingUrlMatch) {
  68. 		const sourceMappingUrl = sourceMappingUrlMatch[1];
  69. 		if (log.isLevelEnabled("silly")) {
  70. 			log.silly(`Found source map reference in content of resource ${resourcePath}: ${sourceMappingUrl}`);
  71. 		}
  73. 		// Strip sourceMappingURL from the resource to be minified
  74. 		// It is not required anymore and will be replaced for in the minified resource
  75. 		// and its debug variant anyways
  76. 		resource.setString(resourceContent.replace(sourceMappingUrlPattern, ""));
  77. 		return sourceMappingUrl;
  78. 	}
  79. 	return null;
  80. }
  82. async function getSourceMapFromUrl({sourceMappingUrl, resourcePath, readFile}) {
  83. 	// =======================================================================
  84. 	// This code is almost identical to code located in lbt/bundle/Builder.js
  85. 	// Please try to update both places when making changes
  86. 	// =======================================================================
  87. 	if (sourceMappingUrl.startsWith("data:")) {
  88. 		// Data-URI indicates an inline source map
  89. 		const expectedTypeAndEncoding = "data:application/json;charset=utf-8;base64,";
  90. 		if (sourceMappingUrl.startsWith(expectedTypeAndEncoding)) {
  91. 			const base64Content = sourceMappingUrl.slice(expectedTypeAndEncoding.length);
  92. 			// Create a resource with a path suggesting it's the source map for the resource
  93. 			// (which it is but inlined)
  94. 			return Buffer.from(base64Content, "base64").toString();
  95. 		} else {
  96. 			log.warn(
  97. 				`Source map reference in resource ${resourcePath} is a data URI but has an unexpected` +
  98. 				`encoding: ${sourceMappingUrl}. Expected it to start with ` +
  99. 				`"data:application/json;charset=utf-8;base64,"`);
  100. 		}
  101. 	} else if (httpPattern.test(sourceMappingUrl)) {
  102. 		log.warn(`Source map reference in resource ${resourcePath} is an absolute URL. ` +
  103. 			`Currently, only relative URLs are supported.`);
  104. 	} else if (posixPath.isAbsolute(sourceMappingUrl)) {
  105. 		log.warn(`Source map reference in resource ${resourcePath} is an absolute path. ` +
  106. 			`Currently, only relative paths are supported.`);
  107. 	} else {
  108. 		const sourceMapPath = posixPath.join(posixPath.dirname(resourcePath), sourceMappingUrl);
  110. 		try {
  111. 			const sourceMapContent = await readFile(sourceMapPath);
  112. 			return sourceMapContent.toString();
  113. 		} catch (e) {
  114. 			// No input source map
  115. 			log.warn(`Unable to read source map for resource ${resourcePath}: ${e.message}`);
  116. 		}
  117. 	}
  118. }
  120. /**
  121.  * @public
  122.  * @module @ui5/builder/processors/minifier
  123.  */
  125. /**
  126.  * Result set
  127.  *
  128.  * @public
  129.  * @typedef {object} MinifierResult
  130.  * @property {@ui5/fs/Resource} resource Minified resource
  131.  * @property {@ui5/fs/Resource} dbgResource Debug (non-minified) variant
  132.  * @property {@ui5/fs/Resource} sourceMap Source Map
  133.  */
  135. /**
  136.  * Minifies the supplied resources.
  137.  *
  138.  * @public
  139.  * @function default
  140.  * @static
  141.  *
  142.  * @param {object} parameters Parameters
  143.  * @param {@ui5/fs/Resource[]} parameters.resources List of resources to be processed
  144.  * @param {fs|module:@ui5/fs/fsInterface} parameters.fs Node fs or custom
  145.  *    [fs interface]{@link module:@ui5/fs/fsInterface}. Required when setting "readSourceMappingUrl" to true
  146.  * @param {@ui5/builder/tasks/TaskUtil|object} [parameters.taskUtil] TaskUtil instance.
  147.  *    Required when using the <code>useWorkers</code> option
  148.  * @param {object} [parameters.options] Options
  149.  * @param {boolean} [parameters.options.readSourceMappingUrl=false]
  150.  *   Whether to make use of any existing source maps referenced in the resources to be minified. Use this option to
  151.  *   preserve references to the original source files, such as TypeScript files, in the generated source map.<br>
  152.  *   If a resource has been modified by a previous task, any existing source map will be ignored regardless of this
  153.  *    setting. This is to ensure that no inconsistent source maps are used. Check the verbose log for details.
  154.  * @param {boolean} [parameters.options.addSourceMappingUrl=true]
  155.  *   Whether to add a sourceMappingURL reference to the end of the minified resource
  156.  * @param {boolean} [parameters.options.useWorkers=false]
  157.  *  Whether to offload the minification task onto separate CPU threads. This often speeds up the build process
  158.  * @returns {Promise<module:@ui5/builder/processors/minifier~MinifierResult[]>}
  159.  *   Promise resolving with object of resource, dbgResource and sourceMap
  160.  */
  161. export default async function({
  162. 	resources, fs, taskUtil, options: {readSourceMappingUrl = false, addSourceMappingUrl = true, useWorkers = false
  163. 	} = {}}) {
  164. 	let minify;
  165. 	if (readSourceMappingUrl && !fs) {
  166. 		throw new Error(`Option 'readSourceMappingUrl' requires parameter 'fs' to be provided`);
  167. 	}
  169. 	if (useWorkers) {
  170. 		if (!taskUtil) {
  171. 			// TaskUtil is required for worker support
  172. 			throw new Error(`Minifier: Option 'useWorkers' requires a taskUtil instance to be provided`);
  173. 		}
  174. 		minify = minifyInWorker;
  175. 	} else {
  176. 		// Do not use workerpool
  177. 		minify = (await import("./minifierWorker.js")).default;
  178. 	}
  180. 	return Promise.all(resources.map(async (resource) => {
  181. 		const resourcePath = resource.getPath();
  182. 		const dbgPath = resourcePath.replace(debugFileRegex, "-dbg$1");
  183. 		const dbgFilename = posixPath.basename(dbgPath);
  185. 		const filename = posixPath.basename(resource.getPath());
  187. 		const sourceMapOptions = {
  188. 			filename
  189. 		};
  190. 		if (addSourceMappingUrl) {
  191. 			sourceMapOptions.url = filename + ".map";
  192. 		}
  194. 		// Remember contentModified flag before making changes to the resource via setString
  195. 		const resourceContentModified = resource.getSourceMetadata()?.contentModified;
  197. 		// In any case: Extract *and remove* source map reference from resource before cloning it
  198. 		const sourceMappingUrl = await extractAndRemoveSourceMappingUrl(resource);
  200. 		const code = await resource.getString();
  201. 		// Create debug variant based off the original resource before minification
  202. 		const dbgResource = await resource.clone();
  203. 		dbgResource.setPath(dbgPath);
  205. 		let dbgSourceMapResource;
  206. 		if (sourceMappingUrl) {
  207. 			if (resourceContentModified) {
  208. 				log.verbose(
  209. 					`Source map found in resource will be ignored because the resource has been ` +
  210. 					`modified in a previous task: ${resourcePath}`);
  211. 			} else if (readSourceMappingUrl) {
  212. 				// Try to find a source map reference in the to-be-minified resource
  213. 				// If we find one, provide it to terser as an input source map and keep using it for the
  214. 				// debug variant of the resource
  215. 				const sourceMapContent = await getSourceMapFromUrl({
  216. 					sourceMappingUrl,
  217. 					resourcePath,
  218. 					readFile: promisify(fs.readFile)
  219. 				});
  221. 				if (sourceMapContent) {
  222. 					const sourceMapJson = JSON.parse(sourceMapContent);
  224. 					if (sourceMapJson.sections) {
  225. 						// TODO 4.0
  226. 						// Module "@jridgewell/trace-mapping" (used by Terser) can't handle index map sections lacking
  227. 						// a "names" array. Since this is a common occurrence for UI5 Tooling bundles, we search for
  228. 						// such cases here and fix them until https://github.com/jridgewell/trace-mapping/pull/29 is
  229. 						// resolved and Terser upgraded the dependency
  231. 						// Create a dedicated clone before modifying the source map as to not alter the debug source map
  232. 						const clonedSourceMapJson = JSON.parse(sourceMapContent);
  233. 						clonedSourceMapJson.sections.forEach(({map}) => {
  234. 							if (!map.names) {
  235. 								// Add missing names array
  236. 								map.names = [];
  237. 							}
  238. 						});
  239. 						// Use modified source map as input source map
  240. 						sourceMapOptions.content = JSON.stringify(clonedSourceMapJson);
  241. 					} else {
  242. 						// Provide source map to terser as "input source map"
  243. 						sourceMapOptions.content = sourceMapContent;
  244. 					}
  246. 					// Use the original source map for the debug variant of the resource
  247. 					// First update the file reference within the source map
  248. 					sourceMapJson.file = dbgFilename;
  250. 					// Then create a new resource
  251. 					dbgSourceMapResource = new Resource({
  252. 						string: JSON.stringify(sourceMapJson),
  253. 						path: dbgPath + ".map"
  254. 					});
  255. 					// And reference the resource in the debug resource
  256. 					dbgResource.setString(code + `//# sourceMappingURL=${dbgFilename}.map\n`);
  257. 				}
  258. 			} else {
  259. 				// If the original resource content was unmodified and the input source map was not parsed,
  260. 				// re-add the original source map reference to the debug variant
  261. 				if (!sourceMappingUrl.startsWith("data:") && !sourceMappingUrl.endsWith(filename + ".map")) {
  262. 					// Do not re-add inline source maps as well as references to the source map of
  263. 					// the minified resource
  264. 					dbgResource.setString(code + `//# sourceMappingURL=${sourceMappingUrl}\n`);
  265. 				}
  266. 			}
  267. 		}
  269. 		const result = await minify({
  270. 			filename,
  271. 			dbgFilename,
  272. 			code,
  273. 			sourceMapOptions
  274. 		}, taskUtil);
  275. 		resource.setString(result.code);
  276. 		const sourceMapResource = new Resource({
  277. 			path: resource.getPath() + ".map",
  278. 			string: result.map
  279. 		});
  280. 		return {resource, dbgResource, sourceMapResource, dbgSourceMapResource};
  281. 	}));
  282. }
  284. export const __localFunctions__ = (process.env.NODE_ENV === "test") ?
  285. 	{getSourceMapFromUrl} : undefined;