project/lib/projectPreprocessor.js

  1. const log = require("@ui5/logger").getLogger("normalizer:projectPreprocessor");
  2. const fs = require("graceful-fs");
  3. const path = require("path");
  4. const {promisify} = require("util");
  5. const readFile = promisify(fs.readFile);
  6. const jsyaml = require("js-yaml");
  7. const typeRepository = require("@ui5/builder").types.typeRepository;
  8. const {validate} = require("./validation/validator");
  9. class ProjectPreprocessor {
  10. constructor({tree}) {
  11. this.tree = tree;
  12. this.processedProjects = {};
  13. this.configShims = {};
  14. this.collections = {};
  15. this.appliedExtensions = {};
  16. }
  17. /*
  18. Adapt and enhance the project tree:
  19. - Replace duplicate projects further away from the root with those closer to the root
  20. - Add configuration to projects
  21. */
  22. async processTree() {
  23. const queue = [{
  24. projects: [this.tree],
  25. parent: null,
  26. level: 0
  27. }];
  28. const configPromises = [];
  29. let startTime;
  30. if (log.isLevelEnabled("verbose")) {
  31. startTime = process.hrtime();
  32. }
  33. // Breadth-first search to prefer projects closer to root
  34. while (queue.length) {
  35. const {projects, parent, level} = queue.shift(); // Get and remove first entry from queue
  36. // Before processing all projects on a level concurrently, we need to set all of them as being processed.
  37. // This prevents transitive dependencies pointing to the same projects from being processed first
  38. // by the dependency lookahead
  39. const projectsToProcess = projects.filter((project) => {
  40. if (!project.id) {
  41. const parentRefText = parent ? `(child of ${parent.id})` : `(root project)`;
  42. throw new Error(`Encountered project with missing id ${parentRefText}`);
  43. }
  44. if (this.isBeingProcessed(parent, project)) {
  45. return false;
  46. }
  47. // Flag this project as being processed
  48. this.processedProjects[project.id] = {
  49. project,
  50. // If a project is referenced multiple times in the dependency tree it is replaced
  51. // with the instance that is closest to the root.
  52. // Here we track the parents referencing that project
  53. parents: [parent]
  54. };
  55. return true;
  56. });
  57. await Promise.all(projectsToProcess.map(async (project) => {
  58. project._level = level;
  59. if (level === 0) {
  60. project._isRoot = true;
  61. }
  62. log.verbose(`Processing project ${project.id} on level ${project._level}...`);
  63. if (project.dependencies && project.dependencies.length) {
  64. // Do a dependency lookahead to apply any extensions that might affect this project
  65. await this.dependencyLookahead(project, project.dependencies);
  66. } else {
  67. // When using the static translator for instance, dependencies is not defined and will
  68. // fail later access calls to it
  69. project.dependencies = [];
  70. }
  71. const {extensions} = await this.loadProjectConfiguration(project);
  72. if (extensions && extensions.length) {
  73. // Project contains additional extensions
  74. // => apply them
  75. // TODO: Check whether extensions get applied twice in case depLookahead already processed them
  76. await Promise.all(extensions.map((extProject) => {
  77. return this.applyExtension(extProject);
  78. }));
  79. }
  80. await this.applyShims(project);
  81. if (this.isConfigValid(project)) {
  82. // Do not apply transparent projects.
  83. // Their only purpose might be to have their dependencies processed
  84. if (!project._transparentProject) {
  85. await this.applyType(project);
  86. this.checkProjectMetadata(parent, project);
  87. }
  88. queue.push({
  89. // copy array, so that the queue is stable while ignored project dependencies are removed
  90. projects: [...project.dependencies],
  91. parent: project,
  92. level: level + 1
  93. });
  94. } else {
  95. if (project === this.tree) {
  96. throw new Error(
  97. `Failed to configure root project "${project.id}". Please check verbose log for details.`);
  98. }
  99. // No config available
  100. // => reject this project by removing it from its parents list of dependencies
  101. log.verbose(`Ignoring project ${project.id} with missing configuration ` +
  102. "(might be a non-UI5 dependency)");
  103. const parents = this.processedProjects[project.id].parents;
  104. for (let i = parents.length - 1; i >= 0; i--) {
  105. parents[i].dependencies.splice(parents[i].dependencies.indexOf(project), 1);
  106. }
  107. this.processedProjects[project.id] = {ignored: true};
  108. }
  109. }));
  110. }
  111. return Promise.all(configPromises).then(() => {
  112. if (log.isLevelEnabled("verbose")) {
  113. const prettyHrtime = require("pretty-hrtime");
  114. const timeDiff = process.hrtime(startTime);
  115. log.verbose(
  116. `Processed ${Object.keys(this.processedProjects).length} projects in ${prettyHrtime(timeDiff)}`);
  117. }
  118. return this.tree;
  119. });
  120. }
  121. async dependencyLookahead(parent, dependencies) {
  122. return Promise.all(dependencies.map(async (project) => {
  123. if (this.isBeingProcessed(parent, project)) {
  124. return;
  125. }
  126. log.verbose(`Processing dependency lookahead for ${parent.id}: ${project.id}`);
  127. // Temporarily flag project as being processed
  128. this.processedProjects[project.id] = {
  129. project,
  130. parents: [parent]
  131. };
  132. const {extensions} = await this.loadProjectConfiguration(project);
  133. if (extensions && extensions.length) {
  134. // Project contains additional extensions
  135. // => apply them
  136. await Promise.all(extensions.map((extProject) => {
  137. return this.applyExtension(extProject);
  138. }));
  139. }
  140. if (project.kind === "extension") {
  141. // Not a project but an extension
  142. // => remove it as from any known projects that depend on it
  143. const parents = this.processedProjects[project.id].parents;
  144. for (let i = parents.length - 1; i >= 0; i--) {
  145. parents[i].dependencies.splice(parents[i].dependencies.indexOf(project), 1);
  146. }
  147. // Also ignore it from further processing by other projects depending on it
  148. this.processedProjects[project.id] = {ignored: true};
  149. if (this.isConfigValid(project)) {
  150. // Finally apply the extension
  151. await this.applyExtension(project);
  152. } else {
  153. log.verbose(`Ignoring extension ${project.id} with missing configuration`);
  154. }
  155. } else {
  156. // Project is not an extension: Reset processing status of lookahead to allow the real processing
  157. this.processedProjects[project.id] = null;
  158. }
  159. }));
  160. }
  161. isBeingProcessed(parent, project) { // Check whether a project is currently being or has already been processed
  162. const processedProject = this.processedProjects[project.id];
  163. if (project.deduped) {
  164. // Ignore deduped modules
  165. return true;
  166. }
  167. if (processedProject) {
  168. if (processedProject.ignored) {
  169. log.verbose(`Dependency of project ${parent.id}, "${project.id}" is flagged as ignored.`);
  170. if (parent.dependencies.includes(project)) {
  171. parent.dependencies.splice(parent.dependencies.indexOf(project), 1);
  172. }
  173. return true;
  174. }
  175. log.verbose(
  176. `Dependency of project ${parent.id}, "${project.id}": ` +
  177. `Distance to root of ${parent._level + 1}. Will be replaced `+
  178. `by project with same ID and distance to root of ${processedProject.project._level}.`);
  179. // Replace with the already processed project (closer to root -> preferred)
  180. parent.dependencies[parent.dependencies.indexOf(project)] = processedProject.project;
  181. processedProject.parents.push(parent);
  182. // No further processing needed
  183. return true;
  184. }
  185. return false;
  186. }
  187. async loadProjectConfiguration(project) {
  188. if (project.specVersion) { // Project might already be configured
  189. // Currently, specVersion is the indicator for configured projects
  190. if (project._transparentProject) {
  191. // Assume that project is already processed
  192. return {};
  193. }
  194. await this.validateAndNormalizeExistingProject(project);
  195. return {};
  196. }
  197. const configs = await this.readConfigFile(project);
  198. if (!configs || !configs.length) {
  199. return {};
  200. }
  201. for (let i = configs.length - 1; i >= 0; i--) {
  202. this.normalizeConfig(configs[i]);
  203. }
  204. const projectConfigs = configs.filter((config) => {
  205. return config.kind === "project";
  206. });
  207. const extensionConfigs = configs.filter((config) => {
  208. return config.kind === "extension";
  209. });
  210. const projectClone = JSON.parse(JSON.stringify(project));
  211. // While a project can contain multiple configurations,
  212. // from a dependency tree perspective it is always a single project
  213. // This means it can represent one "project", plus multiple extensions or
  214. // one extension, plus multiple extensions
  215. if (projectConfigs.length === 1) {
  216. // All well, this is the one. Merge config into project
  217. Object.assign(project, projectConfigs[0]);
  218. } else if (projectConfigs.length > 1) {
  219. throw new Error(`Found ${projectConfigs.length} configurations of kind 'project' for ` +
  220. `project ${project.id}. There is only one project per configuration allowed.`);
  221. } else if (projectConfigs.length === 0 && extensionConfigs.length) {
  222. // No project, but extensions
  223. // => choose one to represent the project -> the first one
  224. Object.assign(project, extensionConfigs.shift());
  225. } else {
  226. throw new Error(`Found ${configs.length} configurations for ` +
  227. `project ${project.id}. None are of valid kind.`);
  228. }
  229. const extensionProjects = extensionConfigs.map((config) => {
  230. // Clone original project
  231. const configuredProject = JSON.parse(JSON.stringify(projectClone));
  232. // Enhance project with its configuration
  233. Object.assign(configuredProject, config);
  234. return configuredProject;
  235. });
  236. return {extensions: extensionProjects};
  237. }
  238. normalizeConfig(config) {
  239. if (!config.kind) {
  240. config.kind = "project"; // default
  241. }
  242. }
  243. isConfigValid(project) {
  244. if (!project.specVersion) {
  245. if (project._isRoot) {
  246. throw new Error(`No specification version defined for root project ${project.id}`);
  247. }
  248. log.verbose(`No specification version defined for project ${project.id}`);
  249. return false; // ignore this project
  250. }
  251. if (project.specVersion !== "0.1" && project.specVersion !== "1.0" &&
  252. project.specVersion !== "1.1" && project.specVersion !== "2.0" &&
  253. project.specVersion !== "2.1" && project.specVersion !== "2.2" &&
  254. project.specVersion !== "2.3" && project.specVersion !== "2.4" &&
  255. project.specVersion !== "2.5" && project.specVersion !== "2.6") {
  256. throw new Error(
  257. `Unsupported specification version ${project.specVersion} defined for project ` +
  258. `${project.id}. Your UI5 CLI installation might be outdated. ` +
  259. `For details see https://sap.github.io/ui5-tooling/pages/Configuration/#specification-versions`);
  260. }
  261. if (!project.type) {
  262. if (project._isRoot) {
  263. throw new Error(`No type configured for root project ${project.id}`);
  264. }
  265. log.verbose(`No type configured for project ${project.id}`);
  266. return false; // ignore this project
  267. }
  268. if (project.kind !== "project" && project._isRoot) {
  269. // This is arguable. It is not the concern of ui5-project to define the entry point of a project tree
  270. // On the other hand, there is no known use case for anything else right now and failing early here
  271. // makes sense in that regard
  272. throw new Error(`Root project needs to be of kind "project". ${project.id} is of kind ${project.kind}`);
  273. }
  274. if (project.kind === "project" && project.type === "application") {
  275. // There must be exactly one application project per dependency tree
  276. // If multiple are found, all but the one closest to the root are rejected (ignored)
  277. // If there are two projects equally close to the root, an error is being thrown
  278. if (!this.qualifiedApplicationProject) {
  279. this.qualifiedApplicationProject = project;
  280. } else if (this.qualifiedApplicationProject._level === project._level) {
  281. throw new Error(`Found at least two projects ${this.qualifiedApplicationProject.id} and ` +
  282. `${project.id} of type application with the same distance to the root project. ` +
  283. "Only one project of type application can be used. Failed to decide which one to ignore.");
  284. } else {
  285. return false; // ignore this project
  286. }
  287. }
  288. return true;
  289. }
  290. async applyType(project) {
  291. let type;
  292. try {
  293. type = typeRepository.getType(project.type);
  294. } catch (err) {
  295. throw new Error(`Failed to retrieve type for project ${project.id}: ${err.message}`);
  296. }
  297. await type.format(project);
  298. }
  299. checkProjectMetadata(parent, project) {
  300. if (project.metadata.deprecated && parent && parent._isRoot) {
  301. // Only warn for direct dependencies of the root project
  302. log.warn(`Dependency ${project.metadata.name} is deprecated and should not be used for new projects!`);
  303. }
  304. if (project.metadata.sapInternal && parent && parent._isRoot && !parent.metadata.allowSapInternal) {
  305. // Only warn for direct dependencies of the root project, except it defines "allowSapInternal"
  306. log.warn(`Dependency ${project.metadata.name} is restricted for use by SAP internal projects only! ` +
  307. `If the project ${parent.metadata.name} is an SAP internal project, add the attribute ` +
  308. `"allowSapInternal: true" to its metadata configuration`);
  309. }
  310. }
  311. async applyExtension(extension) {
  312. if (!extension.metadata || !extension.metadata.name) {
  313. throw new Error(`metadata.name configuration is missing for extension ${extension.id}`);
  314. }
  315. log.verbose(`Applying extension ${extension.metadata.name}...`);
  316. if (!extension.specVersion) {
  317. throw new Error(`No specification version defined for extension ${extension.metadata.name}`);
  318. } else if (extension.specVersion !== "0.1" &&
  319. extension.specVersion !== "1.0" &&
  320. extension.specVersion !== "1.1" &&
  321. extension.specVersion !== "2.0" &&
  322. extension.specVersion !== "2.1" &&
  323. extension.specVersion !== "2.2" &&
  324. extension.specVersion !== "2.3" &&
  325. extension.specVersion !== "2.4" &&
  326. extension.specVersion !== "2.5" &&
  327. extension.specVersion !== "2.6") {
  328. throw new Error(
  329. `Unsupported specification version ${extension.specVersion} defined for extension ` +
  330. `${extension.metadata.name}. Your UI5 CLI installation might be outdated. ` +
  331. `For details see https://sap.github.io/ui5-tooling/pages/Configuration/#specification-versions`);
  332. } else if (this.appliedExtensions[extension.metadata.name]) {
  333. log.verbose(`Extension with the name ${extension.metadata.name} has already been applied. ` +
  334. "This might have been done during dependency lookahead.");
  335. log.verbose(`Already applied extension ID: ${this.appliedExtensions[extension.metadata.name].id}. ` +
  336. `New extension ID: ${extension.id}`);
  337. return;
  338. }
  339. this.appliedExtensions[extension.metadata.name] = extension;
  340. switch (extension.type) {
  341. case "project-shim":
  342. this.handleShim(extension);
  343. break;
  344. case "task":
  345. this.handleTask(extension);
  346. break;
  347. case "server-middleware":
  348. this.handleServerMiddleware(extension);
  349. break;
  350. default:
  351. throw new Error(`Unknown extension type '${extension.type}' for ${extension.id}`);
  352. }
  353. }
  354. async readConfigFile(project) {
  355. // A projects configPath property takes precedence over the default "<projectPath>/ui5.yaml" path
  356. const configPath = project.configPath || path.join(project.path, "ui5.yaml");
  357. let configFile;
  358. try {
  359. configFile = await readFile(configPath, {encoding: "utf8"});
  360. } catch (err) {
  361. const errorText = "Failed to read configuration for project " +
  362. `${project.id} at "${configPath}". Error: ${err.message}`;
  363. // Something else than "File or directory does not exist" or root project
  364. if (err.code !== "ENOENT" || project._isRoot) {
  365. throw new Error(errorText);
  366. } else {
  367. log.verbose(errorText);
  368. return null;
  369. }
  370. }
  371. let configs;
  372. try {
  373. configs = jsyaml.loadAll(configFile, undefined, {
  374. filename: configPath
  375. });
  376. } catch (err) {
  377. if (err.name === "YAMLException") {
  378. throw new Error("Failed to parse configuration for project " +
  379. `${project.id} at "${configPath}"\nError: ${err.message}`);
  380. } else {
  381. throw err;
  382. }
  383. }
  384. if (!configs || !configs.length) {
  385. return configs;
  386. }
  387. const validationResults = await Promise.all(
  388. configs.map(async (config, documentIndex) => {
  389. // Catch validation errors to ensure proper order of rejections within Promise.all
  390. try {
  391. await validate({
  392. config,
  393. project: {
  394. id: project.id
  395. },
  396. yaml: {
  397. path: configPath,
  398. source: configFile,
  399. documentIndex
  400. }
  401. });
  402. } catch (error) {
  403. return error;
  404. }
  405. })
  406. );
  407. const validationErrors = validationResults.filter(($) => $);
  408. if (validationErrors.length > 0) {
  409. // For now just throw the error of the first invalid document
  410. throw validationErrors[0];
  411. }
  412. return configs;
  413. }
  414. handleShim(extension) {
  415. if (!extension.shims) {
  416. throw new Error(`Project shim extension ${extension.id} is missing 'shims' configuration`);
  417. }
  418. const {configurations, dependencies, collections} = extension.shims;
  419. if (configurations) {
  420. log.verbose(`Project shim ${extension.id} contains ` +
  421. `${Object.keys(configurations)} configuration(s)`);
  422. for (const projectId of Object.keys(configurations)) {
  423. this.normalizeConfig(configurations[projectId]); // TODO: Clone object beforehand?
  424. if (this.configShims[projectId]) {
  425. log.verbose(`Project shim ${extension.id}: A configuration shim for project ${projectId} `+
  426. "has already been applied. Skipping.");
  427. } else if (this.isConfigValid(configurations[projectId])) {
  428. log.verbose(`Project shim ${extension.id}: Adding project configuration for ${projectId}...`);
  429. this.configShims[projectId] = configurations[projectId];
  430. } else {
  431. log.verbose(`Project shim ${extension.id}: Ignoring invalid ` +
  432. `configuration shim for project ${projectId}`);
  433. }
  434. }
  435. }
  436. if (dependencies) {
  437. // For the time being, shimmed dependencies only apply to shimmed project configurations
  438. for (const projectId of Object.keys(dependencies)) {
  439. if (this.configShims[projectId]) {
  440. log.verbose(`Project shim ${extension.id}: Adding dependencies ` +
  441. `to project shim '${projectId}'...`);
  442. this.configShims[projectId].dependencies = dependencies[projectId];
  443. } else {
  444. log.verbose(`Project shim ${extension.id}: No configuration shim found for ` +
  445. `project ID '${projectId}'. Dependency shims currently only apply ` +
  446. "to projects with configuration shims.");
  447. }
  448. }
  449. }
  450. if (collections) {
  451. log.verbose(`Project shim ${extension.id} contains ` +
  452. `${Object.keys(collections).length} collection(s)`);
  453. for (const projectId of Object.keys(collections)) {
  454. if (this.collections[projectId]) {
  455. log.verbose(`Project shim ${extension.id}: A collection with id '${projectId}' `+
  456. "is already known. Skipping.");
  457. } else {
  458. log.verbose(`Project shim ${extension.id}: Adding collection with id '${projectId}'...`);
  459. this.collections[projectId] = collections[projectId];
  460. }
  461. }
  462. }
  463. }
  464. async applyShims(project) {
  465. const configShim = this.configShims[project.id];
  466. // Apply configuration shims
  467. if (configShim) {
  468. log.verbose(`Applying configuration shim for project ${project.id}...`);
  469. if (configShim.dependencies && configShim.dependencies.length) {
  470. if (!configShim.shimDependenciesResolved) {
  471. configShim.dependencies = configShim.dependencies.map((depId) => {
  472. const depProject = this.processedProjects[depId].project;
  473. if (!depProject) {
  474. throw new Error(
  475. `Failed to resolve shimmed dependency '${depId}' for project ${project.id}. ` +
  476. `Is a dependency with ID '${depId}' part of the dependency tree?`);
  477. }
  478. return depProject;
  479. });
  480. configShim.shimDependenciesResolved = true;
  481. }
  482. configShim.dependencies.forEach((depProject) => {
  483. const parents = this.processedProjects[depProject.id].parents;
  484. if (parents.indexOf(project) === -1) {
  485. parents.push(project);
  486. } else {
  487. log.verbose(`Project ${project.id} is already parent of shimmed dependency ${depProject.id}`);
  488. }
  489. });
  490. }
  491. Object.assign(project, configShim);
  492. delete project.shimDependenciesResolved; // Remove shim processing metadata from project
  493. await this.validateAndNormalizeExistingProject(project);
  494. }
  495. // Apply collections
  496. for (let i = project.dependencies.length - 1; i >= 0; i--) {
  497. const depId = project.dependencies[i].id;
  498. if (this.collections[depId]) {
  499. log.verbose(`Project ${project.id} depends on collection ${depId}. Resolving...`);
  500. // This project depends on a collection
  501. // => replace collection dependency with first collection project.
  502. const collectionDep = project.dependencies[i];
  503. const collectionModules = this.collections[depId].modules;
  504. const projects = [];
  505. for (const projectId of Object.keys(collectionModules)) {
  506. // Clone and modify collection "project"
  507. const project = JSON.parse(JSON.stringify(collectionDep));
  508. project.id = projectId;
  509. project.path = path.join(project.path, collectionModules[projectId]);
  510. projects.push(project);
  511. }
  512. // Use first collection project to replace the collection dependency
  513. project.dependencies[i] = projects.shift();
  514. // Add any additional collection projects to end of dependency array (already processed)
  515. project.dependencies.push(...projects);
  516. }
  517. }
  518. }
  519. handleTask(extension) {
  520. if (!extension.metadata && !extension.metadata.name) {
  521. throw new Error(`Task extension ${extension.id} is missing 'metadata.name' configuration`);
  522. }
  523. if (!extension.task) {
  524. throw new Error(`Task extension ${extension.id} is missing 'task' configuration`);
  525. }
  526. const taskRepository = require("@ui5/builder").tasks.taskRepository;
  527. const taskPath = path.join(extension.path, extension.task.path);
  528. taskRepository.addTask({
  529. name: extension.metadata.name,
  530. specVersion: extension.specVersion,
  531. taskPath,
  532. });
  533. }
  534. handleServerMiddleware(extension) {
  535. if (!extension.metadata && !extension.metadata.name) {
  536. throw new Error(`Middleware extension ${extension.id} is missing 'metadata.name' configuration`);
  537. }
  538. if (!extension.middleware) {
  539. throw new Error(`Middleware extension ${extension.id} is missing 'middleware' configuration`);
  540. }
  541. const {middlewareRepository} = require("@ui5/server");
  542. const middlewarePath = path.join(extension.path, extension.middleware.path);
  543. middlewareRepository.addMiddleware({
  544. name: extension.metadata.name,
  545. specVersion: extension.specVersion,
  546. middlewarePath
  547. });
  548. }
  549. async validateAndNormalizeExistingProject(project) {
  550. // Validate project config, but exclude additional properties
  551. const excludedProperties = [
  552. "id",
  553. "version",
  554. "path",
  555. "dependencies",
  556. "_level",
  557. "_isRoot"
  558. ];
  559. const config = {};
  560. for (const key of Object.keys(project)) {
  561. if (!excludedProperties.includes(key)) {
  562. config[key] = project[key];
  563. }
  564. }
  565. await validate({
  566. config,
  567. project: {
  568. id: project.id
  569. }
  570. });
  571. this.normalizeConfig(project);
  572. }
  573. }
  574. /**
  575. * The Project Preprocessor enriches the dependency information with project configuration
  576. *
  577. * @public
  578. * @namespace
  579. * @alias module:@ui5/project.projectPreprocessor
  580. */
  581. module.exports = {
  582. /**
  583. * Collects project information and its dependencies to enrich it with project configuration
  584. *
  585. * @public
  586. * @param {object} tree Dependency tree of the project
  587. * @returns {Promise<object>} Promise resolving with the dependency tree and enriched project configuration
  588. */
  589. processTree: function(tree) {
  590. return new ProjectPreprocessor({tree}).processTree();
  591. },
  592. _ProjectPreprocessor: ProjectPreprocessor
  593. };