You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
660 lines
18 KiB
660 lines
18 KiB
/*
|
|
MIT License http://www.opensource.org/licenses/mit-license.php
|
|
Author Tobias Koppers @sokra
|
|
*/
|
|
|
|
"use strict";
|
|
|
|
const asyncLib = require("neo-async");
|
|
const { SyncHook, MultiHook } = require("tapable");
|
|
|
|
const ConcurrentCompilationError = require("./ConcurrentCompilationError");
|
|
const MultiStats = require("./MultiStats");
|
|
const MultiWatching = require("./MultiWatching");
|
|
const WebpackError = require("./WebpackError");
|
|
const ArrayQueue = require("./util/ArrayQueue");
|
|
|
|
/** @template T @typedef {import("tapable").AsyncSeriesHook<T>} AsyncSeriesHook<T> */
|
|
/** @template T @template R @typedef {import("tapable").SyncBailHook<T, R>} SyncBailHook<T, R> */
|
|
/** @typedef {import("../declarations/WebpackOptions").WatchOptions} WatchOptions */
|
|
/** @typedef {import("./Compiler")} Compiler */
|
|
/** @typedef {import("./Stats")} Stats */
|
|
/** @typedef {import("./Watching")} Watching */
|
|
/** @typedef {import("./logging/Logger").Logger} Logger */
|
|
/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
|
|
/** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
|
|
/** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
|
|
/** @typedef {import("./util/fs").WatchFileSystem} WatchFileSystem */
|
|
|
|
/**
|
|
* @template T
|
|
* @callback Callback
|
|
* @param {Error | null} err
|
|
* @param {T=} result
|
|
*/
|
|
|
|
/**
|
|
* @callback RunWithDependenciesHandler
|
|
* @param {Compiler} compiler
|
|
* @param {Callback<MultiStats>} callback
|
|
*/
|
|
|
|
/**
|
|
* @typedef {object} MultiCompilerOptions
|
|
* @property {number=} parallelism how many Compilers are allows to run at the same time in parallel
|
|
*/
|
|
|
|
module.exports = class MultiCompiler {
|
|
/**
|
|
* @param {Compiler[] | Record<string, Compiler>} compilers child compilers
|
|
* @param {MultiCompilerOptions} options options
|
|
*/
|
|
constructor(compilers, options) {
|
|
if (!Array.isArray(compilers)) {
|
|
/** @type {Compiler[]} */
|
|
compilers = Object.keys(compilers).map(name => {
|
|
/** @type {Record<string, Compiler>} */
|
|
(compilers)[name].name = name;
|
|
return /** @type {Record<string, Compiler>} */ (compilers)[name];
|
|
});
|
|
}
|
|
|
|
this.hooks = Object.freeze({
|
|
/** @type {SyncHook<[MultiStats]>} */
|
|
done: new SyncHook(["stats"]),
|
|
/** @type {MultiHook<SyncHook<[string | null, number]>>} */
|
|
invalid: new MultiHook(compilers.map(c => c.hooks.invalid)),
|
|
/** @type {MultiHook<AsyncSeriesHook<[Compiler]>>} */
|
|
run: new MultiHook(compilers.map(c => c.hooks.run)),
|
|
/** @type {SyncHook<[]>} */
|
|
watchClose: new SyncHook([]),
|
|
/** @type {MultiHook<AsyncSeriesHook<[Compiler]>>} */
|
|
watchRun: new MultiHook(compilers.map(c => c.hooks.watchRun)),
|
|
/** @type {MultiHook<SyncBailHook<[string, string, any[]], true>>} */
|
|
infrastructureLog: new MultiHook(
|
|
compilers.map(c => c.hooks.infrastructureLog)
|
|
)
|
|
});
|
|
this.compilers = compilers;
|
|
/** @type {MultiCompilerOptions} */
|
|
this._options = {
|
|
parallelism: options.parallelism || Infinity
|
|
};
|
|
/** @type {WeakMap<Compiler, string[]>} */
|
|
this.dependencies = new WeakMap();
|
|
this.running = false;
|
|
|
|
/** @type {(Stats | null)[]} */
|
|
const compilerStats = this.compilers.map(() => null);
|
|
let doneCompilers = 0;
|
|
for (let index = 0; index < this.compilers.length; index++) {
|
|
const compiler = this.compilers[index];
|
|
const compilerIndex = index;
|
|
let compilerDone = false;
|
|
// eslint-disable-next-line no-loop-func
|
|
compiler.hooks.done.tap("MultiCompiler", stats => {
|
|
if (!compilerDone) {
|
|
compilerDone = true;
|
|
doneCompilers++;
|
|
}
|
|
compilerStats[compilerIndex] = stats;
|
|
if (doneCompilers === this.compilers.length) {
|
|
this.hooks.done.call(
|
|
new MultiStats(/** @type {Stats[]} */ (compilerStats))
|
|
);
|
|
}
|
|
});
|
|
// eslint-disable-next-line no-loop-func
|
|
compiler.hooks.invalid.tap("MultiCompiler", () => {
|
|
if (compilerDone) {
|
|
compilerDone = false;
|
|
doneCompilers--;
|
|
}
|
|
});
|
|
}
|
|
this._validateCompilersOptions();
|
|
}
|
|
|
|
_validateCompilersOptions() {
|
|
if (this.compilers.length < 2) return;
|
|
/**
|
|
* @param {Compiler} compiler compiler
|
|
* @param {WebpackError} warning warning
|
|
*/
|
|
const addWarning = (compiler, warning) => {
|
|
compiler.hooks.thisCompilation.tap("MultiCompiler", compilation => {
|
|
compilation.warnings.push(warning);
|
|
});
|
|
};
|
|
const cacheNames = new Set();
|
|
for (const compiler of this.compilers) {
|
|
if (compiler.options.cache && "name" in compiler.options.cache) {
|
|
const name = compiler.options.cache.name;
|
|
if (cacheNames.has(name)) {
|
|
addWarning(
|
|
compiler,
|
|
new WebpackError(
|
|
`${
|
|
compiler.name
|
|
? `Compiler with name "${compiler.name}" doesn't use unique cache name. `
|
|
: ""
|
|
}Please set unique "cache.name" option. Name "${name}" already used.`
|
|
)
|
|
);
|
|
} else {
|
|
cacheNames.add(name);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
get options() {
|
|
return Object.assign(
|
|
this.compilers.map(c => c.options),
|
|
this._options
|
|
);
|
|
}
|
|
|
|
get outputPath() {
|
|
let commonPath = this.compilers[0].outputPath;
|
|
for (const compiler of this.compilers) {
|
|
while (
|
|
compiler.outputPath.indexOf(commonPath) !== 0 &&
|
|
/[/\\]/.test(commonPath)
|
|
) {
|
|
commonPath = commonPath.replace(/[/\\][^/\\]*$/, "");
|
|
}
|
|
}
|
|
|
|
if (!commonPath && this.compilers[0].outputPath[0] === "/") return "/";
|
|
return commonPath;
|
|
}
|
|
|
|
get inputFileSystem() {
|
|
throw new Error("Cannot read inputFileSystem of a MultiCompiler");
|
|
}
|
|
|
|
/**
|
|
* @param {InputFileSystem} value the new input file system
|
|
*/
|
|
set inputFileSystem(value) {
|
|
for (const compiler of this.compilers) {
|
|
compiler.inputFileSystem = value;
|
|
}
|
|
}
|
|
|
|
get outputFileSystem() {
|
|
throw new Error("Cannot read outputFileSystem of a MultiCompiler");
|
|
}
|
|
|
|
/**
|
|
* @param {OutputFileSystem} value the new output file system
|
|
*/
|
|
set outputFileSystem(value) {
|
|
for (const compiler of this.compilers) {
|
|
compiler.outputFileSystem = value;
|
|
}
|
|
}
|
|
|
|
get watchFileSystem() {
|
|
throw new Error("Cannot read watchFileSystem of a MultiCompiler");
|
|
}
|
|
|
|
/**
|
|
* @param {WatchFileSystem} value the new watch file system
|
|
*/
|
|
set watchFileSystem(value) {
|
|
for (const compiler of this.compilers) {
|
|
compiler.watchFileSystem = value;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {IntermediateFileSystem} value the new intermediate file system
|
|
*/
|
|
set intermediateFileSystem(value) {
|
|
for (const compiler of this.compilers) {
|
|
compiler.intermediateFileSystem = value;
|
|
}
|
|
}
|
|
|
|
get intermediateFileSystem() {
|
|
throw new Error("Cannot read outputFileSystem of a MultiCompiler");
|
|
}
|
|
|
|
/**
|
|
* @param {string | (function(): string)} name name of the logger, or function called once to get the logger name
|
|
* @returns {Logger} a logger with that name
|
|
*/
|
|
getInfrastructureLogger(name) {
|
|
return this.compilers[0].getInfrastructureLogger(name);
|
|
}
|
|
|
|
/**
|
|
* @param {Compiler} compiler the child compiler
|
|
* @param {string[]} dependencies its dependencies
|
|
* @returns {void}
|
|
*/
|
|
setDependencies(compiler, dependencies) {
|
|
this.dependencies.set(compiler, dependencies);
|
|
}
|
|
|
|
/**
|
|
* @param {Callback<MultiStats>} callback signals when the validation is complete
|
|
* @returns {boolean} true if the dependencies are valid
|
|
*/
|
|
validateDependencies(callback) {
|
|
/** @type {Set<{source: Compiler, target: Compiler}>} */
|
|
const edges = new Set();
|
|
/** @type {string[]} */
|
|
const missing = [];
|
|
/**
|
|
* @param {Compiler} compiler compiler
|
|
* @returns {boolean} target was found
|
|
*/
|
|
const targetFound = compiler => {
|
|
for (const edge of edges) {
|
|
if (edge.target === compiler) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
/**
|
|
* @param {{source: Compiler, target: Compiler}} e1 edge 1
|
|
* @param {{source: Compiler, target: Compiler}} e2 edge 2
|
|
* @returns {number} result
|
|
*/
|
|
const sortEdges = (e1, e2) =>
|
|
/** @type {string} */
|
|
(e1.source.name).localeCompare(/** @type {string} */ (e2.source.name)) ||
|
|
/** @type {string} */
|
|
(e1.target.name).localeCompare(/** @type {string} */ (e2.target.name));
|
|
for (const source of this.compilers) {
|
|
const dependencies = this.dependencies.get(source);
|
|
if (dependencies) {
|
|
for (const dep of dependencies) {
|
|
const target = this.compilers.find(c => c.name === dep);
|
|
if (!target) {
|
|
missing.push(dep);
|
|
} else {
|
|
edges.add({
|
|
source,
|
|
target
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
/** @type {string[]} */
|
|
const errors = missing.map(m => `Compiler dependency \`${m}\` not found.`);
|
|
const stack = this.compilers.filter(c => !targetFound(c));
|
|
while (stack.length > 0) {
|
|
const current = stack.pop();
|
|
for (const edge of edges) {
|
|
if (edge.source === current) {
|
|
edges.delete(edge);
|
|
const target = edge.target;
|
|
if (!targetFound(target)) {
|
|
stack.push(target);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (edges.size > 0) {
|
|
/** @type {string[]} */
|
|
const lines = Array.from(edges)
|
|
.sort(sortEdges)
|
|
.map(edge => `${edge.source.name} -> ${edge.target.name}`);
|
|
lines.unshift("Circular dependency found in compiler dependencies.");
|
|
errors.unshift(lines.join("\n"));
|
|
}
|
|
if (errors.length > 0) {
|
|
const message = errors.join("\n");
|
|
callback(new Error(message));
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// TODO webpack 6 remove
|
|
/**
|
|
* @deprecated This method should have been private
|
|
* @param {Compiler[]} compilers the child compilers
|
|
* @param {RunWithDependenciesHandler} fn a handler to run for each compiler
|
|
* @param {Callback<MultiStats>} callback the compiler's handler
|
|
* @returns {void}
|
|
*/
|
|
runWithDependencies(compilers, fn, callback) {
|
|
const fulfilledNames = new Set();
|
|
let remainingCompilers = compilers;
|
|
/**
|
|
* @param {string} d dependency
|
|
* @returns {boolean} when dependency was fulfilled
|
|
*/
|
|
const isDependencyFulfilled = d => fulfilledNames.has(d);
|
|
/**
|
|
* @returns {Compiler[]} compilers
|
|
*/
|
|
const getReadyCompilers = () => {
|
|
const readyCompilers = [];
|
|
const list = remainingCompilers;
|
|
remainingCompilers = [];
|
|
for (const c of list) {
|
|
const dependencies = this.dependencies.get(c);
|
|
const ready =
|
|
!dependencies || dependencies.every(isDependencyFulfilled);
|
|
if (ready) {
|
|
readyCompilers.push(c);
|
|
} else {
|
|
remainingCompilers.push(c);
|
|
}
|
|
}
|
|
return readyCompilers;
|
|
};
|
|
/**
|
|
* @param {Callback<MultiStats>} callback callback
|
|
* @returns {void}
|
|
*/
|
|
const runCompilers = callback => {
|
|
if (remainingCompilers.length === 0) return callback(null);
|
|
asyncLib.map(
|
|
getReadyCompilers(),
|
|
(compiler, callback) => {
|
|
fn(compiler, err => {
|
|
if (err) return callback(err);
|
|
fulfilledNames.add(compiler.name);
|
|
runCompilers(callback);
|
|
});
|
|
},
|
|
(err, results) => {
|
|
callback(err, /** @type {TODO} */ (results));
|
|
}
|
|
);
|
|
};
|
|
runCompilers(callback);
|
|
}
|
|
|
|
/**
|
|
* @template SetupResult
|
|
* @param {function(Compiler, number, Callback<Stats>, function(): boolean, function(): void, function(): void): SetupResult} setup setup a single compiler
|
|
* @param {function(Compiler, SetupResult, Callback<Stats>): void} run run/continue a single compiler
|
|
* @param {Callback<MultiStats>} callback callback when all compilers are done, result includes Stats of all changed compilers
|
|
* @returns {SetupResult[]} result of setup
|
|
*/
|
|
_runGraph(setup, run, callback) {
|
|
/** @typedef {{ compiler: Compiler, setupResult: undefined | SetupResult, result: undefined | Stats, state: "pending" | "blocked" | "queued" | "starting" | "running" | "running-outdated" | "done", children: Node[], parents: Node[] }} Node */
|
|
|
|
// State transitions for nodes:
|
|
// -> blocked (initial)
|
|
// blocked -> starting [running++] (when all parents done)
|
|
// queued -> starting [running++] (when processing the queue)
|
|
// starting -> running (when run has been called)
|
|
// running -> done [running--] (when compilation is done)
|
|
// done -> pending (when invalidated from file change)
|
|
// pending -> blocked [add to queue] (when invalidated from aggregated changes)
|
|
// done -> blocked [add to queue] (when invalidated, from parent invalidation)
|
|
// running -> running-outdated (when invalidated, either from change or parent invalidation)
|
|
// running-outdated -> blocked [running--] (when compilation is done)
|
|
|
|
/** @type {Node[]} */
|
|
const nodes = this.compilers.map(compiler => ({
|
|
compiler,
|
|
setupResult: undefined,
|
|
result: undefined,
|
|
state: "blocked",
|
|
children: [],
|
|
parents: []
|
|
}));
|
|
/** @type {Map<string, Node>} */
|
|
const compilerToNode = new Map();
|
|
for (const node of nodes) {
|
|
compilerToNode.set(/** @type {string} */ (node.compiler.name), node);
|
|
}
|
|
for (const node of nodes) {
|
|
const dependencies = this.dependencies.get(node.compiler);
|
|
if (!dependencies) continue;
|
|
for (const dep of dependencies) {
|
|
const parent = /** @type {Node} */ (compilerToNode.get(dep));
|
|
node.parents.push(parent);
|
|
parent.children.push(node);
|
|
}
|
|
}
|
|
/** @type {ArrayQueue<Node>} */
|
|
const queue = new ArrayQueue();
|
|
for (const node of nodes) {
|
|
if (node.parents.length === 0) {
|
|
node.state = "queued";
|
|
queue.enqueue(node);
|
|
}
|
|
}
|
|
let errored = false;
|
|
let running = 0;
|
|
const parallelism = /** @type {number} */ (this._options.parallelism);
|
|
/**
|
|
* @param {Node} node node
|
|
* @param {(Error | null)=} err error
|
|
* @param {Stats=} stats result
|
|
* @returns {void}
|
|
*/
|
|
const nodeDone = (node, err, stats) => {
|
|
if (errored) return;
|
|
if (err) {
|
|
errored = true;
|
|
return asyncLib.each(
|
|
nodes,
|
|
(node, callback) => {
|
|
if (node.compiler.watching) {
|
|
node.compiler.watching.close(callback);
|
|
} else {
|
|
callback();
|
|
}
|
|
},
|
|
() => callback(err)
|
|
);
|
|
}
|
|
node.result = stats;
|
|
running--;
|
|
if (node.state === "running") {
|
|
node.state = "done";
|
|
for (const child of node.children) {
|
|
if (child.state === "blocked") queue.enqueue(child);
|
|
}
|
|
} else if (node.state === "running-outdated") {
|
|
node.state = "blocked";
|
|
queue.enqueue(node);
|
|
}
|
|
processQueue();
|
|
};
|
|
/**
|
|
* @param {Node} node node
|
|
* @returns {void}
|
|
*/
|
|
const nodeInvalidFromParent = node => {
|
|
if (node.state === "done") {
|
|
node.state = "blocked";
|
|
} else if (node.state === "running") {
|
|
node.state = "running-outdated";
|
|
}
|
|
for (const child of node.children) {
|
|
nodeInvalidFromParent(child);
|
|
}
|
|
};
|
|
/**
|
|
* @param {Node} node node
|
|
* @returns {void}
|
|
*/
|
|
const nodeInvalid = node => {
|
|
if (node.state === "done") {
|
|
node.state = "pending";
|
|
} else if (node.state === "running") {
|
|
node.state = "running-outdated";
|
|
}
|
|
for (const child of node.children) {
|
|
nodeInvalidFromParent(child);
|
|
}
|
|
};
|
|
/**
|
|
* @param {Node} node node
|
|
* @returns {void}
|
|
*/
|
|
const nodeChange = node => {
|
|
nodeInvalid(node);
|
|
if (node.state === "pending") {
|
|
node.state = "blocked";
|
|
}
|
|
if (node.state === "blocked") {
|
|
queue.enqueue(node);
|
|
processQueue();
|
|
}
|
|
};
|
|
|
|
/** @type {SetupResult[]} */
|
|
const setupResults = [];
|
|
for (const [i, node] of nodes.entries()) {
|
|
setupResults.push(
|
|
(node.setupResult = setup(
|
|
node.compiler,
|
|
i,
|
|
nodeDone.bind(null, node),
|
|
() => node.state !== "starting" && node.state !== "running",
|
|
() => nodeChange(node),
|
|
() => nodeInvalid(node)
|
|
))
|
|
);
|
|
}
|
|
let processing = true;
|
|
const processQueue = () => {
|
|
if (processing) return;
|
|
processing = true;
|
|
process.nextTick(processQueueWorker);
|
|
};
|
|
const processQueueWorker = () => {
|
|
// eslint-disable-next-line no-unmodified-loop-condition
|
|
while (running < parallelism && queue.length > 0 && !errored) {
|
|
const node = /** @type {Node} */ (queue.dequeue());
|
|
if (
|
|
node.state === "queued" ||
|
|
(node.state === "blocked" &&
|
|
node.parents.every(p => p.state === "done"))
|
|
) {
|
|
running++;
|
|
node.state = "starting";
|
|
run(
|
|
node.compiler,
|
|
/** @type {SetupResult} */ (node.setupResult),
|
|
nodeDone.bind(null, node)
|
|
);
|
|
node.state = "running";
|
|
}
|
|
}
|
|
processing = false;
|
|
if (
|
|
!errored &&
|
|
running === 0 &&
|
|
nodes.every(node => node.state === "done")
|
|
) {
|
|
const stats = [];
|
|
for (const node of nodes) {
|
|
const result = node.result;
|
|
if (result) {
|
|
node.result = undefined;
|
|
stats.push(result);
|
|
}
|
|
}
|
|
if (stats.length > 0) {
|
|
callback(null, new MultiStats(stats));
|
|
}
|
|
}
|
|
};
|
|
processQueueWorker();
|
|
return setupResults;
|
|
}
|
|
|
|
/**
|
|
* @param {WatchOptions|WatchOptions[]} watchOptions the watcher's options
|
|
* @param {Callback<MultiStats>} handler signals when the call finishes
|
|
* @returns {MultiWatching} a compiler watcher
|
|
*/
|
|
watch(watchOptions, handler) {
|
|
if (this.running) {
|
|
return handler(new ConcurrentCompilationError());
|
|
}
|
|
this.running = true;
|
|
|
|
if (this.validateDependencies(handler)) {
|
|
const watchings = this._runGraph(
|
|
(compiler, idx, callback, isBlocked, setChanged, setInvalid) => {
|
|
const watching = compiler.watch(
|
|
Array.isArray(watchOptions) ? watchOptions[idx] : watchOptions,
|
|
callback
|
|
);
|
|
if (watching) {
|
|
watching._onInvalid = setInvalid;
|
|
watching._onChange = setChanged;
|
|
watching._isBlocked = isBlocked;
|
|
}
|
|
return watching;
|
|
},
|
|
(compiler, watching, callback) => {
|
|
if (compiler.watching !== watching) return;
|
|
if (!watching.running) watching.invalidate();
|
|
},
|
|
handler
|
|
);
|
|
return new MultiWatching(watchings, this);
|
|
}
|
|
|
|
return new MultiWatching([], this);
|
|
}
|
|
|
|
/**
|
|
* @param {Callback<MultiStats>} callback signals when the call finishes
|
|
* @returns {void}
|
|
*/
|
|
run(callback) {
|
|
if (this.running) {
|
|
return callback(new ConcurrentCompilationError());
|
|
}
|
|
this.running = true;
|
|
|
|
if (this.validateDependencies(callback)) {
|
|
this._runGraph(
|
|
() => {},
|
|
(compiler, setupResult, callback) => compiler.run(callback),
|
|
(err, stats) => {
|
|
this.running = false;
|
|
|
|
if (callback !== undefined) {
|
|
return callback(err, stats);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
|
|
purgeInputFileSystem() {
|
|
for (const compiler of this.compilers) {
|
|
if (compiler.inputFileSystem && compiler.inputFileSystem.purge) {
|
|
compiler.inputFileSystem.purge();
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {Callback<void>} callback signals when the compiler closes
|
|
* @returns {void}
|
|
*/
|
|
close(callback) {
|
|
asyncLib.each(
|
|
this.compilers,
|
|
(compiler, callback) => {
|
|
compiler.close(callback);
|
|
},
|
|
error => {
|
|
callback(error);
|
|
}
|
|
);
|
|
}
|
|
};
|