diff --git a/README.md b/README.md index 280226b..96f486a 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,8 @@ This cache is automatically keyed by: - the github [`job_id`](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_id) (if `add-job-id-key` is `"true"`), -- the rustc release / host / hash, +- the rustc release / host / hash (for all installed toolchains when + available), - the following values, if `add-rust-environment-hash-key` is `"true"`: - the value of some compiler-specific environment variables (eg. RUSTFLAGS, etc), and - a hash of all `Cargo.lock` / `Cargo.toml` files found anywhere in the repository (if present). diff --git a/dist/restore/index.js b/dist/restore/index.js index b51c9a7..7ee1ddf 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -149504,6 +149504,8 @@ var external_crypto_default = /*#__PURE__*/__nccwpck_require__.n(external_crypto // EXTERNAL MODULE: external "fs/promises" var promises_ = __nccwpck_require__(91943); var promises_default = /*#__PURE__*/__nccwpck_require__.n(promises_); +;// CONCATENATED MODULE: external "stream/promises" +const external_stream_promises_namespaceObject = require("stream/promises"); // EXTERNAL MODULE: external "os" var external_os_ = __nccwpck_require__(70857); var external_os_default = /*#__PURE__*/__nccwpck_require__.n(external_os_); @@ -150783,6 +150785,7 @@ class Workspace { + const HOME = external_os_default().homedir(); const config_CARGO_HOME = process.env.CARGO_HOME || external_path_default().join(HOME, ".cargo"); const STATE_CONFIG = "RUST_CACHE_CONFIG"; @@ -150806,7 +150809,7 @@ class CacheConfig { /** The prefix portion of the cache key */ this.keyPrefix = ""; /** The rust version considered for the cache key */ - this.keyRust = ""; + this.keyRust = []; /** The environment variables considered for the cache key */ this.keyEnvs = []; /** The files considered for the cache key */ @@ -150860,12 +150863,15 @@ class CacheConfig { // The env vars are sorted, matched by prefix and hashed into the // resulting environment hash. let hasher = external_crypto_default().createHash("sha1"); - const rustVersion = await getRustVersion(cmdFormat); - let keyRust = `${rustVersion.release} ${rustVersion.host}`; - hasher.update(keyRust); - hasher.update(rustVersion["commit-hash"]); - keyRust += ` (${rustVersion["commit-hash"]})`; - self.keyRust = keyRust; + const rustVersions = Array.from(await getRustVersions(cmdFormat)); + // Doesn't matter how they're sorted, just as long as it's deterministic. + rustVersions.sort(); + for (const rustVersion of rustVersions) { + const { release, host, "commit-hash": commitHash } = rustVersion; + const keyRust = `${release} ${host} ${commitHash}`; + hasher.update(keyRust); + self.keyRust.push(keyRust); + } // these prefixes should cover most of the compiler / rust / cargo keys const envPrefixes = ["CARGO", "CC", "CFLAGS", "CXX", "CMAKE", "RUST"]; envPrefixes.push(...lib_core.getInput("env-vars").split(/\s+/).filter(Boolean)); @@ -150980,9 +150986,7 @@ class CacheConfig { } keyFiles = sort_and_uniq(keyFiles); for (const file of keyFiles) { - for await (const chunk of external_fs_default().createReadStream(file)) { - hasher.update(chunk); - } + await (0,external_stream_promises_namespaceObject.pipeline)((0,external_fs_.createReadStream)(file), hasher); } keyFiles.push(...parsedKeyFiles); self.keyFiles = sort_and_uniq(keyFiles); @@ -151051,7 +151055,10 @@ class CacheConfig { lib_core.info(`.. Prefix:`); lib_core.info(` - ${this.keyPrefix}`); lib_core.info(`.. Environment considered:`); - lib_core.info(` - Rust Version: ${this.keyRust}`); + lib_core.info(` - Rust Versions:`); + for (const rust of this.keyRust) { + lib_core.info(` - ${rust}`); + } for (const env of this.keyEnvs) { lib_core.info(` - ${env}`); } @@ -151086,9 +151093,31 @@ function isCacheUpToDate() { function digest(hasher) { return hasher.digest("hex").substring(0, HASH_LENGTH); } -async function getRustVersion(cmdFormat) { - const stdout = await getCmdOutput(cmdFormat, "rustc -vV"); - let splits = stdout +async function getRustVersions(cmdFormat) { + const versions = new Set(); + versions.add(parseRustVersion(await getCmdOutput(cmdFormat, "rustc -vV"))); + const stdout = await (async () => { + try { + return await getCmdOutput(cmdFormat, "rustup toolchain list --quiet"); + } + catch (e) { + lib_core.warning(`Error running rustup toolchain list, falling back to default toolchain only: ${e}`); + return undefined; + } + })(); + if (stdout !== undefined) { + for (const toolchain of stdout.split(/[\n\r]+/)) { + const trimmed = toolchain.trim(); + if (!trimmed) { + continue; + } + versions.add(parseRustVersion(await getCmdOutput(cmdFormat, `rustup run ${toolchain} rustc -vV`))); + } + } + return versions; +} +function parseRustVersion(stdout) { + const splits = stdout .split(/[\n\r]+/) .filter(Boolean) .map((s) => s.split(":").map((s) => s.trim())) @@ -151099,10 +151128,17 @@ async function globFiles(pattern) { const globber = await glob.create(pattern, { followSymbolicLinks: false, }); - // fs.statSync resolve the symbolic link and returns stat for the + // fs.stat resolve the symbolic link and returns stat for the // file it pointed to, so isFile would make sure the resolved // file is actually a regular file. - return (await globber.glob()).filter((file) => external_fs_default().statSync(file).isFile()); + const files = []; + for (const file of await globber.glob()) { + const stats = await promises_default().stat(file); + if (stats.isFile()) { + files.push(file); + } + } + return files; } function sort_and_uniq(a) { return a diff --git a/dist/save/index.js b/dist/save/index.js index ffaf496..bb929e7 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -149506,6 +149506,8 @@ var external_crypto_default = /*#__PURE__*/__nccwpck_require__.n(external_crypto // EXTERNAL MODULE: external "fs/promises" var promises_ = __nccwpck_require__(91943); var promises_default = /*#__PURE__*/__nccwpck_require__.n(promises_); +;// CONCATENATED MODULE: external "stream/promises" +const external_stream_promises_namespaceObject = require("stream/promises"); // EXTERNAL MODULE: external "os" var external_os_ = __nccwpck_require__(70857); var external_os_default = /*#__PURE__*/__nccwpck_require__.n(external_os_); @@ -150783,6 +150785,7 @@ class Workspace { + const HOME = external_os_default().homedir(); const CARGO_HOME = process.env.CARGO_HOME || external_path_default().join(HOME, ".cargo"); const STATE_CONFIG = "RUST_CACHE_CONFIG"; @@ -150806,7 +150809,7 @@ class CacheConfig { /** The prefix portion of the cache key */ this.keyPrefix = ""; /** The rust version considered for the cache key */ - this.keyRust = ""; + this.keyRust = []; /** The environment variables considered for the cache key */ this.keyEnvs = []; /** The files considered for the cache key */ @@ -150860,12 +150863,15 @@ class CacheConfig { // The env vars are sorted, matched by prefix and hashed into the // resulting environment hash. let hasher = external_crypto_default().createHash("sha1"); - const rustVersion = await getRustVersion(cmdFormat); - let keyRust = `${rustVersion.release} ${rustVersion.host}`; - hasher.update(keyRust); - hasher.update(rustVersion["commit-hash"]); - keyRust += ` (${rustVersion["commit-hash"]})`; - self.keyRust = keyRust; + const rustVersions = Array.from(await getRustVersions(cmdFormat)); + // Doesn't matter how they're sorted, just as long as it's deterministic. + rustVersions.sort(); + for (const rustVersion of rustVersions) { + const { release, host, "commit-hash": commitHash } = rustVersion; + const keyRust = `${release} ${host} ${commitHash}`; + hasher.update(keyRust); + self.keyRust.push(keyRust); + } // these prefixes should cover most of the compiler / rust / cargo keys const envPrefixes = ["CARGO", "CC", "CFLAGS", "CXX", "CMAKE", "RUST"]; envPrefixes.push(...core.getInput("env-vars").split(/\s+/).filter(Boolean)); @@ -150980,9 +150986,7 @@ class CacheConfig { } keyFiles = sort_and_uniq(keyFiles); for (const file of keyFiles) { - for await (const chunk of external_fs_default().createReadStream(file)) { - hasher.update(chunk); - } + await (0,external_stream_promises_namespaceObject.pipeline)((0,external_fs_.createReadStream)(file), hasher); } keyFiles.push(...parsedKeyFiles); self.keyFiles = sort_and_uniq(keyFiles); @@ -151051,7 +151055,10 @@ class CacheConfig { core.info(`.. Prefix:`); core.info(` - ${this.keyPrefix}`); core.info(`.. Environment considered:`); - core.info(` - Rust Version: ${this.keyRust}`); + core.info(` - Rust Versions:`); + for (const rust of this.keyRust) { + core.info(` - ${rust}`); + } for (const env of this.keyEnvs) { core.info(` - ${env}`); } @@ -151086,9 +151093,31 @@ function isCacheUpToDate() { function digest(hasher) { return hasher.digest("hex").substring(0, HASH_LENGTH); } -async function getRustVersion(cmdFormat) { - const stdout = await getCmdOutput(cmdFormat, "rustc -vV"); - let splits = stdout +async function getRustVersions(cmdFormat) { + const versions = new Set(); + versions.add(parseRustVersion(await getCmdOutput(cmdFormat, "rustc -vV"))); + const stdout = await (async () => { + try { + return await getCmdOutput(cmdFormat, "rustup toolchain list --quiet"); + } + catch (e) { + core.warning(`Error running rustup toolchain list, falling back to default toolchain only: ${e}`); + return undefined; + } + })(); + if (stdout !== undefined) { + for (const toolchain of stdout.split(/[\n\r]+/)) { + const trimmed = toolchain.trim(); + if (!trimmed) { + continue; + } + versions.add(parseRustVersion(await getCmdOutput(cmdFormat, `rustup run ${toolchain} rustc -vV`))); + } + } + return versions; +} +function parseRustVersion(stdout) { + const splits = stdout .split(/[\n\r]+/) .filter(Boolean) .map((s) => s.split(":").map((s) => s.trim())) @@ -151099,10 +151128,17 @@ async function globFiles(pattern) { const globber = await glob.create(pattern, { followSymbolicLinks: false, }); - // fs.statSync resolve the symbolic link and returns stat for the + // fs.stat resolve the symbolic link and returns stat for the // file it pointed to, so isFile would make sure the resolved // file is actually a regular file. - return (await globber.glob()).filter((file) => external_fs_default().statSync(file).isFile()); + const files = []; + for (const file of await globber.glob()) { + const stats = await promises_default().stat(file); + if (stats.isFile()) { + files.push(file); + } + } + return files; } function sort_and_uniq(a) { return a diff --git a/src/config.ts b/src/config.ts index d81aa4f..d562033 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,8 +1,9 @@ import * as core from "@actions/core"; import * as glob from "@actions/glob"; import crypto from "crypto"; -import fs from "fs"; -import fs_promises from "fs/promises"; +import fs from "fs/promises"; +import { createReadStream } from "fs"; +import { pipeline } from "stream/promises"; import os from "os"; import path from "path"; import * as toml from "smol-toml"; @@ -40,7 +41,7 @@ export class CacheConfig { /** The prefix portion of the cache key */ private keyPrefix = ""; /** The rust version considered for the cache key */ - private keyRust = ""; + private keyRust: Array = []; /** The environment variables considered for the cache key */ private keyEnvs: Array = []; /** The files considered for the cache key */ @@ -103,14 +104,16 @@ export class CacheConfig { // resulting environment hash. let hasher = crypto.createHash("sha1"); - const rustVersion = await getRustVersion(cmdFormat); + const rustVersions = Array.from(await getRustVersions(cmdFormat)); + // Doesn't matter how they're sorted, just as long as it's deterministic. + rustVersions.sort(); - let keyRust = `${rustVersion.release} ${rustVersion.host}`; - hasher.update(keyRust); - hasher.update(rustVersion["commit-hash"]); - - keyRust += ` (${rustVersion["commit-hash"]})`; - self.keyRust = keyRust; + for (const rustVersion of rustVersions) { + const { release, host, "commit-hash": commitHash } = rustVersion; + const keyRust = `${release} ${host} ${commitHash}`; + hasher.update(keyRust); + self.keyRust.push(keyRust); + } // these prefixes should cover most of the compiler / rust / cargo keys const envPrefixes = ["CARGO", "CC", "CFLAGS", "CXX", "CMAKE", "RUST"]; @@ -178,7 +181,7 @@ export class CacheConfig { for (const cargo_manifest of cargo_manifests) { try { - const content = await fs_promises.readFile(cargo_manifest, { encoding: "utf8" }); + const content = await fs.readFile(cargo_manifest, { encoding: "utf8" }); // Use any since TomlPrimitive is not exposed const parsed = toml.parse(content) as { [key: string]: any }; @@ -225,7 +228,7 @@ export class CacheConfig { const cargo_lock = path.join(workspace.root, "Cargo.lock"); if (await exists(cargo_lock)) { try { - const content = await fs_promises.readFile(cargo_lock, { encoding: "utf8" }); + const content = await fs.readFile(cargo_lock, { encoding: "utf8" }); const parsed = toml.parse(content); if ((parsed.version !== 3 && parsed.version !== 4) || !("package" in parsed)) { @@ -253,9 +256,7 @@ export class CacheConfig { keyFiles = sort_and_uniq(keyFiles); for (const file of keyFiles) { - for await (const chunk of fs.createReadStream(file)) { - hasher.update(chunk); - } + await pipeline(createReadStream(file), hasher); } keyFiles.push(...parsedKeyFiles); @@ -335,7 +336,10 @@ export class CacheConfig { core.info(`.. Prefix:`); core.info(` - ${this.keyPrefix}`); core.info(`.. Environment considered:`); - core.info(` - Rust Version: ${this.keyRust}`); + core.info(` - Rust Versions:`); + for (const rust of this.keyRust) { + core.info(` - ${rust}`); + } for (const env of this.keyEnvs) { core.info(` - ${env}`); } @@ -380,9 +384,33 @@ interface RustVersion { "commit-hash": string; } -async function getRustVersion(cmdFormat: string): Promise { - const stdout = await getCmdOutput(cmdFormat, "rustc -vV"); - let splits = stdout +async function getRustVersions(cmdFormat: string): Promise> { + const versions = new Set(); + + versions.add(parseRustVersion(await getCmdOutput(cmdFormat, "rustc -vV"))); + + const stdout = await (async () => { + try { + return await getCmdOutput(cmdFormat, "rustup toolchain list --quiet"); + } catch (e) { + core.warning(`Error running rustup toolchain list, falling back to default toolchain only: ${e}`); + return undefined; + } + })(); + if (stdout !== undefined) { + for (const toolchain of stdout.split(/[\n\r]+/)) { + const trimmed = toolchain.trim(); + if (!trimmed) { + continue; + } + versions.add(parseRustVersion(await getCmdOutput(cmdFormat, `rustup run ${toolchain} rustc -vV`))); + } + } + return versions; +} + +function parseRustVersion(stdout: string): RustVersion { + const splits = stdout .split(/[\n\r]+/) .filter(Boolean) .map((s) => s.split(":").map((s) => s.trim())) @@ -394,10 +422,17 @@ async function globFiles(pattern: string): Promise { const globber = await glob.create(pattern, { followSymbolicLinks: false, }); - // fs.statSync resolve the symbolic link and returns stat for the + // fs.stat resolve the symbolic link and returns stat for the // file it pointed to, so isFile would make sure the resolved // file is actually a regular file. - return (await globber.glob()).filter((file) => fs.statSync(file).isFile()); + const files = []; + for (const file of await globber.glob()) { + const stats = await fs.stat(file); + if (stats.isFile()) { + files.push(file); + } + } + return files; } function sort_and_uniq(a: string[]) {