wip: add incremental mtime restore

action.yml

@@ -48,6 +48,10 @@ inputs:
     description: "Check if a cache entry exists without downloading the cache"
     required: false
     default: "false"
+  incremental:
+    description: "Determines whether to cache incremental builds - speeding up builds for more disk usage. Defaults to false."
+    required: false
+    default: "false"
 outputs:
   cache-hit:
     description: "A boolean value that indicates an exact match was found."

src/cleanup.ts

@@ -7,7 +7,7 @@ import { CARGO_HOME } from "./config";
 import { exists } from "./utils";
 import { Packages } from "./workspace";
 
-export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp = false) {
+export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp: boolean, incremental: boolean) {
   core.debug(`cleaning target directory "${targetDir}"`);
 
   // remove all *files* from the profile directory
@@ -21,18 +21,18 @@ export async function cleanTargetDir(targetDir: string, packages: Packages, chec
 
       try {
         if (isNestedTarget) {
-          await cleanTargetDir(dirName, packages, checkTimestamp);
+          await cleanTargetDir(dirName, packages, checkTimestamp, incremental);
         } else {
-          await cleanProfileTarget(dirName, packages, checkTimestamp);
+          await cleanProfileTarget(dirName, packages, checkTimestamp, incremental);
         }
-      } catch {}
+      } catch { }
     } else if (dirent.name !== "CACHEDIR.TAG") {
       await rm(dir.path, dirent);
     }
   }
 }
 
-async function cleanProfileTarget(profileDir: string, packages: Packages, checkTimestamp = false) {
+async function cleanProfileTarget(profileDir: string, packages: Packages, checkTimestamp: boolean, incremental: boolean) {
   core.debug(`cleaning profile directory "${profileDir}"`);
 
   // Quite a few testing utility crates store compilation artifacts as nested
@@ -42,15 +42,44 @@ async function cleanProfileTarget(profileDir: string, packages: Packages, checkT
     try {
       // https://github.com/vertexclique/kaos/blob/9876f6c890339741cc5be4b7cb9df72baa5a6d79/src/cargo.rs#L25
       // https://github.com/eupn/macrotest/blob/c4151a5f9f545942f4971980b5d264ebcd0b1d11/src/cargo.rs#L27
-      cleanTargetDir(path.join(profileDir, "target"), packages, checkTimestamp);
-    } catch {}
+      cleanTargetDir(path.join(profileDir, "target"), packages, checkTimestamp, incremental);
+    } catch { }
+
     try {
       // https://github.com/dtolnay/trybuild/blob/eec8ca6cb9b8f53d0caf1aa499d99df52cae8b40/src/cargo.rs#L50
-      cleanTargetDir(path.join(profileDir, "trybuild"), packages, checkTimestamp);
-    } catch {}
+      cleanTargetDir(path.join(profileDir, "trybuild"), packages, checkTimestamp, incremental);
+    } catch { }
 
     // Delete everything else.
-    await rmExcept(profileDir, new Set(["target", "trybuild"]), checkTimestamp);
+    let except = new Set(["target", "trybuild"]);
+
+    // Keep the incremental folder if incremental builds are enabled
+    if (incremental) {
+      except.add("incremental");
+
+      // Traverse the incremental folder recursively and collect the modified times in a map
+      const incrementalDir = path.join(profileDir, "incremental");
+      const modifiedTimes = new Map<string, number>();
+      const fillModifiedTimes = async (dir: string) => {
+        const dirEntries = await fs.promises.opendir(dir);
+        for await (const dirent of dirEntries) {
+          if (dirent.isDirectory()) {
+            await fillModifiedTimes(path.join(dir, dirent.name));
+          } else {
+            const fileName = path.join(dir, dirent.name);
+            const { mtime } = await fs.promises.stat(fileName);
+            modifiedTimes.set(fileName, mtime.getTime());
+          }
+        }
+      };
+      await fillModifiedTimes(incrementalDir);
+
+      // Write the modified times to the incremental folder
+      const contents = JSON.stringify({ modifiedTimes });
+      await fs.promises.writeFile(path.join(incrementalDir, "incremental-restore.json"), contents);
+    }
+
+    await rmExcept(profileDir, except, checkTimestamp);
 
     return;
   }
@@ -86,7 +115,7 @@ export async function getCargoBins(): Promise<Set<string>> {
         bins.add(bin);
       }
     }
-  } catch {}
+  } catch { }
   return bins;
 }
 
@@ -117,7 +146,7 @@ export async function cleanRegistry(packages: Packages, crates = true) {
     const credentials = path.join(CARGO_HOME, ".cargo", "credentials.toml");
     core.debug(`deleting "${credentials}"`);
     await fs.promises.unlink(credentials);
-  } catch {}
+  } catch { }
 
   // `.cargo/registry/index`
   let pkgSet = new Set(packages.map((p) => p.name));
@@ -229,7 +258,7 @@ export async function cleanGit(packages: Packages) {
         await rm(dir.path, dirent);
       }
     }
-  } catch {}
+  } catch { }
 
   // clean the checkouts
   try {
@@ -250,7 +279,7 @@ export async function cleanGit(packages: Packages) {
         }
       }
     }
-  } catch {}
+  } catch { }
 }
 
 const ONE_WEEK = 7 * 24 * 3600 * 1000;
@@ -302,7 +331,7 @@ async function rm(parent: string, dirent: fs.Dirent) {
     } else if (dirent.isDirectory()) {
       await io.rmRF(fileName);
     }
-  } catch {}
+  } catch { }
 }
 
 async function rmRF(dirName: string) {

src/config.ts

@@ -34,6 +34,9 @@ export class CacheConfig {
   /** The cargo binaries present during main step */
   public cargoBins: Array<string> = [];
 
+  /** Whether to cache incremental builds */
+  public incremental: boolean = false;
+
   /** The prefix portion of the cache key */
   private keyPrefix = "";
   /** The rust version considered for the cache key */
@@ -43,7 +46,7 @@ export class CacheConfig {
   /** The files considered for the cache key */
   private keyFiles: Array<string> = [];
 
-  private constructor() {}
+  private constructor() { }
 
   /**
    * Constructs a [`CacheConfig`] with all the paths and keys.

src/incremental.ts

@@ -0,0 +1,48 @@
+import * as core from "@actions/core";
+import * as io from "@actions/io";
+import fs from "fs";
+import path from "path";
+
+import { CARGO_HOME } from "./config";
+import { exists } from "./utils";
+import { Packages } from "./workspace";
+
+
+export async function restoreIncremental(targetDir: string) {
+  core.debug(`restoring incremental directory "${targetDir}"`);
+
+
+  let dir = await fs.promises.opendir(targetDir);
+  for await (const dirent of dir) {
+    if (dirent.isDirectory()) {
+      let dirName = path.join(dir.path, dirent.name);
+      // is it a profile dir, or a nested target dir?
+      let isNestedTarget =
+        (await exists(path.join(dirName, "CACHEDIR.TAG"))) || (await exists(path.join(dirName, ".rustc_info.json")));
+
+      try {
+        if (isNestedTarget) {
+          await restoreIncremental(dirName);
+        } else {
+          await restoreIncrementalProfile(dirName);
+        } restoreIncrementalProfile
+      } catch { }
+    }
+  }
+}
+
+async function restoreIncrementalProfile(dirName: string) {
+  core.debug(`restoring incremental profile directory "${dirName}"`);
+  const incrementalJson = path.join(dirName, "incremental-restore.json");
+  if (await exists(incrementalJson)) {
+    const contents = await fs.promises.readFile(incrementalJson, "utf8");
+    const { modifiedTimes } = JSON.parse(contents);
+
+    // Write the mtimes to all the files in the profile directory
+    for (const fileName of Object.keys(modifiedTimes)) {
+      const mtime = modifiedTimes[fileName];
+      const filePath = path.join(dirName, fileName);
+      await fs.promises.utimes(filePath, new Date(mtime), new Date(mtime));
+    }
+  }
+}

src/restore.ts

@@ -3,6 +3,7 @@ import * as core from "@actions/core";
 import { cleanTargetDir } from "./cleanup";
 import { CacheConfig } from "./config";
 import { getCacheProvider, reportError } from "./utils";
+import { restoreIncremental } from "./incremental";
 
 process.on("uncaughtException", (e) => {
   core.error(e.message);
@@ -27,12 +28,15 @@ async function run() {
     var lookupOnly = core.getInput("lookup-only").toLowerCase() === "true";
 
     core.exportVariable("CACHE_ON_FAILURE", cacheOnFailure);
-    core.exportVariable("CARGO_INCREMENTAL", 0);
 
     const config = await CacheConfig.new();
     config.printInfo(cacheProvider);
     core.info("");
 
+    if (!config.incremental) {
+      core.exportVariable("CARGO_INCREMENTAL", 0);
+    }
+
     core.info(`... ${lookupOnly ? "Checking" : "Restoring"} cache ...`);
     const key = config.cacheKey;
     // Pass a copy of cachePaths to avoid mutating the original array as reported by:
@@ -44,12 +48,19 @@ async function run() {
     if (restoreKey) {
       const match = restoreKey === key;
       core.info(`${lookupOnly ? "Found" : "Restored from"} cache key "${restoreKey}" full match: ${match}.`);
+
+      if (config.incremental) {
+        for (const workspace of config.workspaces) {
+          await restoreIncremental(workspace.target);
+        }
+      }
+
       if (!match) {
         // pre-clean the target directory on cache mismatch
         for (const workspace of config.workspaces) {
           try {
-            await cleanTargetDir(workspace.target, [], true);
-          } catch {}
+            await cleanTargetDir(workspace.target, [], true, false);
+          } catch { }
         }
 
         // We restored the cache but it is not a full match.

src/save.ts

@@ -42,7 +42,7 @@ async function run() {
       allPackages.push(...packages);
       try {
         core.info(`... Cleaning ${workspace.target} ...`);
-        await cleanTargetDir(workspace.target, packages);
+        await cleanTargetDir(workspace.target, packages, false, config.incremental);
       } catch (e) {
         core.debug(`${(e as any).stack}`);
       }
@@ -90,5 +90,5 @@ async function macOsWorkaround() {
     // Workaround for https://github.com/actions/cache/issues/403
     // Also see https://github.com/rust-lang/cargo/issues/8603
     await exec.exec("sudo", ["/usr/sbin/purge"], { silent: true });
-  } catch {}
+  } catch { }
 }