Actions/rust-cache
1
Fork 0
mirror of https://github.com/Swatinem/rust-cache.git synced 2025-10-31 19:33:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

explicitly close dir handles, add more logging, cleanups

Browse Source
This commit is contained in:
Arpad Borsos 2022-09-04 13:10:36 +02:00
parent 213334cd98
commit be4be3720d
No known key found for this signature in database
GPG Key ID: FC7BCA77824B3298
5 changed files with 217 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
dist/restore/index.js vendored
View File

@ -64346,7 +64346,7 @@ var cache = __nccwpck_require__(7799);
// EXTERNAL MODULE: ./node_modules/@actions/core/lib/core.js
var lib_core = __nccwpck_require__(2186);
// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js
var io = __nccwpck_require__(7436);
var lib_io = __nccwpck_require__(7436);
// EXTERNAL MODULE: external "fs"
var external_fs_ = __nccwpck_require__(7147);
var external_fs_default = /*#__PURE__*/__nccwpck_require__.n(external_fs_);
@ -64394,6 +64394,7 @@ async function getCmdOutput(cmd, args = [], options = {}) {
;// CONCATENATED MODULE: ./src/workspace.ts
const SAVE_TARGETS = new Set(["lib", "proc-macro"]);
class Workspace {
constructor(root, target) {
this.root = root;
@ -64409,7 +64410,7 @@ class Workspace {
if (pkg.manifest_path.startsWith(this.root)) {
continue;
}
const targets = pkg.targets.filter((t) => t.kind[0] === "lib").map((t) => t.name);
const targets = pkg.targets.filter((t) => t.kind.some((kind) => SAVE_TARGETS.has(kind))).map((t) => t.name);
packages.push({ name: pkg.name, version: pkg.version, targets, path: external_path_default().dirname(pkg.manifest_path) });
}
}
@ -64594,10 +64595,10 @@ async function getRustVersion() {
async function cleanTargetDir(targetDir, packages) {
let dir;
async function cleanTargetDir(targetDir, packages, checkTimestamp = false) {
lib_core.debug(`cleaning target directory "${targetDir}"`);
// remove all *files* from the profile directory
dir = await external_fs_default().promises.opendir(targetDir);
let dir = await external_fs_default().promises.opendir(targetDir);
for await (const dirent of dir) {
if (dirent.isDirectory()) {
let dirName = external_path_default().join(dir.path, dirent.name);
@ -64605,10 +64606,10 @@ async function cleanTargetDir(targetDir, packages) {
let isNestedTarget = (await exists(external_path_default().join(dirName, "CACHEDIR.TAG"))) || (await exists(external_path_default().join(dirName, ".rustc_info.json")));
try {
if (isNestedTarget) {
await cleanTargetDir(dirName, packages);
await cleanTargetDir(dirName, packages, checkTimestamp);
}
else {
await cleanProfileTarget(dirName, packages);
await cleanProfileTarget(dirName, packages, checkTimestamp);
}
}
catch { }
@ -64617,21 +64618,15 @@ async function cleanTargetDir(targetDir, packages) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function cleanProfileTarget(profileDir, packages) {
await rmRF(external_path_default().join(profileDir, "examples"));
await rmRF(external_path_default().join(profileDir, "incremental"));
let dir;
// remove all *files* from the profile directory
dir = await external_fs_default().promises.opendir(profileDir);
for await (const dirent of dir) {
if (dirent.isFile()) {
await rm(dir.path, dirent);
}
}
async function cleanProfileTarget(profileDir, packages, checkTimestamp = false) {
lib_core.debug(`cleaning profile directory "${profileDir}"`);
let keepProfile = new Set(["build", ".fingerprint", "deps"]);
await rmExcept(profileDir, keepProfile);
const keepPkg = new Set(packages.map((p) => p.name));
await rmExcept(external_path_default().join(profileDir, "build"), keepPkg);
await rmExcept(external_path_default().join(profileDir, ".fingerprint"), keepPkg);
await rmExcept(external_path_default().join(profileDir, "build"), keepPkg, checkTimestamp);
await rmExcept(external_path_default().join(profileDir, ".fingerprint"), keepPkg, checkTimestamp);
const keepDeps = new Set(packages.flatMap((p) => {
const names = [];
for (const n of [p.name, ...p.targets]) {
@ -64640,7 +64635,7 @@ async function cleanProfileTarget(profileDir, packages) {
}
return names;
}));
await rmExcept(external_path_default().join(profileDir, "deps"), keepDeps);
await rmExcept(external_path_default().join(profileDir, "deps"), keepDeps, checkTimestamp);
}
async function getCargoBins() {
const bins = new Set();
@ -64667,6 +64662,7 @@ async function cleanBin() {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function cleanRegistry(packages) {
// `.cargo/registry/src`
@ -64683,9 +64679,11 @@ async function cleanRegistry(packages) {
if (await exists(path.join(dir.path, ".git"))) {
await rmRF(path.join(dir.path, ".cache"));
}
await dir.close();
// TODO: else, clean `.cache` based on the `packages`
}
}
await indexDir.close();
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
// `.cargo/registry/cache`
const cacheDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "cache"));
@ -64700,8 +64698,10 @@ async function cleanRegistry(packages) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
}
await cacheDir.close();
}
async function cleanGit(packages) {
const coPath = path.join(CARGO_HOME, "git", "checkouts");
@ -64722,49 +64722,71 @@ async function cleanGit(packages) {
}
// we have to keep both the clone, and the checkout, removing either will
// trigger a rebuild
let dir;
// clean the db
dir = await fs.promises.opendir(dbPath);
for await (const dirent of dir) {
if (!repos.has(dirent.name)) {
await rm(dir.path, dirent);
}
}
// clean the checkouts
dir = await fs.promises.opendir(coPath);
for await (const dirent of dir) {
const refs = repos.get(dirent.name);
if (!refs) {
await rm(dir.path, dirent);
continue;
}
if (!dirent.isDirectory()) {
continue;
}
const refsDir = await fs.promises.opendir(path.join(dir.path, dirent.name));
for await (const dirent of refsDir) {
if (!refs.has(dirent.name)) {
await rm(refsDir.path, dirent);
try {
let dir = await fs.promises.opendir(dbPath);
for await (const dirent of dir) {
if (!repos.has(dirent.name)) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
catch { }
// clean the checkouts
try {
let dir = await fs.promises.opendir(coPath);
for await (const dirent of dir) {
const refs = repos.get(dirent.name);
if (!refs) {
await rm(dir.path, dirent);
continue;
}
if (!dirent.isDirectory()) {
continue;
}
const refsDir = await fs.promises.opendir(path.join(dir.path, dirent.name));
for await (const dirent of refsDir) {
if (!refs.has(dirent.name)) {
await rm(refsDir.path, dirent);
}
}
await refsDir.close();
}
await dir.close();
}
catch { }
}
const ONE_WEEK = 7 * 24 * 3600 * 1000;
async function rmExcept(dirName, keepPrefix) {
/**
* Removes all files or directories in `dirName`, except the ones matching
* any string in the `keepPrefix` set.
*
* The matching strips and trailing `-$hash` suffix.
*
* When the `checkTimestamp` flag is set, this will also remove anything older
* than one week.
*/
async function rmExcept(dirName, keepPrefix, checkTimestamp = false) {
const dir = await external_fs_default().promises.opendir(dirName);
for await (const dirent of dir) {
let name = dirent.name;
// strip the trailing hash
const idx = name.lastIndexOf("-");
if (idx !== -1) {
name = name.slice(0, idx);
}
const fileName = external_path_default().join(dir.path, dirent.name);
const { mtime } = await external_fs_default().promises.stat(fileName);
// we don’t really know
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > ONE_WEEK) {
let isOutdated = false;
if (checkTimestamp) {
const fileName = external_path_default().join(dir.path, dirent.name);
const { mtime } = await external_fs_default().promises.stat(fileName);
isOutdated = Date.now() - mtime.getTime() > ONE_WEEK;
}
if (!keepPrefix.has(name) || isOutdated) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function rm(parent, dirent) {
try {
@ -64774,13 +64796,13 @@ async function rm(parent, dirent) {
await external_fs_default().promises.unlink(fileName);
}
else if (dirent.isDirectory()) {
await io.rmRF(fileName);
await lib_io.rmRF(fileName);
}
}
catch { }
}
async function rmRF(dirName) {
lib_core.debug(`deleting "${dirName}"`);
core.debug(`deleting "${dirName}"`);
await io.rmRF(dirName);
}
async function exists(path) {
@ -64832,7 +64854,7 @@ async function run() {
for (const workspace of config.workspaces) {
try {
const packages = await workspace.getPackages();
await cleanTargetDir(workspace.target, packages);
await cleanTargetDir(workspace.target, packages, true);
}
catch { }
}

116
dist/save/index.js vendored
View File

@ -64394,6 +64394,7 @@ async function getCmdOutput(cmd, args = [], options = {}) {
;// CONCATENATED MODULE: ./src/workspace.ts
const SAVE_TARGETS = new Set(["lib", "proc-macro"]);
class Workspace {
constructor(root, target) {
this.root = root;
@ -64409,7 +64410,7 @@ class Workspace {
if (pkg.manifest_path.startsWith(this.root)) {
continue;
}
const targets = pkg.targets.filter((t) => t.kind[0] === "lib").map((t) => t.name);
const targets = pkg.targets.filter((t) => t.kind.some((kind) => SAVE_TARGETS.has(kind))).map((t) => t.name);
packages.push({ name: pkg.name, version: pkg.version, targets, path: external_path_default().dirname(pkg.manifest_path) });
}
}
@ -64594,10 +64595,10 @@ async function getRustVersion() {
async function cleanTargetDir(targetDir, packages) {
let dir;
async function cleanTargetDir(targetDir, packages, checkTimestamp = false) {
core.debug(`cleaning target directory "${targetDir}"`);
// remove all *files* from the profile directory
dir = await external_fs_default().promises.opendir(targetDir);
let dir = await external_fs_default().promises.opendir(targetDir);
for await (const dirent of dir) {
if (dirent.isDirectory()) {
let dirName = external_path_default().join(dir.path, dirent.name);
@ -64605,10 +64606,10 @@ async function cleanTargetDir(targetDir, packages) {
let isNestedTarget = (await exists(external_path_default().join(dirName, "CACHEDIR.TAG"))) || (await exists(external_path_default().join(dirName, ".rustc_info.json")));
try {
if (isNestedTarget) {
await cleanTargetDir(dirName, packages);
await cleanTargetDir(dirName, packages, checkTimestamp);
}
else {
await cleanProfileTarget(dirName, packages);
await cleanProfileTarget(dirName, packages, checkTimestamp);
}
}
catch { }
@ -64617,21 +64618,15 @@ async function cleanTargetDir(targetDir, packages) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function cleanProfileTarget(profileDir, packages) {
await rmRF(external_path_default().join(profileDir, "examples"));
await rmRF(external_path_default().join(profileDir, "incremental"));
let dir;
// remove all *files* from the profile directory
dir = await external_fs_default().promises.opendir(profileDir);
for await (const dirent of dir) {
if (dirent.isFile()) {
await rm(dir.path, dirent);
}
}
async function cleanProfileTarget(profileDir, packages, checkTimestamp = false) {
core.debug(`cleaning profile directory "${profileDir}"`);
let keepProfile = new Set(["build", ".fingerprint", "deps"]);
await rmExcept(profileDir, keepProfile);
const keepPkg = new Set(packages.map((p) => p.name));
await rmExcept(external_path_default().join(profileDir, "build"), keepPkg);
await rmExcept(external_path_default().join(profileDir, ".fingerprint"), keepPkg);
await rmExcept(external_path_default().join(profileDir, "build"), keepPkg, checkTimestamp);
await rmExcept(external_path_default().join(profileDir, ".fingerprint"), keepPkg, checkTimestamp);
const keepDeps = new Set(packages.flatMap((p) => {
const names = [];
for (const n of [p.name, ...p.targets]) {
@ -64640,7 +64635,7 @@ async function cleanProfileTarget(profileDir, packages) {
}
return names;
}));
await rmExcept(external_path_default().join(profileDir, "deps"), keepDeps);
await rmExcept(external_path_default().join(profileDir, "deps"), keepDeps, checkTimestamp);
}
async function getCargoBins() {
const bins = new Set();
@ -64667,6 +64662,7 @@ async function cleanBin() {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function cleanRegistry(packages) {
// `.cargo/registry/src`
@ -64683,9 +64679,11 @@ async function cleanRegistry(packages) {
if (await exists(external_path_default().join(dir.path, ".git"))) {
await rmRF(external_path_default().join(dir.path, ".cache"));
}
await dir.close();
// TODO: else, clean `.cache` based on the `packages`
}
}
await indexDir.close();
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
// `.cargo/registry/cache`
const cacheDir = await external_fs_default().promises.opendir(external_path_default().join(CARGO_HOME, "registry", "cache"));
@ -64700,8 +64698,10 @@ async function cleanRegistry(packages) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
}
await cacheDir.close();
}
async function cleanGit(packages) {
const coPath = external_path_default().join(CARGO_HOME, "git", "checkouts");
@ -64722,49 +64722,71 @@ async function cleanGit(packages) {
}
// we have to keep both the clone, and the checkout, removing either will
// trigger a rebuild
let dir;
// clean the db
dir = await external_fs_default().promises.opendir(dbPath);
for await (const dirent of dir) {
if (!repos.has(dirent.name)) {
await rm(dir.path, dirent);
}
}
// clean the checkouts
dir = await external_fs_default().promises.opendir(coPath);
for await (const dirent of dir) {
const refs = repos.get(dirent.name);
if (!refs) {
await rm(dir.path, dirent);
continue;
}
if (!dirent.isDirectory()) {
continue;
}
const refsDir = await external_fs_default().promises.opendir(external_path_default().join(dir.path, dirent.name));
for await (const dirent of refsDir) {
if (!refs.has(dirent.name)) {
await rm(refsDir.path, dirent);
try {
let dir = await external_fs_default().promises.opendir(dbPath);
for await (const dirent of dir) {
if (!repos.has(dirent.name)) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
catch { }
// clean the checkouts
try {
let dir = await external_fs_default().promises.opendir(coPath);
for await (const dirent of dir) {
const refs = repos.get(dirent.name);
if (!refs) {
await rm(dir.path, dirent);
continue;
}
if (!dirent.isDirectory()) {
continue;
}
const refsDir = await external_fs_default().promises.opendir(external_path_default().join(dir.path, dirent.name));
for await (const dirent of refsDir) {
if (!refs.has(dirent.name)) {
await rm(refsDir.path, dirent);
}
}
await refsDir.close();
}
await dir.close();
}
catch { }
}
const ONE_WEEK = 7 * 24 * 3600 * 1000;
async function rmExcept(dirName, keepPrefix) {
/**
* Removes all files or directories in `dirName`, except the ones matching
* any string in the `keepPrefix` set.
*
* The matching strips and trailing `-$hash` suffix.
*
* When the `checkTimestamp` flag is set, this will also remove anything older
* than one week.
*/
async function rmExcept(dirName, keepPrefix, checkTimestamp = false) {
const dir = await external_fs_default().promises.opendir(dirName);
for await (const dirent of dir) {
let name = dirent.name;
// strip the trailing hash
const idx = name.lastIndexOf("-");
if (idx !== -1) {
name = name.slice(0, idx);
}
const fileName = external_path_default().join(dir.path, dirent.name);
const { mtime } = await external_fs_default().promises.stat(fileName);
// we don’t really know
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > ONE_WEEK) {
let isOutdated = false;
if (checkTimestamp) {
const fileName = external_path_default().join(dir.path, dirent.name);
const { mtime } = await external_fs_default().promises.stat(fileName);
isOutdated = Date.now() - mtime.getTime() > ONE_WEEK;
}
if (!keepPrefix.has(name) || isOutdated) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function rm(parent, dirent) {
try {

119
src/cleanup.ts
View File

@ -6,10 +6,11 @@ import path from "path";
import { CARGO_HOME, STATE_BINS } from "./config";
import { Packages } from "./workspace";
export async function cleanTargetDir(targetDir: string, packages: Packages) {
let dir: fs.Dir;
export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp = false) {
core.debug(`cleaning target directory "${targetDir}"`);
// remove all *files* from the profile directory
dir = await fs.promises.opendir(targetDir);
let dir = await fs.promises.opendir(targetDir);
for await (const dirent of dir) {
if (dirent.isDirectory()) {
let dirName = path.join(dir.path, dirent.name);
@ -19,33 +20,27 @@ export async function cleanTargetDir(targetDir: string, packages: Packages) {
try {
if (isNestedTarget) {
await cleanTargetDir(dirName, packages);
await cleanTargetDir(dirName, packages, checkTimestamp);
} else {
await cleanProfileTarget(dirName, packages);
await cleanProfileTarget(dirName, packages, checkTimestamp);
}
} catch {}
} else if (dirent.name !== "CACHEDIR.TAG") {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function cleanProfileTarget(profileDir: string, packages: Packages) {
await rmRF(path.join(profileDir, "examples"));
await rmRF(path.join(profileDir, "incremental"));
async function cleanProfileTarget(profileDir: string, packages: Packages, checkTimestamp = false) {
core.debug(`cleaning profile directory "${profileDir}"`);
let dir: fs.Dir;
// remove all *files* from the profile directory
dir = await fs.promises.opendir(profileDir);
for await (const dirent of dir) {
if (dirent.isFile()) {
await rm(dir.path, dirent);
}
}
let keepProfile = new Set(["build", ".fingerprint", "deps"]);
await rmExcept(profileDir, keepProfile);
const keepPkg = new Set(packages.map((p) => p.name));
await rmExcept(path.join(profileDir, "build"), keepPkg);
await rmExcept(path.join(profileDir, ".fingerprint"), keepPkg);
await rmExcept(path.join(profileDir, "build"), keepPkg, checkTimestamp);
await rmExcept(path.join(profileDir, ".fingerprint"), keepPkg, checkTimestamp);
const keepDeps = new Set(
packages.flatMap((p) => {
@ -57,7 +52,7 @@ async function cleanProfileTarget(profileDir: string, packages: Packages) {
return names;
}),
);
await rmExcept(path.join(profileDir, "deps"), keepDeps);
await rmExcept(path.join(profileDir, "deps"), keepDeps, checkTimestamp);
}
export async function getCargoBins(): Promise<Set<string>> {
@ -89,6 +84,7 @@ export async function cleanBin() {
await rm(dir.path, dirent);
}
}
await dir.close();
}
export async function cleanRegistry(packages: Packages) {
@ -108,9 +104,11 @@ export async function cleanRegistry(packages: Packages) {
if (await exists(path.join(dir.path, ".git"))) {
await rmRF(path.join(dir.path, ".cache"));
}
await dir.close();
// TODO: else, clean `.cache` based on the `packages`
}
}
await indexDir.close();
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
@ -127,8 +125,10 @@ export async function cleanRegistry(packages: Packages) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
}
await cacheDir.close();
}
export async function cleanGit(packages: Packages) {
@ -151,52 +151,75 @@ export async function cleanGit(packages: Packages) {
// we have to keep both the clone, and the checkout, removing either will
// trigger a rebuild
let dir: fs.Dir;
// clean the db
dir = await fs.promises.opendir(dbPath);
for await (const dirent of dir) {
if (!repos.has(dirent.name)) {
await rm(dir.path, dirent);
}
}
// clean the checkouts
dir = await fs.promises.opendir(coPath);
for await (const dirent of dir) {
const refs = repos.get(dirent.name);
if (!refs) {
await rm(dir.path, dirent);
continue;
}
if (!dirent.isDirectory()) {
continue;
}
const refsDir = await fs.promises.opendir(path.join(dir.path, dirent.name));
for await (const dirent of refsDir) {
if (!refs.has(dirent.name)) {
await rm(refsDir.path, dirent);
try {
let dir = await fs.promises.opendir(dbPath);
for await (const dirent of dir) {
if (!repos.has(dirent.name)) {
await rm(dir.path, dirent);
}
}
}
await dir.close();
} catch {}
// clean the checkouts
try {
let dir = await fs.promises.opendir(coPath);
for await (const dirent of dir) {
const refs = repos.get(dirent.name);
if (!refs) {
await rm(dir.path, dirent);
continue;
}
if (!dirent.isDirectory()) {
continue;
}
const refsDir = await fs.promises.opendir(path.join(dir.path, dirent.name));
for await (const dirent of refsDir) {
if (!refs.has(dirent.name)) {
await rm(refsDir.path, dirent);
}
}
await refsDir.close();
}
await dir.close();
} catch {}
}
const ONE_WEEK = 7 * 24 * 3600 * 1000;
async function rmExcept(dirName: string, keepPrefix: Set<string>) {
/**
* Removes all files or directories in `dirName`, except the ones matching
* any string in the `keepPrefix` set.
*
* The matching strips and trailing `-$hash` suffix.
*
* When the `checkTimestamp` flag is set, this will also remove anything older
* than one week.
*/
async function rmExcept(dirName: string, keepPrefix: Set<string>, checkTimestamp = false) {
const dir = await fs.promises.opendir(dirName);
for await (const dirent of dir) {
let name = dirent.name;
// strip the trailing hash
const idx = name.lastIndexOf("-");
if (idx !== -1) {
name = name.slice(0, idx);
}
const fileName = path.join(dir.path, dirent.name);
const { mtime } = await fs.promises.stat(fileName);
// we don’t really know
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > ONE_WEEK) {
let isOutdated = false;
if (checkTimestamp) {
const fileName = path.join(dir.path, dirent.name);
const { mtime } = await fs.promises.stat(fileName);
isOutdated = Date.now() - mtime.getTime() > ONE_WEEK;
}
if (!keepPrefix.has(name) || isOutdated) {
await rm(dir.path, dirent);
}
}
await dir.close();
}
async function rm(parent: string, dirent: fs.Dirent) {

2
src/restore.ts
View File

@ -45,7 +45,7 @@ async function run() {
try {
const packages = await workspace.getPackages();
await cleanTargetDir(workspace.target, packages);
await cleanTargetDir(workspace.target, packages, true);
} catch {}
}
}

4
src/workspace.ts
View File

@ -2,6 +2,8 @@ import path from "path";
import { getCmdOutput } from "./utils";
const SAVE_TARGETS = new Set(["lib", "proc-macro"]);
export class Workspace {
constructor(public root: string, public target: string) {}
@ -17,7 +19,7 @@ export class Workspace {
if (pkg.manifest_path.startsWith(this.root)) {
continue;
}
const targets = pkg.targets.filter((t) => t.kind[0] === "lib").map((t) => t.name);
const targets = pkg.targets.filter((t) => t.kind.some((kind) => SAVE_TARGETS.has(kind))).map((t) => t.name);
packages.push({ name: pkg.name, version: pkg.version, targets, path: path.dirname(pkg.manifest_path) });
}
} catch {}