refactor(tools): extract zip utilities into shared @aws-cdk/tools package

.projenrc.ts

@@ -16,6 +16,7 @@ import { PrLabeler } from './projenrc/pr-labeler';
 import { RecordPublishingTimestamp } from './projenrc/record-publishing-timestamp';
 import { DocType, S3DocsPublishing } from './projenrc/s3-docs-publishing';
 import { SelfMutationOnForks } from './projenrc/SelfMutationOnForks';
+import { CdkTypeScriptWorkspace, ToolsWorkspace } from './projenrc/tools';
 import { TypecheckTests } from './projenrc/TypecheckTests';
 
 // #region shared config
@@ -690,22 +691,49 @@ const cliPluginContract = configureProject(
   }),
 );
 
+// #endregion
+//////////////////////////////////////////////////////////////////////
+// #region @aws-cdk/tools
+
+/**
+ * Private (unpublished) package hosting small, self-contained utilities.
+ * Consumers cherry-pick tools via `useTools: ['<name>']` on a
+ * `CdkTypeScriptWorkspace`.
+ *
+ * Adding a new tool: create `packages/@aws-cdk/tools/lib/<tool>/index.ts`.
+ * It is auto-discovered and bundled individually.
+ */
+configureProject(
+  new ToolsWorkspace({
+    ...genericCdkProps({ private: true }),
+    parent: repo,
+    // Runtime deps that `esbuild` bundles into any tool that imports them.
+    deps: ['archiver', 'fast-glob'],
+    devDeps: ['@types/archiver', 'jszip'],
+    tsconfig: {
+      compilerOptions: {
+        ...defaultTsOptions,
+      },
+    },
+  }),
+);
+
 // #endregion
 //////////////////////////////////////////////////////////////////////
 // #region @aws-cdk/cdk-assets-lib
 
 const cdkAssetsLib = configureProject(
-  new yarn.TypeScriptWorkspace({
+  new CdkTypeScriptWorkspace({
     ...genericCdkProps(),
     parent: repo,
     name: '@aws-cdk/cdk-assets-lib',
     majorVersion: 1,
     description: 'CDK Asset Publishing Library',
     srcdir: 'lib',
+    useTools: ['zip'],
     deps: [
       cloudAssemblySchema.customizeReference({ versionType: 'any-future' }),
       cloudAssemblyApi.customizeReference({ versionType: 'exact' }),
-      'archiver',
       'fast-glob',
       'mime@^2',
       sdkDep('@aws-sdk/client-ecr'),
@@ -719,7 +747,6 @@ const cdkAssetsLib = configureProject(
       'picomatch',
     ],
     devDeps: [
-      '@types/archiver',
       '@types/mime@^2',
       '@types/picomatch',
       'fs-extra@^11',
@@ -873,14 +900,15 @@ const toolkitLibTsCompilerOptions = {
 };
 
 const toolkitLib = configureProject(
-  new yarn.TypeScriptWorkspace({
+  new CdkTypeScriptWorkspace({
     ...genericCdkProps(),
     homepage: 'https://docs.aws.amazon.com/cdk/api/toolkit-lib',
     parent: repo,
     name: '@aws-cdk/toolkit-lib',
     description: 'AWS CDK Programmatic Toolkit Library',
     majorVersion: 1,
     srcdir: 'lib',
+    useTools: ['zip'],
     peerDeps: [
       cliPluginContract.customizeReference({ versionType: 'any-minor' }), // allow consumers to easily de-depulicate this
     ],
@@ -917,7 +945,6 @@ const toolkitLib = configureProject(
       smithyDep('@smithy/shared-ini-file-loader'),
       smithyDep('@smithy/util-retry'),
       smithyDep('@smithy/util-waiter'),
-      'archiver',
       'cdk-from-cfn',
       'chalk@^4',
       'chokidar@^4',

packages/@aws-cdk/cdk-assets-lib/lib/private/archive.ts

@@ -1,101 +1,4 @@
-import { createWriteStream, promises as fs } from 'fs';
-import * as path from 'path';
-import type { Options } from 'fast-glob';
-import { globSync } from 'fast-glob';
-
-// namespace object imports won't work in the bundle for function exports
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const archiver = require('archiver');
-
-type EventEmitter = (x: string) => void;
-
-export async function zipDirectory(
-  directory: string,
-  outputFile: string,
-  eventEmitter: EventEmitter,
-): Promise<void> {
-  // We write to a temporary file and rename at the last moment. This is so that if we are
-  // interrupted during this process, we don't leave a half-finished file in the target location.
-  const temporaryOutputFile = `${outputFile}.${randomString()}._tmp`;
-  await writeZipFile(directory, temporaryOutputFile);
-  await moveIntoPlace(temporaryOutputFile, outputFile, eventEmitter);
-}
-
-function writeZipFile(directory: string, outputFile: string): Promise<void> {
-  return new Promise(async (ok, fail) => {
-    // The below options are needed to support following symlinks when building zip files:
-    // - onlyFiles: This will prevent symlinks themselves from being copied into the zip.
-    // - followSymbolicLinks: This will follow symlinks and copy the files within.
-    const globOptions: Options = {
-      dot: true,
-      onlyFiles: true,
-      followSymbolicLinks: true,
-      cwd: directory,
-    };
-    const files = globSync('**', globOptions); // The output here is already sorted
-
-    const output = createWriteStream(outputFile);
-
-    const archive = archiver('zip');
-    archive.on('warning', fail);
-    archive.on('error', fail);
-
-    // archive has been finalized and the output file descriptor has closed, resolve promise
-    // this has to be done before calling `finalize` since the events may fire immediately after.
-    // see https://www.npmjs.com/package/archiver
-    output.once('close', ok);
-
-    archive.pipe(output);
-
-    // Append files serially to ensure file order
-    for (const file of files) {
-      const fullPath = path.resolve(directory, file);
-      // There are exactly 2 promises
-      // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism
-      const [data, stat] = await Promise.all([fs.readFile(fullPath), fs.stat(fullPath)]);
-      archive.append(data, {
-        name: file,
-        date: new Date('1980-01-01T00:00:00.000Z'), // reset dates to get the same hash for the same content
-        mode: stat.mode,
-      });
-    }
-
-    await archive.finalize();
-  });
-}
-
-/**
- * Rename the file to the target location, taking into account:
- *
- * - That we may see EPERM on Windows while an Antivirus scanner still has the
- *   file open, so retry a couple of times.
- * - This same function may be called in parallel and be interrupted at any point.
- */
-async function moveIntoPlace(source: string, target: string, eventEmitter: EventEmitter) {
-  let delay = 100;
-  let attempts = 5;
-  while (true) {
-    try {
-      // 'rename' is guaranteed to overwrite an existing target, as long as it is a file (not a directory)
-      await fs.rename(source, target);
-      return;
-    } catch (e: any) {
-      if (e.code !== 'EPERM' || attempts-- <= 0) {
-        throw e;
-      }
-      eventEmitter(e.message);
-      await sleep(Math.floor(Math.random() * delay));
-      delay *= 2;
-    }
-  }
-}
-
-function sleep(ms: number) {
-  return new Promise((ok) => setTimeout(ok, ms));
-}
-
-function randomString() {
-  return Math.random()
-    .toString(36)
-    .replace(/[^a-z0-9]+/g, '');
-}
+// Re-export the shared, bundled `zip` tool from @aws-cdk/tools.
+// The bundled files are copied into `./tools/zip/` by a pre-compile task
+// (via useTools); see projenrc/tools.ts.
+export { zipDirectory } from './tools/zip';

packages/@aws-cdk/cdk-assets-lib/lib/private/handlers/files.ts

@@ -8,11 +8,11 @@ import type { FileManifestEntry } from '../../asset-manifest';
 import type { IS3Client } from '../../aws';
 import type { PutObjectCommandInput } from '../../aws-types';
 import { EventType, shellEventPublisherFromEventEmitter } from '../../progress';
-import { zipDirectory } from '../archive';
 import type { IAssetHandler, IHandlerHost, PublishOptions } from '../asset-handler';
 import { pathExists } from '../fs-extra';
 import { replaceAwsPlaceholders } from '../placeholders';
 import { shell } from '../shell';
+import { zipDirectory } from '../tools/zip';
 
 /**
  * The size of an empty zip file is 22 bytes