changes to multipart download for s3

packages/warp-cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "github-actions.warp-cache",
-  "version": "1.1.15",
+  "version": "1.1.16",
   "preview": true,
   "description": "Github action to use WarpBuild's in-house cache offering",
   "keywords": [

packages/warp-cache/src/cache.ts

@@ -137,48 +137,28 @@ export async function restoreCache(
         }
 
         try {
-          let readStream: NodeJS.ReadableStream | undefined
-          let downloadCommandPipe = getDownloadCommandPipeForWget(
-            cacheEntry?.s3?.pre_signed_url
-          )
-          await extractStreamingTar(
-            readStream,
-            archivePath,
-            compressionMethod,
-            downloadCommandPipe
+          await cacheHttpClient.downloadCache(
+            cacheEntry.provider,
+            cacheEntry.s3?.pre_signed_url,
+            archivePath
           )
         } catch (error) {
-          core.debug(`Failed to download cache: ${error}`)
-          core.info(
-            `Streaming download failed. Likely a cloud provider issue. Retrying with multipart download`
-          )
-          // Wait 1 second
-          await new Promise(resolve => setTimeout(resolve, 1000))
-
-          try {
-            await cacheHttpClient.downloadCache(
-              cacheEntry.provider,
-              cacheEntry.s3?.pre_signed_url,
-              archivePath
-            )
-          } catch (error) {
-            core.info('Cache Miss. Failed to download cache.')
-            return undefined
-          }
+          core.info('Cache Miss. Failed to download cache.')
+          return undefined
+        }
 
-          if (core.isDebug()) {
-            await listTar(archivePath, compressionMethod)
-          }
+        if (core.isDebug()) {
+          await listTar(archivePath, compressionMethod)
+        }
 
-          const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
-          core.info(
-            `Cache Size: ~${Math.round(
-              archiveFileSize / (1024 * 1024)
-            )} MB (${archiveFileSize} B)`
-          )
+        const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
+        core.info(
+          `Cache Size: ~${Math.round(
+            archiveFileSize / (1024 * 1024)
+          )} MB (${archiveFileSize} B)`
+        )
 
-          await extractTar(archivePath, compressionMethod)
-        }
+        await extractTar(archivePath, compressionMethod)
 
         core.info('Cache restored successfully')
         break

packages/warp-cache/src/internal/cacheHttpClient.ts

@@ -9,6 +9,7 @@ import * as crypto from 'crypto'
 
 import * as utils from './cacheUtils'
 import {CompressionMethod} from './constants'
+import os from 'os'
 import {
   InternalCacheOptions,
   ITypedResponseWithError,
@@ -211,7 +212,14 @@ export async function downloadCache(
 ): Promise<void> {
   switch (provider) {
     case 's3':
-      await downloadCacheMultiConnection(archiveLocation, archivePath, 8)
+      {
+        const numberOfConnections = 2 + os.cpus().length
+        await downloadCacheMultiConnection(
+          archiveLocation,
+          archivePath,
+          Math.min(numberOfConnections, 30)
+        )
+      }
       break
     case 'gcs': {
       if (!gcsToken) {