Deprecate request/requestretry – Switch to Axios/axios-retry, Add Central getStream Helper

lib/fetch.js

@@ -2,24 +2,33 @@
 // SPDX-License-Identifier: MIT
 
 const axios = require('axios')
+const { default: axiosRetry } = require('axios-retry')
 
 const defaultHeaders = Object.freeze({ 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' })
+const defaultMaximumAttempts = 3
 
 axios.defaults.headers.common['User-Agent'] = defaultHeaders['User-Agent']
 
+// Create a shared axios instance with retry configured for the default case
+const defaultRetryInstance = axios.create()
+axiosRetry(defaultRetryInstance, {
+  retries: defaultMaximumAttempts - 1,
+  retryDelay: (retryCount, error) => {
+    return axiosRetry.exponentialDelay(retryCount, error, 100)
+  }
+})
+
 function buildRequestOptions(request) {
   let responseType = 'text'
   if (request.json) {
     responseType = 'json'
   } else if (request.encoding === null) {
     responseType = 'stream'
   }
-
   const validateOptions = {}
   if (request.simple === false) {
     validateOptions.validateStatus = () => true
   }
-
   return {
     method: request.method,
     url: request.url || request.uri,
@@ -62,4 +71,52 @@ async function getStream(opt) {
   return await callFetch(request)
 }
 
-module.exports = { callFetch, withDefaults, defaultHeaders, getStream }
+async function callFetchWithRetry(url, options = {}, retryOptions = {}) {
+  const { maxAttempts = defaultMaximumAttempts, ...otherRetryOpts } = retryOptions
+
+  // Use shared instance if using default settings, otherwise create a new one
+  let axiosInstance = defaultRetryInstance
+  const hasCustomRetryOptions = maxAttempts !== defaultMaximumAttempts || Object.keys(otherRetryOpts).length > 0
+
+  if (hasCustomRetryOptions) {
+    axiosInstance = axios.create()
+    axiosRetry(axiosInstance, {
+      retries: maxAttempts - 1,
+      retryDelay: (retryCount, error) => {
+        return axiosRetry.exponentialDelay(retryCount, error, 100)
+      },
+      ...otherRetryOpts
+    })
+  }
+
+  const resolveWithFullResponse = options.resolveWithFullResponse ?? true
+  const request = { url, ...options, resolveWithFullResponse }
+  try {
+    const response = await callFetch(request, axiosInstance)
+    if (!resolveWithFullResponse) return response
+    return {
+      statusCode: response.status,
+      headers: response.headers,
+      body: response.data,
+      request: response.request
+    }
+  } catch (err) {
+    if (err.response && resolveWithFullResponse) {
+      return {
+        statusCode: err.response.status,
+        headers: err.response.headers,
+        body: err.response.data,
+        request: err.response.request
+      }
+    }
+    throw err
+  }
+}
+
+module.exports = {
+  callFetch,
+  callFetchWithRetry,
+  withDefaults,
+  defaultHeaders,
+  getStream
+}

lib/sourceDiscovery.js

@@ -3,7 +3,6 @@
 
 const { get, uniq, uniqWith } = require('lodash')
 const ghrequestor = require('ghrequestor')
-const request = require('requestretry')
 const geit = require('geit')
 // TODO why not parse-github-repo-url (10x more downloads)
 const parseGitHubUrl = require('parse-github-url')
@@ -75,7 +74,8 @@ async function discoverFromGitHubRefs(version, candidate, options) {
       headers,
       maxAttempts: 3,
       retryDelay: 250,
-      retryStrategy: request.RetryStrategies.HTTPOrNetworkError,
+      // TODO This is the default strategy in axios retry. We ghrequestor currently depends on requestretry, but we will replace it with axios retry soon.
+      // retryStrategy: request.RetryStrategies.HTTPOrNetworkError,
       tokenLowerBound: 10,
       json: true
     })

package.json

@@ -73,7 +73,6 @@
     "parse-github-url": "^1.0.3",
     "patch-package": "^8.0.1",
     "qlimit": "^0.1.1",
-    "requestretry": "^8.0.0",
     "semver": "^7.7.4",
     "sha.js": "^2.4.12",
     "spdx-correct": "^3.2.0",

providers/fetch/nugetFetch.js

@@ -7,7 +7,7 @@ const fs = require('fs')
 const { mkdirp } = require('mkdirp')
 const path = require('path')
 const { promisify } = require('util')
-const requestRetry = require('requestretry').defaults({ maxAttempts: 3, fullResponse: true })
+const { callFetchWithRetry: requestRetry } = require('../../lib/fetch')
 const FetchResult = require('../../lib/fetchResult')
 
 const providerMap = {
@@ -69,9 +69,8 @@ class NuGetFetch extends AbstractFetch {
     // https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource
     // Example: https://api.nuget.org/v3/registration5/moq/4.8.2.json and follow catalogEntry
     // https://api.nuget.org/v3/registration5-gz-semver2/microsoft.powershell.native/7.0.0-preview.1.json
-    const { body, statusCode } = await requestRetry.get(
-      `${baseUrl}/v3/registration5-gz-semver2/${spec.name.toLowerCase()}/${spec.revision}.json`,
-      { gzip: true }
+    const { body, statusCode } = await requestRetry(
+      `${baseUrl}/v3/registration5-gz-semver2/${spec.name.toLowerCase()}/${spec.revision}.json`
     )
     return statusCode !== 200 || !body ? null : JSON.parse(body)
   }
@@ -87,7 +86,7 @@ class NuGetFetch extends AbstractFetch {
     // https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource
     // Example: https://api.nuget.org/v3-flatcontainer/moq/index.json
     const baseUrl = providerMap.nuget
-    const { body, statusCode } = await requestRetry.get(`${baseUrl}/v3-flatcontainer/${name}/index.json`, {
+    const { body, statusCode } = await requestRetry(`${baseUrl}/v3-flatcontainer/${name}/index.json`, {
       json: true
     })
     // If statusCode is not 200, XML may be returned
@@ -99,9 +98,9 @@ class NuGetFetch extends AbstractFetch {
   }
 
   async _getPackage(zip, packageContentUrl) {
+    const response = await requestRetry(packageContentUrl, { json: false, encoding: null })
     return new Promise((resolve, reject) => {
-      requestRetry
-        .get(packageContentUrl, { json: false, encoding: null })
+      response.body
         .pipe(fs.createWriteStream(zip))
         .on('finish', () => resolve(null))
         .on('error', reject)
@@ -110,7 +109,7 @@ class NuGetFetch extends AbstractFetch {
 
   async _getManifest(catalogEntryUrl) {
     // Example: https://api.nuget.org/v3/catalog0/data/2018.10.29.04.23.22/xunit.core.2.4.1.json
-    const { body, statusCode } = await requestRetry.get(catalogEntryUrl)
+    const { body, statusCode } = await requestRetry(catalogEntryUrl)
     if (statusCode !== 200) return null
     return JSON.parse(body)
   }
@@ -119,7 +118,7 @@ class NuGetFetch extends AbstractFetch {
   async _getNuspec(spec) {
     // https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec
     // Example: https://api.nuget.org/v3-flatcontainer/newtonsoft.json/11.0.1/newtonsoft.json.nuspec
-    const { body, statusCode } = await requestRetry.get(
+    const { body, statusCode } = await requestRetry(
       `https://api.nuget.org/v3-flatcontainer/${spec.name.toLowerCase()}/${spec.revision}/${spec.name.toLowerCase()}.nuspec`
     )
     if (statusCode !== 200) return null
@@ -146,7 +145,7 @@ class NuGetFetch extends AbstractFetch {
     if (licenseUrl.toLowerCase().includes('license_url_here_or_delete_this_line')) return
     const downloadedLicenseDirName = path.join(dirName, 'clearlydefined', 'downloaded')
     await mkdirp(downloadedLicenseDirName)
-    const { body, statusCode } = await requestRetry.get(licenseUrl)
+    const { body, statusCode } = await requestRetry(licenseUrl)
     if (statusCode !== 200) return
     await promisify(fs.writeFile)(path.join(downloadedLicenseDirName, 'LICENSE'), body)
   }

providers/fetch/packagistFetch.js

@@ -2,10 +2,9 @@
 // SPDX-License-Identifier: MIT
 
 const AbstractFetch = require('./abstractFetch')
-const requestRetry = require('requestretry').defaults({ maxAttempts: 3, fullResponse: true })
 const fs = require('fs')
 const { get } = require('lodash')
-const { getStream } = require('../../lib/fetch')
+const { getStream, callFetchWithRetry: requestRetry } = require('../../lib/fetch')
 const { promisify } = require('util')
 const readdir = promisify(fs.readdir)
 const FetchResult = require('../../lib/fetchResult')
@@ -40,7 +39,7 @@ class PackagistFetch extends AbstractFetch {
 
   async _getRegistryData(spec) {
     const baseUrl = providerMap.packagist
-    const { body, statusCode } = await requestRetry.get(`${baseUrl}p2/${spec.namespace}/${spec.name}.json`, {
+    const { body, statusCode } = await requestRetry(`${baseUrl}p2/${spec.namespace}/${spec.name}.json`, {
       json: true
     })
     if (statusCode !== 200 || !body) return null

providers/fetch/podFetch.js

@@ -3,12 +3,11 @@
 
 const { clone, get } = require('lodash')
 const AbstractFetch = require('./abstractFetch')
-const { callFetch: request } = require('../../lib/fetch')
+const { callFetch: request, callFetchWithRetry: requestRetry } = require('../../lib/fetch')
 const fs = require('fs')
 const path = require('path')
 const crypto = require('crypto')
 const { exec } = require('child_process')
-const requestRetry = require('requestretry').defaults({ maxAttempts: 3, fullResponse: true })
 const FetchResult = require('../../lib/fetchResult')
 
 const services = {
@@ -131,7 +130,7 @@ class PodFetch extends AbstractFetch {
 
   async _getVersion(spec) {
     // Example: https://trunk.cocoapods.org/api/v1/pods/SwiftLCS
-    const { body, statusCode } = await requestRetry.get(`${services.trunk}/pods/${spec.name}`, {
+    const { body, statusCode } = await requestRetry(`${services.trunk}/pods/${spec.name}`, {
       json: true
     })
 

providers/fetch/pypiFetch.js

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: MIT
 
 const AbstractFetch = require('./abstractFetch')
-const requestRetry = require('./requestRetryWithDefaults')
 const nodeRequest = require('../../lib/fetch')
 const fs = require('fs')
 const spdxCorrect = require('spdx-correct')
@@ -46,7 +45,7 @@ class PyPiFetch extends AbstractFetch {
 
   async _getRegistryData(spec) {
     const baseUrl = providerMap.pypi
-    const { body, statusCode } = await requestRetry.get(`${baseUrl}/pypi/${spec.name}/json`, {
+    const { body, statusCode } = await nodeRequest.callFetchWithRetry(`${baseUrl}/pypi/${spec.name}/json`, {
       json: true
     })
     if (statusCode !== 200 || !body) return null

providers/fetch/rubyGemsFetch.js

@@ -3,7 +3,7 @@
 
 const AbstractFetch = require('./abstractFetch')
 const { getStream } = require('../../lib/fetch')
-const requestRetry = require('requestretry').defaults({ maxAttempts: 3, fullResponse: true })
+const requestRetry = require('../../lib/fetch').callFetchWithRetry
 const fs = require('fs')
 const zlib = require('zlib')
 const path = require('path')
@@ -48,7 +48,7 @@ class RubyGemsFetch extends AbstractFetch {
 
   async _getRegistryData(spec) {
     const baseUrl = providerMap.rubyGems
-    const { body, statusCode } = await requestRetry.get(`${baseUrl}/api/v1/gems/${spec.name}.json`, {
+    const { body, statusCode } = await requestRetry(`${baseUrl}/api/v1/gems/${spec.name}.json`, {
       json: true
     })
     return statusCode === 200 && body ? body : null

providers/process/top.js

@@ -10,7 +10,8 @@ const ghrequestor = require('ghrequestor')
 const linebyline = require('linebyline')
 const path = require('path')
 const Request = require('../../ghcrawler').request
-const requestRetry = require('requestretry').defaults({ json: true, maxAttempts: 3, fullResponse: false })
+const requestRetry = require('../../lib/fetch').callFetchWithRetry
+const defaultOptions = { json: true, resolveWithFullResponse: false }
 
 class TopProcessor extends AbstractProcessor {
   canHandle(request) {
@@ -89,7 +90,8 @@ class TopProcessor extends AbstractProcessor {
     if (!end || end - start <= 0) end = start + 1000
     const initialOffset = Math.floor(start / 36) * 36
     for (let offset = initialOffset; offset < end; offset += 36) {
-      const response = await requestRetry.get(`https://www.npmjs.com/browse/depended?offset=${offset}`, {
+      const response = await requestRetry(`https://www.npmjs.com/browse/depended?offset=${offset}`, {
+        ...defaultOptions,
         headers: { 'x-spiferack': 1 }
       })
       const packages = response.packages || []
@@ -154,8 +156,9 @@ class TopProcessor extends AbstractProcessor {
     if (!end || end - start <= 0) end = start + 1000
     for (let offset = start; offset < end; offset += 100) {
       const page = offset / 100 + 1
-      const response = await requestRetry.get(
-        `https://crates.io/api/v1/crates?page=${page}&per_page=100&sort=downloads`
+      const response = await requestRetry(
+        `https://crates.io/api/v1/crates?page=${page}&per_page=100&sort=downloads`,
+        defaultOptions
       )
       const requestsPage = response.crates.map(
         x => new Request('package', `cd:/crate/cratesio/-/${x.name}/${x.max_version}`)
@@ -316,8 +319,9 @@ class TopProcessor extends AbstractProcessor {
     if (!start || start < 0) start = 0
     if (!end || end - start <= 0) end = start + 1000
     for (let offset = start; offset < end; offset += pageSize) {
-      const topComponents = await requestRetry.get(
-        `https://api-v2v3search-0.nuget.org/query?prerelease=false&skip=${offset}&take=${pageSize}`
+      const topComponents = await requestRetry(
+        `https://api-v2v3search-0.nuget.org/query?prerelease=false&skip=${offset}&take=${pageSize}`,
+        defaultOptions
       )
       const requests = topComponents.data.map(component => {
         return new Request('package', `cd:/nuget/nuget/-/${component.id}`)
@@ -349,7 +353,8 @@ class TopProcessor extends AbstractProcessor {
     })
     const requests = []
     for (let i = 0; i < repos.length; i++) {
-      const commits = await requestRetry.get(`https://api.github.com/repos/${namespace}/${repos[i].name}/commits`, {
+      const commits = await requestRetry(`https://api.github.com/repos/${namespace}/${repos[i].name}/commits`, {
+        ...defaultOptions,
         headers
       })
       if (commits.length > 0) {