From 6a6b5e9e0d1affe12829510759e130ee04be97cc Mon Sep 17 00:00:00 2001
From: Roman Iakovlev <iakovlev@live.de>
Date: Tue, 22 Oct 2024 16:27:50 +0200
Subject: [PATCH 1/4] Add workflow for running integration tests on a dedicated
 VM

---
 .github/workflows/cleanup-template.json       |   8 +
 .github/workflows/integration-test-custom.yml | 260 ++++++++++++++++++
 .github/workflows/vm-template.json            | 180 ++++++++++++
 .../test/integration/definitionDiff.js        |   6 +-
 .../e2e-test-service/attachmentTest.js        |  11 +-
 .../e2e-test-service/curationTest.js          |  27 +-
 .../e2e-test-service/definitionTest.js        |  16 +-
 .../e2e-test-service/noticeTest.js            |   4 +-
 .../integration/e2e-test-service/statsTest.js |   8 +-
 .../e2e-test-service/statusTest.js            |   8 +-
 .../test/integration/harvestTest.js           |   4 +-
 .../test/integration/testConfig.js            |  10 +-
 12 files changed, 498 insertions(+), 44 deletions(-)
 create mode 100644 .github/workflows/cleanup-template.json
 create mode 100644 .github/workflows/integration-test-custom.yml
 create mode 100644 .github/workflows/vm-template.json

diff --git a/.github/workflows/cleanup-template.json b/.github/workflows/cleanup-template.json
new file mode 100644
index 00000000..37de7b09
--- /dev/null
+++ b/.github/workflows/cleanup-template.json
@@ -0,0 +1,8 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {},
+    "variables": {},
+    "resources": [],
+    "outputs": {}
+  }
\ No newline at end of file
diff --git a/.github/workflows/integration-test-custom.yml b/.github/workflows/integration-test-custom.yml
new file mode 100644
index 00000000..25fe2164
--- /dev/null
+++ b/.github/workflows/integration-test-custom.yml
@@ -0,0 +1,260 @@
+name: Integration Test On Custom Infra
+
+on:
+  workflow_dispatch:
+    inputs:
+      baseFolderPath:
+        description: 'Base folder path for diffs'
+        required: true
+        default: 'diffs'
+      dynamicCoordinates:
+        description: 'Array of booleans for dynamic coordinates (e.g., [true, false])'
+        required: true
+        default: '[true, false]'
+      location:
+        description: 'Azure Region'
+        required: true
+        default: 'eastus'
+      vm_name:
+        description: 'VM Name'
+        required: true
+        default: 'integration-test-vm'
+      repo_refs:
+        description: 'Repo refs in format "clearlydefined/service:master,clearlydefined/crawler:master"'
+        required: true
+        default: 'clearlydefined/docker_dev_env_experiment:main,clearlydefined/service:master,clearlydefined/crawler:master,clearlydefined/website:master'
+
+
+permissions:
+  contents: read
+
+env:
+  REPOS: |
+    clearlydefined/service
+    clearlydefined/crawler
+    clearlydefined/website
+  COMPOSE_REPO: "clearlydefined/docker_dev_env_experiment"
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    # Add permissions for the GitHub token
+    permissions:
+      contents: read
+      packages: read
+    defaults:
+      run:
+        working-directory: ./tools/integration
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - uses: actions/setup-node@v4.0.1
+        with:
+          node-version: 18
+          cache: 'npm'
+          cache-dependency-path: './tools/integration/'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests on tools
+        run: npm test
+
+  deploy-and-run:
+    runs-on: ubuntu-latest
+    needs: unit-test
+    # Add permissions for the GitHub token
+    permissions:
+      contents: read
+      packages: read
+    outputs:
+      vm_ip: ${{ steps.create_vm.outputs.publicIpAddress }}
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - name: Azure Login
+        uses: azure/login@v2.2.0
+        with:
+          creds: ${{ secrets.VM_INT_AZURE_CREDENTIALS }}
+
+      - name: Cleanup existing resources
+        id: cleanup
+        uses: azure/arm-deploy@v2
+        with:
+          scope: resourcegroup
+          subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: integration-test-vm
+          template: ${{ github.workspace }}/.github/workflows/cleanup-template.json
+          deploymentMode: Complete
+          deploymentName: cleanup-deployment
+      
+      - name: Wait for cleanup to complete
+        uses: azure/cli@v2.1.0
+        with:
+          inlineScript: |
+            echo "Waiting for cleanup deployment to complete..."
+            az deployment group wait \
+              --name cleanup-deployment \
+              --resource-group integration-test-vm \
+              --created
+
+      - name: Create VM with NSG
+        id: create_vm
+        uses: azure/arm-deploy@v2
+        with:
+          scope: resourcegroup
+          subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: integration-test-vm
+          template: ${{ github.workspace }}/.github/workflows/vm-template.json
+          parameters: vmName="${{ github.event.inputs.vm_name }}" location="${{ github.event.inputs.location }}" sshPublicKey="${{ secrets.SSH_PUBLIC_KEY }}"
+      
+      - name: Wait SSH to become available
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ steps.create_vm.outputs.publicIpAddress }}
+          username: azureuser
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script: |
+            for i in `seq 1 12`; do
+              echo "Attempt $i: Waiting for SSH to become available"
+              if nc -zvw3 ${{ steps.create_vm.outputs.publicIpAddress }} 22; then
+                echo "SSH is available!"
+                exit 0
+              fi
+              sleep 10
+            done
+            echo "SSH is not available after 2 minutes; failing"
+            exit 1
+
+      - name: Install Docker
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ steps.create_vm.outputs.publicIpAddress }}
+          username: azureuser
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script: |
+            curl -fsSL https://get.docker.com -o get-docker.sh
+            sudo sh get-docker.sh
+            sudo systemctl enable docker
+            sudo systemctl start docker
+            sudo usermod -aG docker $USER
+            sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+            sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Clone Repos and Setup Environment
+        uses: appleboy/ssh-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.IT_GH_ACCESS_TOKEN }}
+          REPO_REFS: ${{ inputs.repo_refs }}
+        with:
+          host: ${{ steps.create_vm.outputs.publicIpAddress }}
+          username: azureuser
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          envs: GITHUB_TOKEN,REPO_REFS
+          script: |
+            # Function to get ref for a repo
+            get_ref() {
+              echo "$REPO_REFS" | tr ',' '\n' | grep "^$1:" | cut -d':' -f2 || echo "master"
+            }
+            
+            # First clone the compose repo
+            COMPOSE_REPO="${{ env.COMPOSE_REPO }}"
+            COMPOSE_REF=$(get_ref "$COMPOSE_REPO")
+            echo "Cloning compose repo https://github.com/${COMPOSE_REPO}.git branch/ref: ${COMPOSE_REF}"
+            git clone "https://github.com/${COMPOSE_REPO}.git" -b "$COMPOSE_REF"
+            
+            # Navigate to compose repo
+            REPO_NAME=$(echo ${COMPOSE_REPO} | cut -d'/' -f2)
+            cd $REPO_NAME
+            
+            # Clone other repositories inside the compose repo directory
+            echo "${{ env.REPOS }}" | while read repo; do
+              if [ ! -z "$repo" ] && [ "$repo" != "$COMPOSE_REPO" ]; then
+                REF=$(get_ref "$repo")
+                echo "Cloning https://github.com/${repo}.git branch/ref: ${REF}"
+                git clone "https://github.com/${repo}.git" -b "$REF"
+              fi
+            done
+            
+            # Copy sample_env to .env and replace tokens
+            cp sample_env .env
+            
+            # Replace both GitHub tokens with the provided token
+            sed -i "s/^CRAWLER_GITHUB_TOKEN=.*/CRAWLER_GITHUB_TOKEN=${GITHUB_TOKEN}/" .env
+            sed -i "s/^CURATION_GITHUB_TOKEN=.*/CURATION_GITHUB_TOKEN=${GITHUB_TOKEN}/" .env
+            echo "" >> .env && echo "CRAWLER_SCANCODE_PARALLELISM=14" >> .env
+            
+            # Start containers
+            sudo docker-compose up -d
+
+  integration-test:
+    runs-on: ubuntu-latest
+    needs: deploy-and-run
+    env:
+      DEV_API_BASE_URL: http://${{ needs.deploy-and-run.outputs.vm_ip }}:4000
+    strategy:
+      fail-fast: false
+      matrix:
+        dynamicCoordinates: ${{ fromJson(github.event.inputs.dynamicCoordinates) }}
+    defaults:
+      run:
+        working-directory: ./tools/integration
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - uses: actions/setup-node@v4.0.1
+        with:
+          node-version: 18
+          cache: 'npm'
+          cache-dependency-path: './tools/integration/'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Trigger harvest and verify completion
+        run: DEV_API_BASE_URL=${{ env.DEV_API_BASE_URL }} DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-harvest
+
+      - name: Verify service functions
+        id: verify-service-functions
+        continue-on-error: true
+        run: DEV_API_BASE_URL=${{ env.DEV_API_BASE_URL }} DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-service
+
+      - name: Generate structured diffs
+        run: DEV_API_BASE_URL=${{ env.DEV_API_BASE_URL }} DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run definitions-diff ${{ github.event.inputs.baseFolderPath }}
+
+      - name: Upload diffs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffs-${{ matrix.dynamicCoordinates == 'true' && 'dynamic' || 'static' }}
+          path: ./tools/integration/${{ github.event.inputs.baseFolderPath }}
+      
+      - name: Mark build status
+        if: steps.verify-service-functions.outcome == 'failure'
+        run: exit 1
+        
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: integration-test
+    if: ${{ always() }}
+    # Add permissions for the GitHub token
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - name: Azure Login
+        uses: azure/login@v2.2.0
+        with:
+          creds: ${{ secrets.VM_INT_AZURE_CREDENTIALS }}
+
+      - name: Cleanup existing resources
+        id: cleanup
+        uses: azure/arm-deploy@v2
+        with:
+          scope: resourcegroup
+          subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: integration-test-vm
+          template: ${{ github.workspace }}/.github/workflows/cleanup-template.json
+          deploymentMode: Complete
+          deploymentName: cleanup-deployment
\ No newline at end of file
diff --git a/.github/workflows/vm-template.json b/.github/workflows/vm-template.json
new file mode 100644
index 00000000..177164bb
--- /dev/null
+++ b/.github/workflows/vm-template.json
@@ -0,0 +1,180 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "vmName": {
+      "type": "string",
+      "metadata": {
+        "description": "Name of the VM"
+      }
+    },
+    "location": {
+      "type": "string",
+      "metadata": {
+        "description": "Location for the resources"
+      }
+    },
+    "sshPublicKey": {
+      "type": "string",
+      "metadata": {
+        "description": "SSH public key for authentication"
+      }
+    }
+  },
+  "variables": {
+    "nsgName": "[concat(parameters('vmName'), '-nsg')]",
+    "pipName": "[concat(parameters('vmName'), '-ip')]"
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Network/publicIPAddresses",
+      "apiVersion": "2021-02-01",
+      "name": "[variables('pipName')]",
+      "location": "[parameters('location')]",
+      "sku": {
+        "name": "Standard"
+      },
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {
+      "type": "Microsoft.Network/virtualNetworks",
+      "apiVersion": "2021-02-01",
+      "name": "[concat(parameters('vmName'), '-vnet')]",
+      "location": "[parameters('location')]",
+      "properties": {
+        "addressSpace": {
+          "addressPrefixes": ["10.0.0.0/16"]
+        },
+        "subnets": [
+          {
+            "name": "default",
+            "properties": {
+              "addressPrefix": "10.0.0.0/24"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "Microsoft.Network/networkSecurityGroups",
+      "apiVersion": "2021-02-01",
+      "name": "[variables('nsgName')]",
+      "location": "[parameters('location')]",
+      "properties": {
+        "securityRules": [
+          {
+            "name": "allow-ssh",
+            "properties": {
+              "priority": 1000,
+              "protocol": "Tcp",
+              "access": "Allow",
+              "direction": "Inbound",
+              "sourceAddressPrefix": "*",
+              "sourcePortRange": "*",
+              "destinationAddressPrefix": "*",
+              "destinationPortRange": "22"
+            }
+          },
+          {
+            "name": "allow-4000",
+            "properties": {
+              "priority": 1010,
+              "protocol": "Tcp",
+              "access": "Allow",
+              "direction": "Inbound",
+              "sourceAddressPrefix": "*",
+              "sourcePortRange": "*",
+              "destinationAddressPrefix": "*",
+              "destinationPortRange": "4000"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "Microsoft.Network/networkInterfaces",
+      "apiVersion": "2021-02-01",
+      "name": "[concat(parameters('vmName'), '-nic')]",
+      "location": "[parameters('location')]",
+      "dependsOn": [
+        "[resourceId('Microsoft.Network/publicIPAddresses', variables('pipName'))]",
+        "[resourceId('Microsoft.Network/virtualNetworks', concat(parameters('vmName'), '-vnet'))]",
+        "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "ipconfig1",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('pipName'))]"
+              },
+              "subnet": {
+                "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', concat(parameters('vmName'), '-vnet'), 'default')]"
+              }
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]"
+        }
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "apiVersion": "2021-03-01",
+      "name": "[parameters('vmName')]",
+      "location": "[parameters('location')]",
+      "dependsOn": [
+        "[resourceId('Microsoft.Network/networkInterfaces', concat(parameters('vmName'), '-nic'))]"
+      ],
+      "properties": {
+        "hardwareProfile": {
+          "vmSize": "Standard_D16_v5"
+        },
+        "osProfile": {
+          "computerName": "[parameters('vmName')]",
+          "adminUsername": "azureuser",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": true,
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "/home/azureuser/.ssh/authorized_keys",
+                  "keyData": "[parameters('sshPublicKey')]"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {
+            "publisher": "Canonical",
+            "offer": "0001-com-ubuntu-server-jammy",
+            "sku": "22_04-lts",
+            "version": "latest"
+          },
+          "osDisk": {
+            "createOption": "FromImage"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', concat(parameters('vmName'), '-nic'))]"
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "outputs": {
+    "publicIpAddress": {
+      "type": "string",
+      "value": "[reference(variables('pipName')).ipAddress]"
+    }
+  }
+}
\ No newline at end of file
diff --git a/tools/integration/test/integration/definitionDiff.js b/tools/integration/test/integration/definitionDiff.js
index 05e2f823..5b7a2803 100644
--- a/tools/integration/test/integration/definitionDiff.js
+++ b/tools/integration/test/integration/definitionDiff.js
@@ -1,7 +1,7 @@
 const fs = require('fs').promises
 const path = require('path')
 const { callFetch } = require('../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, getComponents } = require('./testConfig')
+const { getDevApiBaseUrl, prodApiBaseUrl, getComponents } = require('./testConfig')
 const { compareDocuments } = require('../../lib/compareDefinitions')
 
 async function main() {
@@ -22,7 +22,7 @@ async function main() {
 
 async function fetchAndCompareDefinition(coordinates, baseFolderPath) {
   const [recomputedDef, expectedDef] = await Promise.all([
-    getDefinition(devApiBaseUrl, coordinates, true),
+    getDefinition(getDevApiBaseUrl(), coordinates, true),
     getDefinition(prodApiBaseUrl, coordinates)
   ])
   const diff = compareDocuments(recomputedDef, expectedDef, [
@@ -37,7 +37,7 @@ async function fetchAndCompareDefinition(coordinates, baseFolderPath) {
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
+  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
diff --git a/tools/integration/test/integration/e2e-test-service/attachmentTest.js b/tools/integration/test/integration/e2e-test-service/attachmentTest.js
index 56eb3838..b03a2811 100644
--- a/tools/integration/test/integration/e2e-test-service/attachmentTest.js
+++ b/tools/integration/test/integration/e2e-test-service/attachmentTest.js
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
+const { getDevApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const { strictEqual } = require('assert')
 
 ;(async function () {
@@ -33,15 +33,16 @@ async function findAttachments(apiBaseUrl, coordinates) {
 
 async function compareAttachment(sha256) {
   const [devAttachment, prodAttachment] = await Promise.all(
-    [callFetch(`${devApiBaseUrl}/attachments/${sha256}`), callFetch(`${prodApiBaseUrl}/attachments/${sha256}`)].map(p =>
-      p.then(r => r.text())
-    )
+    [
+      callFetch(`${getDevApiBaseUrl()}/attachments/${sha256}`),
+      callFetch(`${prodApiBaseUrl}/attachments/${sha256}`)
+    ].map(p => p.then(r => r.text()))
   )
   strictEqual(devAttachment, prodAttachment)
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
+  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
diff --git a/tools/integration/test/integration/e2e-test-service/curationTest.js b/tools/integration/test/integration/e2e-test-service/curationTest.js
index 08a0122e..97b36fe1 100644
--- a/tools/integration/test/integration/e2e-test-service/curationTest.js
+++ b/tools/integration/test/integration/e2e-test-service/curationTest.js
@@ -3,7 +3,7 @@
 
 const { deepStrictEqual, strictEqual, ok } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { devApiBaseUrl, definition } = require('../testConfig')
+const { getDevApiBaseUrl, definition } = require('../testConfig')
 
 describe('Validate curation', function () {
   this.timeout(definition.timeout)
@@ -24,7 +24,7 @@ describe('Validate curation', function () {
 
     before('curate', async function () {
       const curationResponse = await callFetch(
-        `${devApiBaseUrl}/curations`,
+        `${getDevApiBaseUrl()}/curations`,
         buildCurationOpts(coordinates, type, provider, namespace, name, revision, curation)
       ).then(r => r.json())
       prNumber = curationResponse.prNumber
@@ -35,22 +35,22 @@ describe('Validate curation', function () {
     })
 
     it(`should get the curation by PR via /curations/${coordinates}/pr`, async function () {
-      const fetchedCuration = await callFetch(`${devApiBaseUrl}/curations/${coordinates}/pr/${prNumber}`).then(r =>
+      const fetchedCuration = await callFetch(`${getDevApiBaseUrl()}/curations/${coordinates}/pr/${prNumber}`).then(r =>
         r.json()
       )
       deepStrictEqual(fetchedCuration, curation)
     })
 
     it(`should reflect the PR in definition preview via definitions/${coordinates}/pr`, async function () {
-      const curatedDefinition = await callFetch(`${devApiBaseUrl}/definitions/${coordinates}/pr/${prNumber}`).then(r =>
-        r.json()
+      const curatedDefinition = await callFetch(`${getDevApiBaseUrl()}/definitions/${coordinates}/pr/${prNumber}`).then(
+        r => r.json()
       )
       strictEqual(curatedDefinition.described.releaseDate, curation.described.releaseDate)
     })
 
     it(`should get of list of PRs for component via /curations/${type}/${provider}/${namespace}/${name}`, async function () {
-      const response = await callFetch(`${devApiBaseUrl}/curations/${type}/${provider}/${namespace}/${name}`).then(r =>
-        r.json()
+      const response = await callFetch(`${getDevApiBaseUrl()}/curations/${type}/${provider}/${namespace}/${name}`).then(
+        r => r.json()
       )
       const proposedPR = response.contributions.filter(c => c.prNumber === prNumber)
       ok(proposedPR)
@@ -58,9 +58,10 @@ describe('Validate curation', function () {
 
     it('should get PRs for components via post /curations', async function () {
       const revisionlessCoordinates = `${type}/${provider}/${namespace}/${name}`
-      const response = await callFetch(`${devApiBaseUrl}/curations`, buildPostOpts([revisionlessCoordinates])).then(r =>
-        r.json()
-      )
+      const response = await callFetch(
+        `${getDevApiBaseUrl()}/curations`,
+        buildPostOpts([revisionlessCoordinates])
+      ).then(r => r.json())
       const proposedPR = response[revisionlessCoordinates].contributions.filter(c => c.prNumber === prNumber)
       ok(proposedPR)
     })
@@ -74,12 +75,12 @@ describe('Validate curation', function () {
       }
     }
     it(`should get merged curation for coordinates via /curations/${curatedCoordinates}`, async function () {
-      const response = await callFetch(`${devApiBaseUrl}/curations/${curatedCoordinates}`).then(r => r.json())
+      const response = await callFetch(`${getDevApiBaseUrl()}/curations/${curatedCoordinates}`).then(r => r.json())
       deepStrictEqual(response, expected)
     })
 
     it(`should reflect merged curation in definition for coordinates ${curatedCoordinates}`, async function () {
-      const curatedDefinition = await getDefinition(devApiBaseUrl, curatedCoordinates, true)
+      const curatedDefinition = await getDefinition(getDevApiBaseUrl(), curatedCoordinates, true)
       deepStrictEqual(curatedDefinition.licensed.declared, expected.licensed.declared)
     })
   })
@@ -105,7 +106,7 @@ function buildCurationOpts(coordinates, type, provider, namespace, name, revisio
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
+  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
diff --git a/tools/integration/test/integration/e2e-test-service/definitionTest.js b/tools/integration/test/integration/e2e-test-service/definitionTest.js
index 8ee9b35a..915e7d22 100644
--- a/tools/integration/test/integration/e2e-test-service/definitionTest.js
+++ b/tools/integration/test/integration/e2e-test-service/definitionTest.js
@@ -4,7 +4,7 @@
 const { omit, isEqual, pick } = require('lodash')
 const { deepStrictEqual, strictEqual, ok } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
+const { getDevApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')
 
@@ -35,7 +35,7 @@ const fs = require('fs')
         it(`should find definition for ${coordinates}`, async function () {
           const [foundDef, expectedDef] = await Promise.all([
             findDefinition(coordinates),
-            getDefinition(devApiBaseUrl, coordinates)
+            getDefinition(getDevApiBaseUrl(), coordinates)
           ])
           deepStrictEqualExpectedEntries(foundDef, omit(expectedDef, ['files', '_id']))
         })
@@ -43,19 +43,19 @@ const fs = require('fs')
 
       describe('Search coordinates via pattern', function () {
         it(`should find coordinates for aws-sdk-java`, async function () {
-          const response = await callFetch(`${devApiBaseUrl}/definitions?pattern=aws-sdk-java`).then(r => r.json())
+          const response = await callFetch(`${getDevApiBaseUrl()}/definitions?pattern=aws-sdk-java`).then(r => r.json())
           ok(response.length > 0)
         })
       })
 
       describe('Post to /definitions', function () {
         it(`should get definition via post to /definitions for ${coordinates}`, async function () {
-          const postDefinitions = callFetch(`${devApiBaseUrl}/definitions`, buildPostOpts([coordinates])).then(r =>
+          const postDefinitions = callFetch(`${getDevApiBaseUrl()}/definitions`, buildPostOpts([coordinates])).then(r =>
             r.json()
           )
           const [actualDef, expectedDef] = await Promise.all([
             postDefinitions.then(r => r[coordinates]),
-            getDefinition(devApiBaseUrl, coordinates)
+            getDefinition(getDevApiBaseUrl(), coordinates)
           ])
           deepStrictEqualExpectedEntries(actualDef, expectedDef)
         })
@@ -66,14 +66,14 @@ const fs = require('fs')
 
 async function fetchAndCompareDefinition(coordinates) {
   const [recomputedDef, expectedDef] = await Promise.all([
-    getDefinition(devApiBaseUrl, coordinates, true),
+    getDefinition(getDevApiBaseUrl(), coordinates, true),
     getDefinition(prodApiBaseUrl, coordinates)
   ])
   compareDefinition(recomputedDef, expectedDef)
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
+  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
@@ -136,7 +136,7 @@ async function findDefinition(coordinates) {
   let coordinatesString = `type=${type}&provider=${provider}&name=${name}`
   coordinatesString += namespace && namespace !== '-' ? `&namespace=${namespace}` : ''
   const response = await callFetch(
-    `${devApiBaseUrl}/definitions?${coordinatesString}&sortDesc=true&sort=revision`
+    `${getDevApiBaseUrl()}/definitions?${coordinatesString}&sortDesc=true&sort=revision`
   ).then(r => r.json())
   return response.data.find(d => d.coordinates.revision === revision)
 }
diff --git a/tools/integration/test/integration/e2e-test-service/noticeTest.js b/tools/integration/test/integration/e2e-test-service/noticeTest.js
index a87f27c2..0c6f3f6a 100644
--- a/tools/integration/test/integration/e2e-test-service/noticeTest.js
+++ b/tools/integration/test/integration/e2e-test-service/noticeTest.js
@@ -3,7 +3,7 @@
 
 const { deepStrictEqual } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
+const { getDevApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')
 
@@ -32,7 +32,7 @@ async function fetchAndCompareNotices(coordinates) {
   const [computedNotice, expectedNotice] = await Promise.all(
     [
       callFetch(
-        `${devApiBaseUrl}/notices`,
+        `${getDevApiBaseUrl()}/notices`,
         buildPostOpts({
           coordinates: [coordinates]
         })
diff --git a/tools/integration/test/integration/e2e-test-service/statsTest.js b/tools/integration/test/integration/e2e-test-service/statsTest.js
index 0ddd7a95..3cfcc652 100644
--- a/tools/integration/test/integration/e2e-test-service/statsTest.js
+++ b/tools/integration/test/integration/e2e-test-service/statsTest.js
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { devApiBaseUrl, definition } = require('../testConfig')
+const { getDevApiBaseUrl, definition } = require('../testConfig')
 const { ok } = require('assert')
 
 describe('Test for StatsService', function () {
@@ -12,7 +12,7 @@ describe('Test for StatsService', function () {
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout)))
 
   it('should retrieve the list of supported stats', async function () {
-    const url = `${devApiBaseUrl}/stats`
+    const url = `${getDevApiBaseUrl()}/stats`
     const result = await callFetch(url).then(r => r.json())
     const expected = [
       'total',
@@ -35,14 +35,14 @@ describe('Test for StatsService', function () {
   })
 
   it('should retrieve stats for total', async function () {
-    const url = `${devApiBaseUrl}/stats/total`
+    const url = `${getDevApiBaseUrl()}/stats/total`
     const result = await callFetch(url).then(r => r.json())
     ok(result.value.totalCount > 0)
     ok(result.value.declaredLicenseBreakdown)
   })
 
   it('should retrieve stats for composer', async function () {
-    const url = `${devApiBaseUrl}/stats/composer`
+    const url = `${getDevApiBaseUrl()}/stats/composer`
     const result = await callFetch(url).then(r => r.json())
     ok(result.value.totalCount > 0)
     ok(result.value.declaredLicenseBreakdown)
diff --git a/tools/integration/test/integration/e2e-test-service/statusTest.js b/tools/integration/test/integration/e2e-test-service/statusTest.js
index 77600e1e..0afd46a7 100644
--- a/tools/integration/test/integration/e2e-test-service/statusTest.js
+++ b/tools/integration/test/integration/e2e-test-service/statusTest.js
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { devApiBaseUrl, definition } = require('../testConfig')
+const { getDevApiBaseUrl, definition } = require('../testConfig')
 const { ok } = require('assert')
 
 describe('Test for StatusService', function () {
@@ -12,7 +12,7 @@ describe('Test for StatusService', function () {
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout)))
 
   it('should retrieve the list of supported status queries', async function () {
-    const url = `${devApiBaseUrl}/status`
+    const url = `${getDevApiBaseUrl()}/status`
     const result = await callFetch(url).then(r => r.json())
     const expected = [
       'requestcount',
@@ -27,14 +27,14 @@ describe('Test for StatusService', function () {
   })
 
   it('should retrieve toolsranperday status via crawler query', async function () {
-    const url = `${devApiBaseUrl}/status/toolsranperday`
+    const url = `${getDevApiBaseUrl()}/status/toolsranperday`
     const result = await callFetch(url).then(r => r.json())
     ok(result.length > 0)
     ok(result[0].clearlydefined > 0 || result[0].licensee > 0 || result[0].reuse > 0 || result[0].scancode > 0)
   })
 
   it('should retrieve requestCount status (including today) via service query', async function () {
-    const url = `${devApiBaseUrl}/status/requestCount`
+    const url = `${getDevApiBaseUrl()}/status/requestCount`
     const result = await callFetch(url).then(r => r.json())
     const sortedDates = Object.keys(result).sort((a, b) => b.localeCompare(a))
     ok(sortedDates.length > 0)
diff --git a/tools/integration/test/integration/harvestTest.js b/tools/integration/test/integration/harvestTest.js
index ac258e6c..39fb5a4e 100644
--- a/tools/integration/test/integration/harvestTest.js
+++ b/tools/integration/test/integration/harvestTest.js
@@ -1,7 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const { getComponents, devApiBaseUrl, harvest } = require('./testConfig')
+const { getComponents, getDevApiBaseUrl, harvest } = require('./testConfig')
 const Poller = require('../../lib/poller')
 const Harvester = require('../../lib/harvester')
 const { strictEqual } = require('assert')
@@ -24,7 +24,7 @@ async function harvestTillCompletion(components) {
   if (components.length === 0) return new Map()
 
   const { poll, tools } = harvest
-  const harvester = new Harvester(devApiBaseUrl)
+  const harvester = new Harvester(getDevApiBaseUrl())
 
   const oneComponent = components.shift()
   const versionPoller = new Poller(poll.interval / 5, poll.maxTime)
diff --git a/tools/integration/test/integration/testConfig.js b/tools/integration/test/integration/testConfig.js
index 36049862..65cfaa6c 100644
--- a/tools/integration/test/integration/testConfig.js
+++ b/tools/integration/test/integration/testConfig.js
@@ -3,10 +3,14 @@
 const fs = require('fs').promises
 const path = require('path')
 
-const devApiBaseUrl = 'https://dev-api.clearlydefined.io'
+function getDevApiBaseUrl() {
+  const url = process.env.DEV_API_BASE_URL || 'https://dev-api.clearlydefined.io'
+  console.info(`Using dev api base url: ${url}`)
+  return url
+}
 const prodApiBaseUrl = 'https://api.clearlydefined.io'
 
-const pollingInterval = 1000 * 60 * 5 // 5 minutes
+const pollingInterval = 1000 * 60 * 1 // 1 minute
 const pollingMaxTime = 1000 * 60 * 60 // 60 minutes
 
 //Havest tools to check for harvest completeness
@@ -72,7 +76,7 @@ const componentsDynamic = async () => {
 }
 
 module.exports = {
-  devApiBaseUrl,
+  getDevApiBaseUrl,
   prodApiBaseUrl,
   getComponents,
   harvest: {

From 26f80173aa66cc1cd9dd0f1a18ae49dbdcc7aeef Mon Sep 17 00:00:00 2001
From: Roman Iakovlev <iakovlev@live.de>
Date: Mon, 18 Nov 2024 11:54:17 +0100
Subject: [PATCH 2/4] Add docs for scancode parallelism

---
 overview.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/overview.md b/overview.md
index 43ebb1c1..bd12b6f2 100644
--- a/overview.md
+++ b/overview.md
@@ -163,6 +163,7 @@ The environmental variables for the cdcrawler-dev App Service include:
 * CRAWLER_STORE_PROVIDER
 * CRAWLER_WEBHOOK_TOKEN
 * CRAWLER_WEBHOOK_URL
+* CRAWLER_SCANCODE_PARALLELISM
 * DOCKER_CUSTOM_IMAGE_NAME
 * DOCKER_ENABLE_CI
 * DOCKER_REGISTRY_SERVER_PASSWORD
@@ -264,6 +265,13 @@ In this case, the webhook url is "https://dev-api.clearlydefined.io/webhook".
 
 The token is what we use to authenticate to the API (so that only the crawler can call that part of the ClearlyDefined Service api)
 
+**CRAWLER_SCANCODE_PARALLELISM**
+
+This environment variable is a number of `scancode-toolkit` processes to run in parallel. `scancode-toolkit` is one of the main
+tools that collect the licensing data that goes into the final definition, and increasing parallelism, if the CPU allows,
+speeds up processing of individual definitions. The default value is `2`, and a good ballpark value is ~80% of total CPUs 
+available for crawler.
+
 **DOCKER**
 
 The Docker environmental variables define what container image is used for the Crawler, as well as what registry that image is kept in, and authentication info for the registry.

From 8a8d5556f0b6ac9cd0607d7609476bc83abcdb9a Mon Sep 17 00:00:00 2001
From: Roman Iakovlev <iakovlev@live.de>
Date: Mon, 18 Nov 2024 11:55:37 +0100
Subject: [PATCH 3/4] Only run definition tests on custom integration infra

---
 .github/workflows/integration-test.yml | 2 +-
 tools/integration/package.json         | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 0665f458..12fe1a47 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Verify service functions
         id: verify-service-functions
         continue-on-error: true
-        run: DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-service
+        run: DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-definition
 
       - name: Generate structured diffs
         run: DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run definitions-diff ${{ github.event.inputs.baseFolderPath }}
diff --git a/tools/integration/package.json b/tools/integration/package.json
index 2eea4fc0..9de6bcb6 100644
--- a/tools/integration/package.json
+++ b/tools/integration/package.json
@@ -6,6 +6,7 @@
     "test": "npm run mocha && npm run lint",
     "e2e-test-harvest": "mocha test/integration/harvestTest.js",
     "e2e-test-service": "mocha --exit \"test/integration/e2e-test-service/**/*.js\"",
+    "e2e-test-definition": "mocha --exit \"test/integration/e2e-test-service/definitionTest.js\"",
     "mocha": "mocha --exit \"test/lib/**/*.js\"",
     "lint": "npm run prettier:check && npm run eslint",
     "lint:fix": "npm run prettier:write && npm run eslint:fix",

From d441c0839c8e22893cef5f8bd2823b1a8341e9d8 Mon Sep 17 00:00:00 2001
From: Roman Iakovlev <iakovlev@live.de>
Date: Mon, 18 Nov 2024 11:56:20 +0100
Subject: [PATCH 4/4] Use constant for integration test crawler url

---
 .../test/integration/definitionDiff.js        |  6 ++---
 .../e2e-test-service/attachmentTest.js        | 11 ++++----
 .../e2e-test-service/curationTest.js          | 27 +++++++++----------
 .../e2e-test-service/definitionTest.js        | 16 +++++------
 .../e2e-test-service/noticeTest.js            |  4 +--
 .../integration/e2e-test-service/statsTest.js |  8 +++---
 .../e2e-test-service/statusTest.js            |  8 +++---
 .../test/integration/harvestTest.js           |  4 +--
 .../test/integration/testConfig.js            |  3 ++-
 9 files changed, 43 insertions(+), 44 deletions(-)

diff --git a/tools/integration/test/integration/definitionDiff.js b/tools/integration/test/integration/definitionDiff.js
index 5b7a2803..05e2f823 100644
--- a/tools/integration/test/integration/definitionDiff.js
+++ b/tools/integration/test/integration/definitionDiff.js
@@ -1,7 +1,7 @@
 const fs = require('fs').promises
 const path = require('path')
 const { callFetch } = require('../../lib/fetch')
-const { getDevApiBaseUrl, prodApiBaseUrl, getComponents } = require('./testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents } = require('./testConfig')
 const { compareDocuments } = require('../../lib/compareDefinitions')
 
 async function main() {
@@ -22,7 +22,7 @@ async function main() {
 
 async function fetchAndCompareDefinition(coordinates, baseFolderPath) {
   const [recomputedDef, expectedDef] = await Promise.all([
-    getDefinition(getDevApiBaseUrl(), coordinates, true),
+    getDefinition(devApiBaseUrl, coordinates, true),
     getDefinition(prodApiBaseUrl, coordinates)
   ])
   const diff = compareDocuments(recomputedDef, expectedDef, [
@@ -37,7 +37,7 @@ async function fetchAndCompareDefinition(coordinates, baseFolderPath) {
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
+  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
diff --git a/tools/integration/test/integration/e2e-test-service/attachmentTest.js b/tools/integration/test/integration/e2e-test-service/attachmentTest.js
index b03a2811..56eb3838 100644
--- a/tools/integration/test/integration/e2e-test-service/attachmentTest.js
+++ b/tools/integration/test/integration/e2e-test-service/attachmentTest.js
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { getDevApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const { strictEqual } = require('assert')
 
 ;(async function () {
@@ -33,16 +33,15 @@ async function findAttachments(apiBaseUrl, coordinates) {
 
 async function compareAttachment(sha256) {
   const [devAttachment, prodAttachment] = await Promise.all(
-    [
-      callFetch(`${getDevApiBaseUrl()}/attachments/${sha256}`),
-      callFetch(`${prodApiBaseUrl}/attachments/${sha256}`)
-    ].map(p => p.then(r => r.text()))
+    [callFetch(`${devApiBaseUrl}/attachments/${sha256}`), callFetch(`${prodApiBaseUrl}/attachments/${sha256}`)].map(p =>
+      p.then(r => r.text())
+    )
   )
   strictEqual(devAttachment, prodAttachment)
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
+  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
diff --git a/tools/integration/test/integration/e2e-test-service/curationTest.js b/tools/integration/test/integration/e2e-test-service/curationTest.js
index 97b36fe1..08a0122e 100644
--- a/tools/integration/test/integration/e2e-test-service/curationTest.js
+++ b/tools/integration/test/integration/e2e-test-service/curationTest.js
@@ -3,7 +3,7 @@
 
 const { deepStrictEqual, strictEqual, ok } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { getDevApiBaseUrl, definition } = require('../testConfig')
+const { devApiBaseUrl, definition } = require('../testConfig')
 
 describe('Validate curation', function () {
   this.timeout(definition.timeout)
@@ -24,7 +24,7 @@ describe('Validate curation', function () {
 
     before('curate', async function () {
       const curationResponse = await callFetch(
-        `${getDevApiBaseUrl()}/curations`,
+        `${devApiBaseUrl}/curations`,
         buildCurationOpts(coordinates, type, provider, namespace, name, revision, curation)
       ).then(r => r.json())
       prNumber = curationResponse.prNumber
@@ -35,22 +35,22 @@ describe('Validate curation', function () {
     })
 
     it(`should get the curation by PR via /curations/${coordinates}/pr`, async function () {
-      const fetchedCuration = await callFetch(`${getDevApiBaseUrl()}/curations/${coordinates}/pr/${prNumber}`).then(r =>
+      const fetchedCuration = await callFetch(`${devApiBaseUrl}/curations/${coordinates}/pr/${prNumber}`).then(r =>
         r.json()
       )
       deepStrictEqual(fetchedCuration, curation)
     })
 
     it(`should reflect the PR in definition preview via definitions/${coordinates}/pr`, async function () {
-      const curatedDefinition = await callFetch(`${getDevApiBaseUrl()}/definitions/${coordinates}/pr/${prNumber}`).then(
-        r => r.json()
+      const curatedDefinition = await callFetch(`${devApiBaseUrl}/definitions/${coordinates}/pr/${prNumber}`).then(r =>
+        r.json()
       )
       strictEqual(curatedDefinition.described.releaseDate, curation.described.releaseDate)
     })
 
     it(`should get of list of PRs for component via /curations/${type}/${provider}/${namespace}/${name}`, async function () {
-      const response = await callFetch(`${getDevApiBaseUrl()}/curations/${type}/${provider}/${namespace}/${name}`).then(
-        r => r.json()
+      const response = await callFetch(`${devApiBaseUrl}/curations/${type}/${provider}/${namespace}/${name}`).then(r =>
+        r.json()
       )
       const proposedPR = response.contributions.filter(c => c.prNumber === prNumber)
       ok(proposedPR)
@@ -58,10 +58,9 @@ describe('Validate curation', function () {
 
     it('should get PRs for components via post /curations', async function () {
       const revisionlessCoordinates = `${type}/${provider}/${namespace}/${name}`
-      const response = await callFetch(
-        `${getDevApiBaseUrl()}/curations`,
-        buildPostOpts([revisionlessCoordinates])
-      ).then(r => r.json())
+      const response = await callFetch(`${devApiBaseUrl}/curations`, buildPostOpts([revisionlessCoordinates])).then(r =>
+        r.json()
+      )
       const proposedPR = response[revisionlessCoordinates].contributions.filter(c => c.prNumber === prNumber)
       ok(proposedPR)
     })
@@ -75,12 +74,12 @@ describe('Validate curation', function () {
       }
     }
     it(`should get merged curation for coordinates via /curations/${curatedCoordinates}`, async function () {
-      const response = await callFetch(`${getDevApiBaseUrl()}/curations/${curatedCoordinates}`).then(r => r.json())
+      const response = await callFetch(`${devApiBaseUrl}/curations/${curatedCoordinates}`).then(r => r.json())
       deepStrictEqual(response, expected)
     })
 
     it(`should reflect merged curation in definition for coordinates ${curatedCoordinates}`, async function () {
-      const curatedDefinition = await getDefinition(getDevApiBaseUrl(), curatedCoordinates, true)
+      const curatedDefinition = await getDefinition(devApiBaseUrl, curatedCoordinates, true)
       deepStrictEqual(curatedDefinition.licensed.declared, expected.licensed.declared)
     })
   })
@@ -106,7 +105,7 @@ function buildCurationOpts(coordinates, type, provider, namespace, name, revisio
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
+  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
diff --git a/tools/integration/test/integration/e2e-test-service/definitionTest.js b/tools/integration/test/integration/e2e-test-service/definitionTest.js
index 915e7d22..8ee9b35a 100644
--- a/tools/integration/test/integration/e2e-test-service/definitionTest.js
+++ b/tools/integration/test/integration/e2e-test-service/definitionTest.js
@@ -4,7 +4,7 @@
 const { omit, isEqual, pick } = require('lodash')
 const { deepStrictEqual, strictEqual, ok } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { getDevApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')
 
@@ -35,7 +35,7 @@ const fs = require('fs')
         it(`should find definition for ${coordinates}`, async function () {
           const [foundDef, expectedDef] = await Promise.all([
             findDefinition(coordinates),
-            getDefinition(getDevApiBaseUrl(), coordinates)
+            getDefinition(devApiBaseUrl, coordinates)
           ])
           deepStrictEqualExpectedEntries(foundDef, omit(expectedDef, ['files', '_id']))
         })
@@ -43,19 +43,19 @@ const fs = require('fs')
 
       describe('Search coordinates via pattern', function () {
         it(`should find coordinates for aws-sdk-java`, async function () {
-          const response = await callFetch(`${getDevApiBaseUrl()}/definitions?pattern=aws-sdk-java`).then(r => r.json())
+          const response = await callFetch(`${devApiBaseUrl}/definitions?pattern=aws-sdk-java`).then(r => r.json())
           ok(response.length > 0)
         })
       })
 
       describe('Post to /definitions', function () {
         it(`should get definition via post to /definitions for ${coordinates}`, async function () {
-          const postDefinitions = callFetch(`${getDevApiBaseUrl()}/definitions`, buildPostOpts([coordinates])).then(r =>
+          const postDefinitions = callFetch(`${devApiBaseUrl}/definitions`, buildPostOpts([coordinates])).then(r =>
             r.json()
           )
           const [actualDef, expectedDef] = await Promise.all([
             postDefinitions.then(r => r[coordinates]),
-            getDefinition(getDevApiBaseUrl(), coordinates)
+            getDefinition(devApiBaseUrl, coordinates)
           ])
           deepStrictEqualExpectedEntries(actualDef, expectedDef)
         })
@@ -66,14 +66,14 @@ const fs = require('fs')
 
 async function fetchAndCompareDefinition(coordinates) {
   const [recomputedDef, expectedDef] = await Promise.all([
-    getDefinition(getDevApiBaseUrl(), coordinates, true),
+    getDefinition(devApiBaseUrl, coordinates, true),
     getDefinition(prodApiBaseUrl, coordinates)
   ])
   compareDefinition(recomputedDef, expectedDef)
 }
 
 async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
-  reCompute = apiBaseUrl === getDevApiBaseUrl() && reCompute
+  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
   let url = `${apiBaseUrl}/definitions/${coordinates}`
   if (reCompute) url += '?force=true'
   return await callFetch(url).then(r => r.json())
@@ -136,7 +136,7 @@ async function findDefinition(coordinates) {
   let coordinatesString = `type=${type}&provider=${provider}&name=${name}`
   coordinatesString += namespace && namespace !== '-' ? `&namespace=${namespace}` : ''
   const response = await callFetch(
-    `${getDevApiBaseUrl()}/definitions?${coordinatesString}&sortDesc=true&sort=revision`
+    `${devApiBaseUrl}/definitions?${coordinatesString}&sortDesc=true&sort=revision`
   ).then(r => r.json())
   return response.data.find(d => d.coordinates.revision === revision)
 }
diff --git a/tools/integration/test/integration/e2e-test-service/noticeTest.js b/tools/integration/test/integration/e2e-test-service/noticeTest.js
index 0c6f3f6a..a87f27c2 100644
--- a/tools/integration/test/integration/e2e-test-service/noticeTest.js
+++ b/tools/integration/test/integration/e2e-test-service/noticeTest.js
@@ -3,7 +3,7 @@
 
 const { deepStrictEqual } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { getDevApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')
 
@@ -32,7 +32,7 @@ async function fetchAndCompareNotices(coordinates) {
   const [computedNotice, expectedNotice] = await Promise.all(
     [
       callFetch(
-        `${getDevApiBaseUrl()}/notices`,
+        `${devApiBaseUrl}/notices`,
         buildPostOpts({
           coordinates: [coordinates]
         })
diff --git a/tools/integration/test/integration/e2e-test-service/statsTest.js b/tools/integration/test/integration/e2e-test-service/statsTest.js
index 3cfcc652..0ddd7a95 100644
--- a/tools/integration/test/integration/e2e-test-service/statsTest.js
+++ b/tools/integration/test/integration/e2e-test-service/statsTest.js
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { getDevApiBaseUrl, definition } = require('../testConfig')
+const { devApiBaseUrl, definition } = require('../testConfig')
 const { ok } = require('assert')
 
 describe('Test for StatsService', function () {
@@ -12,7 +12,7 @@ describe('Test for StatsService', function () {
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout)))
 
   it('should retrieve the list of supported stats', async function () {
-    const url = `${getDevApiBaseUrl()}/stats`
+    const url = `${devApiBaseUrl}/stats`
     const result = await callFetch(url).then(r => r.json())
     const expected = [
       'total',
@@ -35,14 +35,14 @@ describe('Test for StatsService', function () {
   })
 
   it('should retrieve stats for total', async function () {
-    const url = `${getDevApiBaseUrl()}/stats/total`
+    const url = `${devApiBaseUrl}/stats/total`
     const result = await callFetch(url).then(r => r.json())
     ok(result.value.totalCount > 0)
     ok(result.value.declaredLicenseBreakdown)
   })
 
   it('should retrieve stats for composer', async function () {
-    const url = `${getDevApiBaseUrl()}/stats/composer`
+    const url = `${devApiBaseUrl}/stats/composer`
     const result = await callFetch(url).then(r => r.json())
     ok(result.value.totalCount > 0)
     ok(result.value.declaredLicenseBreakdown)
diff --git a/tools/integration/test/integration/e2e-test-service/statusTest.js b/tools/integration/test/integration/e2e-test-service/statusTest.js
index 0afd46a7..77600e1e 100644
--- a/tools/integration/test/integration/e2e-test-service/statusTest.js
+++ b/tools/integration/test/integration/e2e-test-service/statusTest.js
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { getDevApiBaseUrl, definition } = require('../testConfig')
+const { devApiBaseUrl, definition } = require('../testConfig')
 const { ok } = require('assert')
 
 describe('Test for StatusService', function () {
@@ -12,7 +12,7 @@ describe('Test for StatusService', function () {
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout)))
 
   it('should retrieve the list of supported status queries', async function () {
-    const url = `${getDevApiBaseUrl()}/status`
+    const url = `${devApiBaseUrl}/status`
     const result = await callFetch(url).then(r => r.json())
     const expected = [
       'requestcount',
@@ -27,14 +27,14 @@ describe('Test for StatusService', function () {
   })
 
   it('should retrieve toolsranperday status via crawler query', async function () {
-    const url = `${getDevApiBaseUrl()}/status/toolsranperday`
+    const url = `${devApiBaseUrl}/status/toolsranperday`
     const result = await callFetch(url).then(r => r.json())
     ok(result.length > 0)
     ok(result[0].clearlydefined > 0 || result[0].licensee > 0 || result[0].reuse > 0 || result[0].scancode > 0)
   })
 
   it('should retrieve requestCount status (including today) via service query', async function () {
-    const url = `${getDevApiBaseUrl()}/status/requestCount`
+    const url = `${devApiBaseUrl}/status/requestCount`
     const result = await callFetch(url).then(r => r.json())
     const sortedDates = Object.keys(result).sort((a, b) => b.localeCompare(a))
     ok(sortedDates.length > 0)
diff --git a/tools/integration/test/integration/harvestTest.js b/tools/integration/test/integration/harvestTest.js
index 39fb5a4e..ac258e6c 100644
--- a/tools/integration/test/integration/harvestTest.js
+++ b/tools/integration/test/integration/harvestTest.js
@@ -1,7 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const { getComponents, getDevApiBaseUrl, harvest } = require('./testConfig')
+const { getComponents, devApiBaseUrl, harvest } = require('./testConfig')
 const Poller = require('../../lib/poller')
 const Harvester = require('../../lib/harvester')
 const { strictEqual } = require('assert')
@@ -24,7 +24,7 @@ async function harvestTillCompletion(components) {
   if (components.length === 0) return new Map()
 
   const { poll, tools } = harvest
-  const harvester = new Harvester(getDevApiBaseUrl())
+  const harvester = new Harvester(devApiBaseUrl)
 
   const oneComponent = components.shift()
   const versionPoller = new Poller(poll.interval / 5, poll.maxTime)
diff --git a/tools/integration/test/integration/testConfig.js b/tools/integration/test/integration/testConfig.js
index 65cfaa6c..a03c8561 100644
--- a/tools/integration/test/integration/testConfig.js
+++ b/tools/integration/test/integration/testConfig.js
@@ -9,6 +9,7 @@ function getDevApiBaseUrl() {
   return url
 }
 const prodApiBaseUrl = 'https://api.clearlydefined.io'
+const devApiBaseUrl = getDevApiBaseUrl()
 
 const pollingInterval = 1000 * 60 * 1 // 1 minute
 const pollingMaxTime = 1000 * 60 * 60 // 60 minutes
@@ -76,7 +77,7 @@ const componentsDynamic = async () => {
 }
 
 module.exports = {
-  getDevApiBaseUrl,
+  devApiBaseUrl,
   prodApiBaseUrl,
   getComponents,
   harvest: {