Skip to content

Add a download_scan_data action in REST API packages endpoint #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tdruez merged 10 commits into from
Jun 4, 2025
Merged

Add a download_scan_data action in REST API packages endpoint #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
49f211f
Add a `download_scan_data` action in REST API packages endpoint #272
tdruez May 29, 2025
aaa533c
Merge main and fix conflicts
tdruez Jun 3, 2025
0ae2fb0
Add unit test for the download_scan_data action #272
tdruez Jun 3, 2025
4eaf2fd
Refactor duplicate code #272
tdruez Jun 3, 2025
3398e7c
Refine the scan_data_as_zip implementation #272
tdruez Jun 3, 2025
0df7ee8
Rename get_scan_results to get_project_info for clarity #272
tdruez Jun 4, 2025
4936383
Simplify the get_project_info api #272
tdruez Jun 4, 2025
4e682c8
Add scan_info, scan_results, and scan_summary actions in API #272
tdruez Jun 4, 2025
00d9810
Add unit tests and changelog #272
tdruez Jun 4, 2025
3c72c3a
Fix changelog code format issue #272
tdruez Jun 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ Release notes
https://github.com/aboutcode-org/dejacode/pull/315
https://github.com/aboutcode-org/dejacode/pull/312

- Add REST API "actions" in package endpoint to track the scan status and fetch results:
* `/packages/{uuid}/scan_info/` Scan information including the current status.
* `/packages/{uuid}/scan_results/` Scan results.
* `/packages/{uuid}/scan_summary/` Scan summary.
* `/packages/{uuid}/scan_data_download_zip/` Download all scan data: results and
summary, as a zip file.
https://github.com/aboutcode-org/dejacode/issues/272

- Add new `is_locked` "Locked inventory" field to the ProductStatus model.
When a Product is locked through his status, its inventory cannot be modified.
https://github.com/aboutcode-org/dejacode/issues/189
Expand Down
80 changes: 80 additions & 0 deletions component_catalog/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,15 @@

from django.db import transaction
from django.forms.widgets import HiddenInput
from django.http import FileResponse

import django_filters
from packageurl.contrib import url2purl
from packageurl.contrib.django.filters import PackageURLFilter
from rest_framework import serializers
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.exceptions import APIException
from rest_framework.fields import ListField
from rest_framework.response import Response

Expand Down Expand Up @@ -864,6 +867,16 @@ def collect_create_scan(download_url, user):
return package


class ScanCodeUnavailable(APIException):
status_code = status.HTTP_400_BAD_REQUEST
default_detail = "The ScanCode.io service is not available"


class ScanDataUnavailable(APIException):
status_code = status.HTTP_400_BAD_REQUEST
default_detail = "Scan data is not available"


class PackageViewSet(
SendAboutFilesMixin,
AboutCodeFilesActionMixin,
Expand Down Expand Up @@ -919,6 +932,73 @@ def about(self, request, uuid):
package = self.get_object()
return Response({"about_data": package.as_about_yaml()})

def _get_scancodeio_project_info(self, scancodeio, package):
if not scancodeio.is_available():
raise ScanCodeUnavailable()

project_info = scancodeio.get_project_info(download_url=package.download_url)
if not project_info:
raise ScanDataUnavailable()

return project_info

@action(detail=True, name="Scan informations")
def scan_info(self, request, uuid):
"""Return information about the scan from ScanCode.io."""
package = self.get_object()
dataspace = request.user.dataspace
scancodeio = ScanCodeIO(dataspace)
project_info = self._get_scancodeio_project_info(scancodeio, package)

return Response(project_info)

@action(detail=True, name="Scan results")
def scan_results(self, request, uuid):
"""Return the scan results from ScanCode.io."""
package = self.get_object()
dataspace = request.user.dataspace
scancodeio = ScanCodeIO(dataspace)
project_info = self._get_scancodeio_project_info(scancodeio, package)

project_uuid = project_info.get("uuid")
scan_results_url = scancodeio.get_scan_action_url(project_uuid, "results")
scan_results = scancodeio.fetch_scan_data(scan_results_url)

return Response(scan_results)

@action(detail=True, name="Scan summary")
def scan_summary(self, request, uuid):
"""Return the scan summary from ScanCode.io."""
package = self.get_object()
dataspace = request.user.dataspace
scancodeio = ScanCodeIO(dataspace)
project_info = self._get_scancodeio_project_info(scancodeio, package)

project_uuid = project_info.get("uuid")
scan_summary_url = scancodeio.get_scan_action_url(project_uuid, "summary")
scan_summary = scancodeio.fetch_scan_data(scan_summary_url)

return Response(scan_summary)

@action(detail=True, name="Scan data (download as .zip)")
def scan_data_download_zip(self, request, uuid):
"""Download scan data: results and summary, as a zip file."""
package = self.get_object()
dataspace = request.user.dataspace
scancodeio = ScanCodeIO(dataspace)
project_info = self._get_scancodeio_project_info(scancodeio, package)

project_uuid = project_info.get("uuid")
filename = package.filename or package.package_url_filename

scan_data_as_zip = scancodeio.scan_data_as_zip(project_uuid, filename)
return FileResponse(
scan_data_as_zip,
filename=f"{filename}_scan.zip",
as_attachment=True,
content_type="application/zip",
)

@action(detail=False, methods=["post"], name="Package Add")
def add(self, request):
"""
Expand Down
95 changes: 95 additions & 0 deletions component_catalog/tests/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1475,6 +1475,101 @@ def test_api_package_viewset_aboutcode_files_action(self):
'attachment; filename="package1.zip_about.zip"', response["content-disposition"]
)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
def test_api_package_viewset_scan_actions_errors(
self, mock_is_available, mock_get_project_info
):
scan_actions_urls = [
reverse("api_v2:package-scan-info", args=[self.package1.uuid]),
reverse("api_v2:package-scan-results", args=[self.package1.uuid]),
reverse("api_v2:package-scan-summary", args=[self.package1.uuid]),
reverse("api_v2:package-scan-data-download-zip", args=[self.package1.uuid]),
]

mock_get_project_info.return_value = None
for action_url in scan_actions_urls:
response = self.client.get(action_url)
self.assertEqual(403, response.status_code, msg=action_url)
response = self.client.post(action_url)
self.assertEqual(403, response.status_code, msg=action_url)

self.client.login(username=self.base_user.username, password="secret")
mock_is_available.return_value = False
for action_url in scan_actions_urls:
response = self.client.get(action_url)
self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code, msg=action_url)
expected = "The ScanCode.io service is not available"
self.assertEqual(expected, str(response.data["detail"]), msg=action_url)

mock_is_available.return_value = True
for action_url in scan_actions_urls:
response = self.client.get(action_url)
self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code, msg=action_url)
expected = "Scan data is not available"
self.assertEqual(expected, str(response.data["detail"]), msg=action_url)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
def test_api_package_viewset_scan_info_action(self, mock_is_available, mock_get_project_info):
self.client.login(username=self.base_user.username, password="secret")
action_url = reverse("api_v2:package-scan-info", args=[self.package1.uuid])
mock_is_available.return_value = True
project_info = {"uuid": "abcdef"}
mock_get_project_info.return_value = project_info

response = self.client.get(action_url)
self.assertEqual(200, response.status_code)
self.assertEqual(project_info, response.data)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
def test_api_package_viewset_scan_results_action(
self, mock_is_available, mock_fetch_scan_data, mock_get_project_info
):
self.client.login(username=self.base_user.username, password="secret")
action_url = reverse("api_v2:package-scan-results", args=[self.package1.uuid])
mock_is_available.return_value = True
mock_get_project_info.return_value = {"uuid": "abcdef"}
mock_fetch_scan_data.return_value = {"results": ""}
response = self.client.get(action_url)
self.assertEqual(200, response.status_code)
self.assertEqual({"results": ""}, response.data)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
def test_api_package_viewset_scan_summary_action(
self, mock_is_available, mock_fetch_scan_data, mock_get_project_info
):
self.client.login(username=self.base_user.username, password="secret")
action_url = reverse("api_v2:package-scan-summary", args=[self.package1.uuid])
mock_is_available.return_value = True
mock_get_project_info.return_value = {"uuid": "abcdef"}
mock_fetch_scan_data.return_value = {"summary": ""}
response = self.client.get(action_url)
self.assertEqual(200, response.status_code)
self.assertEqual({"summary": ""}, response.data)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
def test_api_package_viewset_scan_data_download_zip_action(
self, mock_is_available, mock_fetch_scan_data, mock_get_project_info
):
self.client.login(username=self.base_user.username, password="secret")
action_url = reverse("api_v2:package-scan-data-download-zip", args=[self.package1.uuid])
mock_is_available.return_value = True
mock_get_project_info.return_value = {"uuid": "abcdef"}
mock_fetch_scan_data.return_value = {}
response = self.client.get(action_url)
self.assertEqual(200, response.status_code)
self.assertEqual("application/zip", response["content-type"])
self.assertEqual(
'attachment; filename="package1.zip_scan.zip"', response["content-disposition"]
)

def test_api_package_protected_fields_as_read_only(self):
policy = UsagePolicy.objects.create(
label="PackagePolicy",
Expand Down
25 changes: 10 additions & 15 deletions component_catalog/tests/test_scancodeio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,23 +113,18 @@ def test_scancodeio_fetch_scan_info(self, mock_session_get):

scancodeio.fetch_scan_info(uri=uri)
params = mock_session_get.call_args.kwargs["params"]
expected = {"format": "json", "name__startswith": get_hash_uid(uri)}
self.assertEqual(expected, params)

scancodeio.fetch_scan_info(
uri=uri,
user=self.basic_user,
dataspace=self.basic_user.dataspace,
)
params = mock_session_get.call_args.kwargs["params"]
expected = {
"format": "json",
"name__startswith": get_hash_uid(uri),
"name__contains": get_hash_uid(self.basic_user.dataspace.uuid),
"name__endswith": get_hash_uid(self.basic_user.uuid),
"format": "json",
}
self.assertEqual(expected, params)

scancodeio.fetch_scan_info(uri=uri, user=self.basic_user)
params = mock_session_get.call_args.kwargs["params"]
expected["name__endswith"] = get_hash_uid(self.basic_user.uuid)
self.assertEqual(expected, params)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.request_get")
def test_scancodeio_find_project(self, mock_request_get):
scancodeio = ScanCodeIO(self.dataspace)
Expand Down Expand Up @@ -166,9 +161,9 @@ def test_scancodeio_find_project(self, mock_request_get):
}
self.assertIsNone(scancodeio.find_project(name="project_name"))

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_scan_results")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_scan_results):
def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_project_info):
license_policy = make_usage_policy(self.dataspace, model=License)
package_policy = make_usage_policy(self.dataspace, model=Package)
make_associated_policy(license_policy, package_policy)
Expand All @@ -180,13 +175,13 @@ def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_scan_r
self.package1.save()
scancodeio = ScanCodeIO(self.dataspace)

mock_get_scan_results.return_value = None
mock_get_project_info.return_value = None
mock_fetch_scan_data.return_value = None

updated_fields = scancodeio.update_from_scan(self.package1, self.super_user)
self.assertEqual([], updated_fields)

mock_get_scan_results.return_value = {"url": "https://scancode.io/"}
mock_get_project_info.return_value = {"url": "https://scancode.io/"}
updated_fields = scancodeio.update_from_scan(self.package1, self.super_user)
self.assertEqual([], updated_fields)

Expand Down
4 changes: 3 additions & 1 deletion component_catalog/tests/test_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2971,7 +2971,9 @@ def test_delete_scan_view(self, mock_fetch_scan_list, mock_delete_scan):
self.assertEqual(404, response.status_code)

@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
def test_send_scan_data_as_file_view(self, mock_fetch_scan_data):
@mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
def test_send_scan_data_as_file_view(self, mock_is_available, mock_fetch_scan_data):
mock_is_available.return_value = True
mock_fetch_scan_data.return_value = {}

project_uuid = "348df847-f48f-4ac7-b864-5785b44c65e2"
Expand Down
Loading